diff options
Diffstat (limited to 'fs/verity')
| -rw-r--r-- | fs/verity/Kconfig | 49 | ||||
| -rw-r--r-- | fs/verity/Makefile | 12 | ||||
| -rw-r--r-- | fs/verity/enable.c | 413 | ||||
| -rw-r--r-- | fs/verity/fsverity_private.h | 171 | ||||
| -rw-r--r-- | fs/verity/hash_algs.c | 188 | ||||
| -rw-r--r-- | fs/verity/init.c | 70 | ||||
| -rw-r--r-- | fs/verity/measure.c | 191 | ||||
| -rw-r--r-- | fs/verity/open.c | 425 | ||||
| -rw-r--r-- | fs/verity/pagecache.c | 80 | ||||
| -rw-r--r-- | fs/verity/read_metadata.c | 204 | ||||
| -rw-r--r-- | fs/verity/signature.c | 138 | ||||
| -rw-r--r-- | fs/verity/verify.c | 499 |
12 files changed, 2440 insertions, 0 deletions
diff --git a/fs/verity/Kconfig b/fs/verity/Kconfig new file mode 100644 index 000000000..b20882963 --- /dev/null +++ b/fs/verity/Kconfig @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0 + +config FS_VERITY + bool "FS Verity (read-only file-based authenticity protection)" + # Filesystems cache the Merkle tree at a 64K aligned offset in the + # pagecache. That approach assumes the page size is at most 64K. + depends on PAGE_SHIFT <= 16 + select CRYPTO_HASH_INFO + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + help + This option enables fs-verity. fs-verity is the dm-verity + mechanism implemented at the file level. On supported + filesystems (currently ext4, f2fs, and btrfs), userspace can + use an ioctl to enable verity for a file, which causes the + filesystem to build a Merkle tree for the file. The filesystem + will then transparently verify any data read from the file + against the Merkle tree. The file is also made read-only. + + This serves as an integrity check, but the availability of the + Merkle tree root hash also allows efficiently supporting + various use cases where normally the whole file would need to + be hashed at once, such as: (a) auditing (logging the file's + hash), or (b) authenticity verification (comparing the hash + against a known good value, e.g. from a digital signature). + + fs-verity is especially useful on large files where not all + the contents may actually be needed. Also, fs-verity verifies + data each time it is paged back in, which provides better + protection against malicious disks vs. an ahead-of-time hash. + + If unsure, say N. + +config FS_VERITY_BUILTIN_SIGNATURES + bool "FS Verity builtin signature support" + depends on FS_VERITY + select SYSTEM_DATA_VERIFICATION + help + This option adds support for in-kernel verification of + fs-verity builtin signatures. + + Please take great care before using this feature. It is not + the only way to do signatures with fs-verity, and the + alternatives (such as userspace signature verification, and + IMA appraisal) can be much better. For details about the + limitations of this feature, see + Documentation/filesystems/fsverity.rst. + + If unsure, say N. diff --git a/fs/verity/Makefile b/fs/verity/Makefile new file mode 100644 index 000000000..ddb4a88a0 --- /dev/null +++ b/fs/verity/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_FS_VERITY) += enable.o \ + hash_algs.o \ + init.o \ + measure.o \ + open.o \ + pagecache.o \ + read_metadata.o \ + verify.o + +obj-$(CONFIG_FS_VERITY_BUILTIN_SIGNATURES) += signature.o diff --git a/fs/verity/enable.c b/fs/verity/enable.c new file mode 100644 index 000000000..42dfed1ce --- /dev/null +++ b/fs/verity/enable.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ioctl to enable verity on a file + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +#include <linux/export.h> +#include <linux/mount.h> +#include <linux/sched/signal.h> +#include <linux/uaccess.h> + +struct block_buffer { + u32 filled; + bool is_root_hash; + u8 *data; +}; + +/* Hash a block, writing the result to the next level's pending block buffer. */ +static int hash_one_block(const struct merkle_tree_params *params, + struct block_buffer *cur) +{ + struct block_buffer *next = cur + 1; + + /* + * Safety check to prevent a buffer overflow in case of a filesystem bug + * that allows the file size to change despite deny_write_access(), or a + * bug in the Merkle tree logic itself + */ + if (WARN_ON_ONCE(next->is_root_hash && next->filled != 0)) + return -EINVAL; + + /* Zero-pad the block if it's shorter than the block size. */ + memset(&cur->data[cur->filled], 0, params->block_size - cur->filled); + + fsverity_hash_block(params, cur->data, &next->data[next->filled]); + next->filled += params->digest_size; + cur->filled = 0; + return 0; +} + +static int write_merkle_tree_block(struct file *file, const u8 *buf, + unsigned long index, + const struct merkle_tree_params *params) +{ + struct inode *inode = file_inode(file); + u64 pos = (u64)index << params->log_blocksize; + int err; + + err = inode->i_sb->s_vop->write_merkle_tree_block(file, buf, pos, + params->block_size); + if (err) + fsverity_err(inode, "Error %d writing Merkle tree block %lu", + err, index); + return err; +} + +/* + * Build the Merkle tree for the given file using the given parameters, and + * return the root hash in @root_hash. + * + * The tree is written to a filesystem-specific location as determined by the + * ->write_merkle_tree_block() method. However, the blocks that comprise the + * tree are the same for all filesystems. + */ +static int build_merkle_tree(struct file *filp, + const struct merkle_tree_params *params, + u8 *root_hash) +{ + struct inode *inode = file_inode(filp); + const u64 data_size = inode->i_size; + const int num_levels = params->num_levels; + struct block_buffer _buffers[1 + FS_VERITY_MAX_LEVELS + 1] = {}; + struct block_buffer *buffers = &_buffers[1]; + unsigned long level_offset[FS_VERITY_MAX_LEVELS]; + int level; + u64 offset; + int err; + + if (data_size == 0) { + /* Empty file is a special case; root hash is all 0's */ + memset(root_hash, 0, params->digest_size); + return 0; + } + + /* + * Allocate the block buffers. Buffer "-1" is for data blocks. + * Buffers 0 <= level < num_levels are for the actual tree levels. + * Buffer 'num_levels' is for the root hash. + */ + for (level = -1; level < num_levels; level++) { + buffers[level].data = kzalloc(params->block_size, GFP_KERNEL); + if (!buffers[level].data) { + err = -ENOMEM; + goto out; + } + } + buffers[num_levels].data = root_hash; + buffers[num_levels].is_root_hash = true; + + BUILD_BUG_ON(sizeof(level_offset) != sizeof(params->level_start)); + memcpy(level_offset, params->level_start, sizeof(level_offset)); + + /* Hash each data block, also hashing the tree blocks as they fill up */ + for (offset = 0; offset < data_size; offset += params->block_size) { + ssize_t bytes_read; + loff_t pos = offset; + + buffers[-1].filled = min_t(u64, params->block_size, + data_size - offset); + bytes_read = __kernel_read(filp, buffers[-1].data, + buffers[-1].filled, &pos); + if (bytes_read < 0) { + err = bytes_read; + fsverity_err(inode, "Error %d reading file data", err); + goto out; + } + if (bytes_read != buffers[-1].filled) { + err = -EINVAL; + fsverity_err(inode, "Short read of file data"); + goto out; + } + err = hash_one_block(params, &buffers[-1]); + if (err) + goto out; + for (level = 0; level < num_levels; level++) { + if (buffers[level].filled + params->digest_size <= + params->block_size) { + /* Next block at @level isn't full yet */ + break; + } + /* Next block at @level is full */ + + err = hash_one_block(params, &buffers[level]); + if (err) + goto out; + err = write_merkle_tree_block(filp, + buffers[level].data, + level_offset[level], + params); + if (err) + goto out; + level_offset[level]++; + } + if (fatal_signal_pending(current)) { + err = -EINTR; + goto out; + } + cond_resched(); + } + /* Finish all nonempty pending tree blocks. */ + for (level = 0; level < num_levels; level++) { + if (buffers[level].filled != 0) { + err = hash_one_block(params, &buffers[level]); + if (err) + goto out; + err = write_merkle_tree_block(filp, + buffers[level].data, + level_offset[level], + params); + if (err) + goto out; + } + } + /* The root hash was filled by the last call to hash_one_block(). */ + if (WARN_ON_ONCE(buffers[num_levels].filled != params->digest_size)) { + err = -EINVAL; + goto out; + } + err = 0; +out: + for (level = -1; level < num_levels; level++) + kfree(buffers[level].data); + return err; +} + +static int enable_verity(struct file *filp, + const struct fsverity_enable_arg *arg) +{ + struct inode *inode = file_inode(filp); + const struct fsverity_operations *vops = inode->i_sb->s_vop; + struct merkle_tree_params params = { }; + struct fsverity_descriptor *desc; + size_t desc_size = struct_size(desc, signature, arg->sig_size); + struct fsverity_info *vi; + int err; + + /* Start initializing the fsverity_descriptor */ + desc = kzalloc(desc_size, GFP_KERNEL); + if (!desc) + return -ENOMEM; + desc->version = 1; + desc->hash_algorithm = arg->hash_algorithm; + desc->log_blocksize = ilog2(arg->block_size); + + /* Get the salt if the user provided one */ + if (arg->salt_size && + copy_from_user(desc->salt, u64_to_user_ptr(arg->salt_ptr), + arg->salt_size)) { + err = -EFAULT; + goto out; + } + desc->salt_size = arg->salt_size; + + /* Get the builtin signature if the user provided one */ + if (arg->sig_size && + copy_from_user(desc->signature, u64_to_user_ptr(arg->sig_ptr), + arg->sig_size)) { + err = -EFAULT; + goto out; + } + desc->sig_size = cpu_to_le32(arg->sig_size); + + desc->data_size = cpu_to_le64(inode->i_size); + + /* Prepare the Merkle tree parameters */ + err = fsverity_init_merkle_tree_params(¶ms, inode, + arg->hash_algorithm, + desc->log_blocksize, + desc->salt, desc->salt_size); + if (err) + goto out; + + trace_fsverity_enable(inode, ¶ms); + + /* + * Start enabling verity on this file, serialized by the inode lock. + * Fail if verity is already enabled or is already being enabled. + */ + inode_lock(inode); + if (IS_VERITY(inode)) + err = -EEXIST; + else + err = vops->begin_enable_verity(filp); + inode_unlock(inode); + if (err) + goto out; + + /* + * Build the Merkle tree. Don't hold the inode lock during this, since + * on huge files this may take a very long time and we don't want to + * force unrelated syscalls like chown() to block forever. We don't + * need the inode lock here because deny_write_access() already prevents + * the file from being written to or truncated, and we still serialize + * ->begin_enable_verity() and ->end_enable_verity() using the inode + * lock and only allow one process to be here at a time on a given file. + */ + BUILD_BUG_ON(sizeof(desc->root_hash) < FS_VERITY_MAX_DIGEST_SIZE); + err = build_merkle_tree(filp, ¶ms, desc->root_hash); + if (err) { + fsverity_err(inode, "Error %d building Merkle tree", err); + goto rollback; + } + + /* + * Create the fsverity_info. Don't bother trying to save work by + * reusing the merkle_tree_params from above. Instead, just create the + * fsverity_info from the fsverity_descriptor as if it were just loaded + * from disk. This is simpler, and it serves as an extra check that the + * metadata we're writing is valid before actually enabling verity. + */ + vi = fsverity_create_info(inode, desc); + if (IS_ERR(vi)) { + err = PTR_ERR(vi); + goto rollback; + } + + trace_fsverity_tree_done(inode, vi, ¶ms); + + /* + * Add the fsverity_info into the hash table before finishing the + * initialization so that we don't have to undo the enabling when memory + * allocation for the hash table fails. This is safe because looking up + * the fsverity_info always first checks the S_VERITY flag on the inode, + * which will only be set at the very end of the ->end_enable_verity + * method. + */ + err = fsverity_set_info(vi); + if (err) { + fsverity_free_info(vi); + goto rollback; + } + + /* + * Tell the filesystem to finish enabling verity on the file. + * Serialized with ->begin_enable_verity() by the inode lock. The file + * system needs to set the S_VERITY flag on the inode at the very end of + * the method, at which point the fsverity information can be accessed + * by other threads. + */ + inode_lock(inode); + err = vops->end_enable_verity(filp, desc, desc_size, params.tree_size); + inode_unlock(inode); + if (err) { + fsverity_err(inode, "%ps() failed with err %d", + vops->end_enable_verity, err); + fsverity_remove_info(vi); + } else if (WARN_ON_ONCE(!IS_VERITY(inode))) { + fsverity_remove_info(vi); + err = -EINVAL; + } +out: + kfree(params.hashstate); + kfree(desc); + return err; + +rollback: + inode_lock(inode); + (void)vops->end_enable_verity(filp, NULL, 0, params.tree_size); + inode_unlock(inode); + goto out; +} + +/** + * fsverity_ioctl_enable() - enable verity on a file + * @filp: file to enable verity on + * @uarg: user pointer to fsverity_enable_arg + * + * Enable fs-verity on a file. See the "FS_IOC_ENABLE_VERITY" section of + * Documentation/filesystems/fsverity.rst for the documentation. + * + * Return: 0 on success, -errno on failure + */ +int fsverity_ioctl_enable(struct file *filp, const void __user *uarg) +{ + struct inode *inode = file_inode(filp); + struct fsverity_enable_arg arg; + int err; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (arg.version != 1) + return -EINVAL; + + if (arg.__reserved1 || + memchr_inv(arg.__reserved2, 0, sizeof(arg.__reserved2))) + return -EINVAL; + + if (!is_power_of_2(arg.block_size)) + return -EINVAL; + + if (arg.salt_size > sizeof_field(struct fsverity_descriptor, salt)) + return -EMSGSIZE; + + if (arg.sig_size > FS_VERITY_MAX_SIGNATURE_SIZE) + return -EMSGSIZE; + + /* + * Require a regular file with write access. But the actual fd must + * still be readonly so that we can lock out all writers. This is + * needed to guarantee that no writable fds exist to the file once it + * has verity enabled, and to stabilize the data being hashed. + */ + + err = file_permission(filp, MAY_WRITE); + if (err) + return err; + /* + * __kernel_read() is used while building the Merkle tree. So, we can't + * allow file descriptors that were opened for ioctl access only, using + * the special nonstandard access mode 3. O_RDONLY only, please! + */ + if (!(filp->f_mode & FMODE_READ)) + return -EBADF; + + if (IS_APPEND(inode)) + return -EPERM; + + if (S_ISDIR(inode->i_mode)) + return -EISDIR; + + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + err = mnt_want_write_file(filp); + if (err) /* -EROFS */ + return err; + + err = deny_write_access(filp); + if (err) /* -ETXTBSY */ + goto out_drop_write; + + err = enable_verity(filp, &arg); + + /* + * We no longer drop the inode's pagecache after enabling verity. This + * used to be done to try to avoid a race condition where pages could be + * evicted after being used in the Merkle tree construction, then + * re-instantiated by a concurrent read. Such pages are unverified, and + * the backing storage could have filled them with different content, so + * they shouldn't be used to fulfill reads once verity is enabled. + * + * But, dropping the pagecache has a big performance impact, and it + * doesn't fully solve the race condition anyway. So for those reasons, + * and also because this race condition isn't very important relatively + * speaking (especially for small-ish files, where the chance of a page + * being used, evicted, *and* re-instantiated all while enabling verity + * is quite small), we no longer drop the inode's pagecache. + */ + + /* + * allow_write_access() is needed to pair with deny_write_access(). + * Regardless, the filesystem won't allow writing to verity files. + */ + allow_write_access(filp); +out_drop_write: + mnt_drop_write_file(filp); + return err; +} +EXPORT_SYMBOL_GPL(fsverity_ioctl_enable); diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h new file mode 100644 index 000000000..881d46f25 --- /dev/null +++ b/fs/verity/fsverity_private.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs-verity: read-only file-based authenticity protection + * + * Copyright 2019 Google LLC + */ + +#ifndef _FSVERITY_PRIVATE_H +#define _FSVERITY_PRIVATE_H + +#define pr_fmt(fmt) "fs-verity: " fmt + +#include <linux/fsverity.h> +#include <linux/rhashtable.h> + +/* + * Implementation limit: maximum depth of the Merkle tree. For now 8 is plenty; + * it's enough for over U64_MAX bytes of data using SHA-256 and 4K blocks. + */ +#define FS_VERITY_MAX_LEVELS 8 + +/* A hash algorithm supported by fs-verity */ +struct fsverity_hash_alg { + const char *name; /* crypto API name, e.g. sha256 */ + unsigned int digest_size; /* digest size in bytes, e.g. 32 for SHA-256 */ + unsigned int block_size; /* block size in bytes, e.g. 64 for SHA-256 */ + /* + * The HASH_ALGO_* constant for this algorithm. This is different from + * FS_VERITY_HASH_ALG_*, which uses a different numbering scheme. + */ + enum hash_algo algo_id; +}; + +union fsverity_hash_ctx { + struct sha256_ctx sha256; + struct sha512_ctx sha512; +}; + +/* Merkle tree parameters: hash algorithm, initial hash state, and topology */ +struct merkle_tree_params { + const struct fsverity_hash_alg *hash_alg; /* the hash algorithm */ + /* initial hash state if salted, NULL if unsalted */ + const union fsverity_hash_ctx *hashstate; + unsigned int digest_size; /* same as hash_alg->digest_size */ + unsigned int block_size; /* size of data and tree blocks */ + unsigned int hashes_per_block; /* number of hashes per tree block */ + unsigned int blocks_per_page; /* PAGE_SIZE / block_size */ + u8 log_digestsize; /* log2(digest_size) */ + u8 log_blocksize; /* log2(block_size) */ + u8 log_arity; /* log2(hashes_per_block) */ + u8 log_blocks_per_page; /* log2(blocks_per_page) */ + unsigned int num_levels; /* number of levels in Merkle tree */ + u64 tree_size; /* Merkle tree size in bytes */ + unsigned long tree_pages; /* Merkle tree size in pages */ + + /* the hash of an all-zeroes block */ + u8 zero_digest[FS_VERITY_MAX_DIGEST_SIZE]; + + /* + * Starting block index for each tree level, ordered from leaf level (0) + * to root level ('num_levels - 1') + */ + unsigned long level_start[FS_VERITY_MAX_LEVELS]; +}; + +/* + * fsverity_info - cached verity metadata for an inode + * + * When a verity file is first opened, an instance of this struct is allocated + * and a pointer to it is stored in the global hash table, indexed by the inode + * pointer value. It remains alive until the inode is evicted. It caches + * information about the Merkle tree that's needed to efficiently verify data + * read from the file. It also caches the file digest. The Merkle tree pages + * themselves are not cached here, but the filesystem may cache them. + */ +struct fsverity_info { + struct rhash_head rhash_head; + struct merkle_tree_params tree_params; + u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE]; + u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE]; + struct inode *inode; + unsigned long *hash_block_verified; +}; + +#define FS_VERITY_MAX_SIGNATURE_SIZE (FS_VERITY_MAX_DESCRIPTOR_SIZE - \ + sizeof(struct fsverity_descriptor)) + +/* hash_algs.c */ + +extern const struct fsverity_hash_alg fsverity_hash_algs[]; + +const struct fsverity_hash_alg *fsverity_get_hash_alg(const struct inode *inode, + unsigned int num); +union fsverity_hash_ctx * +fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg, + const u8 *salt, size_t salt_size); +void fsverity_hash_block(const struct merkle_tree_params *params, + const void *data, u8 *out); +void fsverity_hash_buffer(const struct fsverity_hash_alg *alg, + const void *data, size_t size, u8 *out); +void __init fsverity_check_hash_algs(void); + +/* init.c */ + +void __printf(3, 4) __cold +fsverity_msg(const struct inode *inode, const char *level, + const char *fmt, ...); + +#define fsverity_warn(inode, fmt, ...) \ + fsverity_msg((inode), KERN_WARNING, fmt, ##__VA_ARGS__) +#define fsverity_err(inode, fmt, ...) \ + fsverity_msg((inode), KERN_ERR, fmt, ##__VA_ARGS__) + +/* measure.c */ + +#ifdef CONFIG_BPF_SYSCALL +void __init fsverity_init_bpf(void); +#else +static inline void fsverity_init_bpf(void) +{ +} +#endif + +/* open.c */ + +int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, + const struct inode *inode, + unsigned int hash_algorithm, + unsigned int log_blocksize, + const u8 *salt, size_t salt_size); + +struct fsverity_info *fsverity_create_info(struct inode *inode, + struct fsverity_descriptor *desc); + +int fsverity_set_info(struct fsverity_info *vi); +void fsverity_free_info(struct fsverity_info *vi); +void fsverity_remove_info(struct fsverity_info *vi); + +int fsverity_get_descriptor(struct inode *inode, + struct fsverity_descriptor **desc_ret); + +void __init fsverity_init_info_cache(void); + +/* signature.c */ + +#ifdef CONFIG_FS_VERITY_BUILTIN_SIGNATURES +extern int fsverity_require_signatures; +int fsverity_verify_signature(const struct fsverity_info *vi, + const u8 *signature, size_t sig_size); + +void __init fsverity_init_signature(void); +#else /* !CONFIG_FS_VERITY_BUILTIN_SIGNATURES */ +static inline int +fsverity_verify_signature(const struct fsverity_info *vi, + const u8 *signature, size_t sig_size) +{ + return 0; +} + +static inline void fsverity_init_signature(void) +{ +} +#endif /* !CONFIG_FS_VERITY_BUILTIN_SIGNATURES */ + +/* verify.c */ + +void __init fsverity_init_workqueue(void); + +#include <trace/events/fsverity.h> + +#endif /* _FSVERITY_PRIVATE_H */ diff --git a/fs/verity/hash_algs.c b/fs/verity/hash_algs.c new file mode 100644 index 000000000..de53e14c8 --- /dev/null +++ b/fs/verity/hash_algs.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs-verity hash algorithms + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +/* The hash algorithms supported by fs-verity */ +const struct fsverity_hash_alg fsverity_hash_algs[] = { + [FS_VERITY_HASH_ALG_SHA256] = { + .name = "sha256", + .digest_size = SHA256_DIGEST_SIZE, + .block_size = SHA256_BLOCK_SIZE, + .algo_id = HASH_ALGO_SHA256, + }, + [FS_VERITY_HASH_ALG_SHA512] = { + .name = "sha512", + .digest_size = SHA512_DIGEST_SIZE, + .block_size = SHA512_BLOCK_SIZE, + .algo_id = HASH_ALGO_SHA512, + }, +}; + +/** + * fsverity_get_hash_alg() - get a hash algorithm by number + * @inode: optional inode for logging purposes + * @num: the hash algorithm number + * + * Get the struct fsverity_hash_alg for the given hash algorithm number. + * + * Return: pointer to the hash alg if it's known, otherwise NULL. + */ +const struct fsverity_hash_alg *fsverity_get_hash_alg(const struct inode *inode, + unsigned int num) +{ + if (num >= ARRAY_SIZE(fsverity_hash_algs) || + !fsverity_hash_algs[num].name) { + fsverity_warn(inode, "Unknown hash algorithm number: %u", num); + return NULL; + } + return &fsverity_hash_algs[num]; +} + +/** + * fsverity_prepare_hash_state() - precompute the initial hash state + * @alg: hash algorithm + * @salt: a salt which is to be prepended to all data to be hashed + * @salt_size: salt size in bytes + * + * Return: the kmalloc()'ed initial hash state, or NULL if out of memory. + */ +union fsverity_hash_ctx * +fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg, + const u8 *salt, size_t salt_size) +{ + u8 *padded_salt = NULL; + size_t padded_salt_size; + union fsverity_hash_ctx ctx; + void *res = NULL; + + /* + * Zero-pad the salt to the next multiple of the input size of the hash + * algorithm's compression function, e.g. 64 bytes for SHA-256 or 128 + * bytes for SHA-512. This ensures that the hash algorithm won't have + * any bytes buffered internally after processing the salt, thus making + * salted hashing just as fast as unsalted hashing. + */ + padded_salt_size = round_up(salt_size, alg->block_size); + padded_salt = kzalloc(padded_salt_size, GFP_KERNEL); + if (!padded_salt) + return NULL; + memcpy(padded_salt, salt, salt_size); + + switch (alg->algo_id) { + case HASH_ALGO_SHA256: + sha256_init(&ctx.sha256); + sha256_update(&ctx.sha256, padded_salt, padded_salt_size); + res = kmemdup(&ctx.sha256, sizeof(ctx.sha256), GFP_KERNEL); + break; + case HASH_ALGO_SHA512: + sha512_init(&ctx.sha512); + sha512_update(&ctx.sha512, padded_salt, padded_salt_size); + res = kmemdup(&ctx.sha512, sizeof(ctx.sha512), GFP_KERNEL); + break; + default: + WARN_ON_ONCE(1); + } + kfree(padded_salt); + return res; +} + +/** + * fsverity_hash_block() - hash a single data or hash block + * @params: the Merkle tree's parameters + * @data: virtual address of a buffer containing the block to hash + * @out: output digest, size 'params->digest_size' bytes + * + * Hash a single data or hash block. The hash is salted if a salt is specified + * in the Merkle tree parameters. + */ +void fsverity_hash_block(const struct merkle_tree_params *params, + const void *data, u8 *out) +{ + union fsverity_hash_ctx ctx; + + if (!params->hashstate) { + fsverity_hash_buffer(params->hash_alg, data, params->block_size, + out); + return; + } + + switch (params->hash_alg->algo_id) { + case HASH_ALGO_SHA256: + ctx.sha256 = params->hashstate->sha256; + sha256_update(&ctx.sha256, data, params->block_size); + sha256_final(&ctx.sha256, out); + return; + case HASH_ALGO_SHA512: + ctx.sha512 = params->hashstate->sha512; + sha512_update(&ctx.sha512, data, params->block_size); + sha512_final(&ctx.sha512, out); + return; + default: + BUG(); + } +} + +/** + * fsverity_hash_buffer() - hash some data + * @alg: the hash algorithm to use + * @data: the data to hash + * @size: size of data to hash, in bytes + * @out: output digest, size 'alg->digest_size' bytes + */ +void fsverity_hash_buffer(const struct fsverity_hash_alg *alg, + const void *data, size_t size, u8 *out) +{ + switch (alg->algo_id) { + case HASH_ALGO_SHA256: + sha256(data, size, out); + return; + case HASH_ALGO_SHA512: + sha512(data, size, out); + return; + default: + BUG(); + } +} + +void __init fsverity_check_hash_algs(void) +{ + size_t i; + + /* + * Sanity check the hash algorithms (could be a build-time check, but + * they're in an array) + */ + for (i = 0; i < ARRAY_SIZE(fsverity_hash_algs); i++) { + const struct fsverity_hash_alg *alg = &fsverity_hash_algs[i]; + + if (!alg->name) + continue; + + /* + * 0 must never be allocated as an FS_VERITY_HASH_ALG_* value, + * as it is reserved for users that use 0 to mean unspecified or + * a default value. fs/verity/ itself doesn't care and doesn't + * have a default algorithm, but some users make use of this. + */ + BUG_ON(i == 0); + + BUG_ON(alg->digest_size > FS_VERITY_MAX_DIGEST_SIZE); + + /* + * For efficiency, the implementation currently assumes the + * digest and block sizes are powers of 2. This limitation can + * be lifted if the code is updated to handle other values. + */ + BUG_ON(!is_power_of_2(alg->digest_size)); + BUG_ON(!is_power_of_2(alg->block_size)); + + /* Verify that there is a valid mapping to HASH_ALGO_*. */ + BUG_ON(alg->algo_id == 0); + BUG_ON(alg->digest_size != hash_digest_size[alg->algo_id]); + } +} diff --git a/fs/verity/init.c b/fs/verity/init.c new file mode 100644 index 000000000..3aa55dec8 --- /dev/null +++ b/fs/verity/init.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs-verity module initialization and logging + * + * Copyright 2019 Google LLC + */ + +#define CREATE_TRACE_POINTS +#include "fsverity_private.h" + +#include <linux/ratelimit.h> + +#ifdef CONFIG_SYSCTL +static const struct ctl_table fsverity_sysctl_table[] = { +#ifdef CONFIG_FS_VERITY_BUILTIN_SIGNATURES + { + .procname = "require_signatures", + .data = &fsverity_require_signatures, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +#endif +}; + +static void __init fsverity_init_sysctl(void) +{ + register_sysctl_init("fs/verity", fsverity_sysctl_table); +} +#else /* CONFIG_SYSCTL */ +static inline void fsverity_init_sysctl(void) +{ +} +#endif /* !CONFIG_SYSCTL */ + +void fsverity_msg(const struct inode *inode, const char *level, + const char *fmt, ...) +{ + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + struct va_format vaf; + va_list args; + + if (!__ratelimit(&rs)) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (inode) + printk("%sfs-verity (%s, inode %llu): %pV\n", + level, inode->i_sb->s_id, inode->i_ino, &vaf); + else + printk("%sfs-verity: %pV\n", level, &vaf); + va_end(args); +} + +static int __init fsverity_init(void) +{ + fsverity_check_hash_algs(); + fsverity_init_info_cache(); + fsverity_init_workqueue(); + fsverity_init_sysctl(); + fsverity_init_signature(); + fsverity_init_bpf(); + return 0; +} +late_initcall(fsverity_init) diff --git a/fs/verity/measure.c b/fs/verity/measure.c new file mode 100644 index 000000000..68dfccb69 --- /dev/null +++ b/fs/verity/measure.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ioctl to get a verity file's digest + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +#include <linux/bpf.h> +#include <linux/btf.h> +#include <linux/export.h> +#include <linux/uaccess.h> + +/** + * fsverity_ioctl_measure() - get a verity file's digest + * @filp: file to get digest of + * @_uarg: user pointer to fsverity_digest + * + * Retrieve the file digest that the kernel is enforcing for reads from a verity + * file. See the "FS_IOC_MEASURE_VERITY" section of + * Documentation/filesystems/fsverity.rst for the documentation. + * + * Return: 0 on success, -errno on failure + */ +int fsverity_ioctl_measure(struct file *filp, void __user *_uarg) +{ + const struct inode *inode = file_inode(filp); + struct fsverity_digest __user *uarg = _uarg; + const struct fsverity_info *vi; + const struct fsverity_hash_alg *hash_alg; + struct fsverity_digest arg; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + hash_alg = vi->tree_params.hash_alg; + + /* + * The user specifies the digest_size their buffer has space for; we can + * return the digest if it fits in the available space. We write back + * the actual size, which may be shorter than the user-specified size. + */ + + if (get_user(arg.digest_size, &uarg->digest_size)) + return -EFAULT; + if (arg.digest_size < hash_alg->digest_size) + return -EOVERFLOW; + + memset(&arg, 0, sizeof(arg)); + arg.digest_algorithm = hash_alg - fsverity_hash_algs; + arg.digest_size = hash_alg->digest_size; + + if (copy_to_user(uarg, &arg, sizeof(arg))) + return -EFAULT; + + if (copy_to_user(uarg->digest, vi->file_digest, hash_alg->digest_size)) + return -EFAULT; + + return 0; +} +EXPORT_SYMBOL_GPL(fsverity_ioctl_measure); + +/** + * fsverity_get_digest() - get a verity file's digest + * @inode: inode to get digest of + * @raw_digest: (out) the raw file digest + * @alg: (out) the digest's algorithm, as a FS_VERITY_HASH_ALG_* value + * @halg: (out) the digest's algorithm, as a HASH_ALGO_* value + * + * Retrieves the fsverity digest of the given file. The + * fsverity_ensure_verity_info() must be called on the inode beforehand; + * otherwise this function will not recognize when fsverity is enabled. + * + * The file's fsverity digest consists of @raw_digest in combination with either + * @alg or @halg. (The caller can choose which one of @alg or @halg to use.) + * + * IMPORTANT: Callers *must* make use of one of the two algorithm IDs, since + * @raw_digest is meaningless without knowing which algorithm it uses! fsverity + * provides no security guarantee for users who ignore the algorithm ID, even if + * they use the digest size (since algorithms can share the same digest size). + * + * Return: The size of the raw digest in bytes, or 0 if the file doesn't have + * fsverity enabled. + */ +int fsverity_get_digest(struct inode *inode, + u8 raw_digest[FS_VERITY_MAX_DIGEST_SIZE], + u8 *alg, enum hash_algo *halg) +{ + const struct fsverity_info *vi; + const struct fsverity_hash_alg *hash_alg; + + vi = fsverity_get_info(inode); + if (!vi) + return 0; /* not a verity file */ + + hash_alg = vi->tree_params.hash_alg; + memcpy(raw_digest, vi->file_digest, hash_alg->digest_size); + if (alg) + *alg = hash_alg - fsverity_hash_algs; + if (halg) + *halg = hash_alg->algo_id; + return hash_alg->digest_size; +} +EXPORT_SYMBOL_GPL(fsverity_get_digest); + +#ifdef CONFIG_BPF_SYSCALL + +/* bpf kfuncs */ +__bpf_kfunc_start_defs(); + +/** + * bpf_get_fsverity_digest: read fsverity digest of file + * @file: file to get digest from + * @digest_p: (out) dynptr for struct fsverity_digest + * + * Read fsverity_digest of *file* into *digest_ptr*. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_get_fsverity_digest(struct file *file, const struct bpf_dynptr *digest_p) +{ + const struct bpf_dynptr_kern *digest_ptr = (struct bpf_dynptr_kern *)digest_p; + const struct inode *inode = file_inode(file); + u64 dynptr_sz = __bpf_dynptr_size(digest_ptr); + struct fsverity_digest *arg; + const struct fsverity_info *vi; + const struct fsverity_hash_alg *hash_alg; + u64 out_digest_sz; + + if (dynptr_sz < sizeof(struct fsverity_digest)) + return -EINVAL; + + arg = __bpf_dynptr_data_rw(digest_ptr, dynptr_sz); + if (!arg) + return -EINVAL; + + if (!IS_ALIGNED((uintptr_t)arg, __alignof__(*arg))) + return -EINVAL; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + + hash_alg = vi->tree_params.hash_alg; + + out_digest_sz = dynptr_sz - sizeof(struct fsverity_digest); + if (out_digest_sz < hash_alg->digest_size) + return -EOVERFLOW; + + arg->digest_algorithm = hash_alg - fsverity_hash_algs; + arg->digest_size = hash_alg->digest_size; + + /* copy digest */ + memcpy(arg->digest, vi->file_digest, hash_alg->digest_size); + + /* fill the extra buffer with zeros */ + if (out_digest_sz > hash_alg->digest_size) + memset(arg->digest + hash_alg->digest_size, 0, + out_digest_sz - hash_alg->digest_size); + + return 0; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(fsverity_set_ids) +BTF_ID_FLAGS(func, bpf_get_fsverity_digest) +BTF_KFUNCS_END(fsverity_set_ids) + +static int bpf_get_fsverity_digest_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + if (!btf_id_set8_contains(&fsverity_set_ids, kfunc_id)) + return 0; + + /* Only allow to attach from LSM hooks, to avoid recursion */ + return prog->type != BPF_PROG_TYPE_LSM ? -EACCES : 0; +} + +static const struct btf_kfunc_id_set bpf_fsverity_set = { + .owner = THIS_MODULE, + .set = &fsverity_set_ids, + .filter = bpf_get_fsverity_digest_filter, +}; + +void __init fsverity_init_bpf(void) +{ + register_btf_kfunc_id_set(BPF_PROG_TYPE_LSM, &bpf_fsverity_set); +} + +#endif /* CONFIG_BPF_SYSCALL */ diff --git a/fs/verity/open.c b/fs/verity/open.c new file mode 100644 index 000000000..d0c56a7fa --- /dev/null +++ b/fs/verity/open.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Opening fs-verity files + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +#include <linux/export.h> +#include <linux/mm.h> +#include <linux/slab.h> + +static struct kmem_cache *fsverity_info_cachep; +static struct rhashtable fsverity_info_hash; + +static const struct rhashtable_params fsverity_info_hash_params = { + .key_len = sizeof_field(struct fsverity_info, inode), + .key_offset = offsetof(struct fsverity_info, inode), + .head_offset = offsetof(struct fsverity_info, rhash_head), + .automatic_shrinking = true, +}; + +/** + * fsverity_init_merkle_tree_params() - initialize Merkle tree parameters + * @params: the parameters struct to initialize + * @inode: the inode for which the Merkle tree is being built + * @hash_algorithm: number of hash algorithm to use + * @log_blocksize: log base 2 of block size to use + * @salt: pointer to salt (optional) + * @salt_size: size of salt, possibly 0 + * + * Validate the hash algorithm and block size, then compute the tree topology + * (num levels, num blocks in each level, etc.) and initialize @params. + * + * Return: 0 on success, -errno on failure + */ +int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, + const struct inode *inode, + unsigned int hash_algorithm, + unsigned int log_blocksize, + const u8 *salt, size_t salt_size) +{ + const struct fsverity_hash_alg *hash_alg; + int err; + u64 blocks; + u64 blocks_in_level[FS_VERITY_MAX_LEVELS]; + u64 offset; + int level; + + memset(params, 0, sizeof(*params)); + + hash_alg = fsverity_get_hash_alg(inode, hash_algorithm); + if (!hash_alg) + return -EINVAL; + params->hash_alg = hash_alg; + params->digest_size = hash_alg->digest_size; + + if (salt_size) { + params->hashstate = + fsverity_prepare_hash_state(hash_alg, salt, salt_size); + if (!params->hashstate) { + err = -ENOMEM; + goto out_err; + } + } + + /* + * fs/verity/ directly assumes that the Merkle tree block size is a + * power of 2 less than or equal to PAGE_SIZE. Another restriction + * arises from the interaction between fs/verity/ and the filesystems + * themselves: filesystems expect to be able to verify a single + * filesystem block of data at a time. Therefore, the Merkle tree block + * size must also be less than or equal to the filesystem block size. + * + * The above are the only hard limitations, so in theory the Merkle tree + * block size could be as small as twice the digest size. However, + * that's not useful, and it would result in some unusually deep and + * large Merkle trees. So we currently require that the Merkle tree + * block size be at least 1024 bytes. That's small enough to test the + * sub-page block case on systems with 4K pages, but not too small. + */ + if (log_blocksize < 10 || log_blocksize > PAGE_SHIFT || + log_blocksize > inode->i_blkbits) { + fsverity_warn(inode, "Unsupported log_blocksize: %u", + log_blocksize); + err = -EINVAL; + goto out_err; + } + params->log_blocksize = log_blocksize; + params->block_size = 1 << log_blocksize; + params->log_blocks_per_page = PAGE_SHIFT - log_blocksize; + params->blocks_per_page = 1 << params->log_blocks_per_page; + + if (WARN_ON_ONCE(!is_power_of_2(params->digest_size))) { + err = -EINVAL; + goto out_err; + } + if (params->block_size < 2 * params->digest_size) { + fsverity_warn(inode, + "Merkle tree block size (%u) too small for hash algorithm \"%s\"", + params->block_size, hash_alg->name); + err = -EINVAL; + goto out_err; + } + params->log_digestsize = ilog2(params->digest_size); + params->log_arity = log_blocksize - params->log_digestsize; + params->hashes_per_block = 1 << params->log_arity; + + /* + * Compute the number of levels in the Merkle tree and create a map from + * level to the starting block of that level. Level 'num_levels - 1' is + * the root and is stored first. Level 0 is the level directly "above" + * the data blocks and is stored last. + */ + + /* Compute number of levels and the number of blocks in each level */ + blocks = ((u64)inode->i_size + params->block_size - 1) >> log_blocksize; + while (blocks > 1) { + if (params->num_levels >= FS_VERITY_MAX_LEVELS) { + fsverity_err(inode, "Too many levels in Merkle tree"); + err = -EFBIG; + goto out_err; + } + blocks = (blocks + params->hashes_per_block - 1) >> + params->log_arity; + blocks_in_level[params->num_levels++] = blocks; + } + + /* Compute the starting block of each level */ + offset = 0; + for (level = (int)params->num_levels - 1; level >= 0; level--) { + params->level_start[level] = offset; + offset += blocks_in_level[level]; + } + + /* + * With block_size != PAGE_SIZE, an in-memory bitmap will need to be + * allocated to track the "verified" status of hash blocks. Don't allow + * this bitmap to get too large. For now, limit it to 1 MiB, which + * limits the file size to about 4.4 TB with SHA-256 and 4K blocks. + * + * Together with the fact that the data, and thus also the Merkle tree, + * cannot have more than ULONG_MAX pages, this implies that hash block + * indices can always fit in an 'unsigned long'. But to be safe, we + * explicitly check for that too. Note, this is only for hash block + * indices; data block indices might not fit in an 'unsigned long'. + */ + if ((params->block_size != PAGE_SIZE && offset > 1 << 23) || + offset > ULONG_MAX) { + fsverity_err(inode, "Too many blocks in Merkle tree"); + err = -EFBIG; + goto out_err; + } + + fsverity_hash_block(params, page_address(ZERO_PAGE(0)), + params->zero_digest); + + params->tree_size = offset << log_blocksize; + params->tree_pages = PAGE_ALIGN(params->tree_size) >> PAGE_SHIFT; + return 0; + +out_err: + kfree(params->hashstate); + memset(params, 0, sizeof(*params)); + return err; +} + +/* + * Compute the file digest by hashing the fsverity_descriptor excluding the + * builtin signature and with the sig_size field set to 0. + */ +static void compute_file_digest(const struct fsverity_hash_alg *hash_alg, + struct fsverity_descriptor *desc, + u8 *file_digest) +{ + __le32 sig_size = desc->sig_size; + + desc->sig_size = 0; + fsverity_hash_buffer(hash_alg, desc, sizeof(*desc), file_digest); + desc->sig_size = sig_size; +} + +/* + * Create a new fsverity_info from the given fsverity_descriptor (with optional + * appended builtin signature), and check the signature if present. The + * fsverity_descriptor must have already undergone basic validation. + */ +struct fsverity_info *fsverity_create_info(struct inode *inode, + struct fsverity_descriptor *desc) +{ + struct fsverity_info *vi; + int err; + + vi = kmem_cache_zalloc(fsverity_info_cachep, GFP_KERNEL); + if (!vi) + return ERR_PTR(-ENOMEM); + vi->inode = inode; + + err = fsverity_init_merkle_tree_params(&vi->tree_params, inode, + desc->hash_algorithm, + desc->log_blocksize, + desc->salt, desc->salt_size); + if (err) { + fsverity_err(inode, + "Error %d initializing Merkle tree parameters", + err); + goto fail; + } + + memcpy(vi->root_hash, desc->root_hash, vi->tree_params.digest_size); + + compute_file_digest(vi->tree_params.hash_alg, desc, vi->file_digest); + + err = fsverity_verify_signature(vi, desc->signature, + le32_to_cpu(desc->sig_size)); + if (err) + goto fail; + + if (vi->tree_params.block_size != PAGE_SIZE) { + /* + * When the Merkle tree block size and page size differ, we use + * a bitmap to keep track of which hash blocks have been + * verified. This bitmap must contain one bit per hash block, + * including alignment to a page boundary at the end. + * + * Eventually, to support extremely large files in an efficient + * way, it might be necessary to make pages of this bitmap + * reclaimable. But for now, simply allocating the whole bitmap + * is a simple solution that works well on the files on which + * fsverity is realistically used. E.g., with SHA-256 and 4K + * blocks, a 100MB file only needs a 24-byte bitmap, and the + * bitmap for any file under 17GB fits in a 4K page. + */ + unsigned long num_bits = + vi->tree_params.tree_pages << + vi->tree_params.log_blocks_per_page; + + vi->hash_block_verified = kvcalloc(BITS_TO_LONGS(num_bits), + sizeof(unsigned long), + GFP_KERNEL); + if (!vi->hash_block_verified) { + err = -ENOMEM; + goto fail; + } + } + + return vi; + +fail: + fsverity_free_info(vi); + return ERR_PTR(err); +} + +int fsverity_set_info(struct fsverity_info *vi) +{ + return rhashtable_lookup_insert_fast(&fsverity_info_hash, + &vi->rhash_head, + fsverity_info_hash_params); +} + +struct fsverity_info *__fsverity_get_info(const struct inode *inode) +{ + return rhashtable_lookup_fast(&fsverity_info_hash, &inode, + fsverity_info_hash_params); +} +EXPORT_SYMBOL_GPL(__fsverity_get_info); + +static bool validate_fsverity_descriptor(struct inode *inode, + const struct fsverity_descriptor *desc, + size_t desc_size) +{ + if (desc_size < sizeof(*desc)) { + fsverity_err(inode, "Unrecognized descriptor size: %zu bytes", + desc_size); + return false; + } + + if (desc->version != 1) { + fsverity_err(inode, "Unrecognized descriptor version: %u", + desc->version); + return false; + } + + if (memchr_inv(desc->__reserved, 0, sizeof(desc->__reserved))) { + fsverity_err(inode, "Reserved bits set in descriptor"); + return false; + } + + if (desc->salt_size > sizeof(desc->salt)) { + fsverity_err(inode, "Invalid salt_size: %u", desc->salt_size); + return false; + } + + if (le64_to_cpu(desc->data_size) != inode->i_size) { + fsverity_err(inode, + "Wrong data_size: %llu (desc) != %lld (inode)", + le64_to_cpu(desc->data_size), inode->i_size); + return false; + } + + if (le32_to_cpu(desc->sig_size) > desc_size - sizeof(*desc)) { + fsverity_err(inode, "Signature overflows verity descriptor"); + return false; + } + + return true; +} + +/* + * Read the inode's fsverity_descriptor (with optional appended builtin + * signature) from the filesystem, and do basic validation of it. + */ +int fsverity_get_descriptor(struct inode *inode, + struct fsverity_descriptor **desc_ret) +{ + int res; + struct fsverity_descriptor *desc; + + res = inode->i_sb->s_vop->get_verity_descriptor(inode, NULL, 0); + if (res < 0) { + fsverity_err(inode, + "Error %d getting verity descriptor size", res); + return res; + } + if (res > FS_VERITY_MAX_DESCRIPTOR_SIZE) { + fsverity_err(inode, "Verity descriptor is too large (%d bytes)", + res); + return -EMSGSIZE; + } + desc = kmalloc(res, GFP_KERNEL); + if (!desc) + return -ENOMEM; + res = inode->i_sb->s_vop->get_verity_descriptor(inode, desc, res); + if (res < 0) { + fsverity_err(inode, "Error %d reading verity descriptor", res); + kfree(desc); + return res; + } + + if (!validate_fsverity_descriptor(inode, desc, res)) { + kfree(desc); + return -EINVAL; + } + + *desc_ret = desc; + return 0; +} + +static int ensure_verity_info(struct inode *inode) +{ + struct fsverity_info *vi = fsverity_get_info(inode), *found; + struct fsverity_descriptor *desc; + int err; + + if (vi) + return 0; + + err = fsverity_get_descriptor(inode, &desc); + if (err) + return err; + + vi = fsverity_create_info(inode, desc); + if (IS_ERR(vi)) { + err = PTR_ERR(vi); + goto out_free_desc; + } + + /* + * Multiple tasks may race to set the inode's verity info, in which case + * we might find an existing fsverity_info in the hash table. + */ + found = rhashtable_lookup_get_insert_fast(&fsverity_info_hash, + &vi->rhash_head, + fsverity_info_hash_params); + if (found) { + fsverity_free_info(vi); + if (IS_ERR(found)) + err = PTR_ERR(found); + } + +out_free_desc: + kfree(desc); + return err; +} + +int __fsverity_file_open(struct inode *inode, struct file *filp) +{ + if (filp->f_mode & FMODE_WRITE) + return -EPERM; + return ensure_verity_info(inode); +} +EXPORT_SYMBOL_GPL(__fsverity_file_open); + +void fsverity_free_info(struct fsverity_info *vi) +{ + kfree(vi->tree_params.hashstate); + kvfree(vi->hash_block_verified); + kmem_cache_free(fsverity_info_cachep, vi); +} + +void fsverity_remove_info(struct fsverity_info *vi) +{ + rhashtable_remove_fast(&fsverity_info_hash, &vi->rhash_head, + fsverity_info_hash_params); + fsverity_free_info(vi); +} + +void fsverity_cleanup_inode(struct inode *inode) +{ + struct fsverity_info *vi = fsverity_get_info(inode); + + if (vi) + fsverity_remove_info(vi); +} + +void __init fsverity_init_info_cache(void) +{ + if (rhashtable_init(&fsverity_info_hash, &fsverity_info_hash_params)) + panic("failed to initialize fsverity hash\n"); + fsverity_info_cachep = KMEM_CACHE_USERCOPY( + fsverity_info, + SLAB_RECLAIM_ACCOUNT | SLAB_PANIC, + file_digest); +} diff --git a/fs/verity/pagecache.c b/fs/verity/pagecache.c new file mode 100644 index 000000000..99f5f53ee --- /dev/null +++ b/fs/verity/pagecache.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ +#include "fsverity_private.h" + +#include <linux/export.h> +#include <linux/fsverity.h> +#include <linux/pagemap.h> + +/** + * generic_read_merkle_tree_page - generic ->read_merkle_tree_page helper + * @inode: inode containing the Merkle tree + * @index: 0-based index of the Merkle tree page in the inode + * + * The caller needs to adjust @index from the Merkle-tree relative index passed + * to ->read_merkle_tree_page to the actual index where the Merkle tree is + * stored in the page cache for @inode. + */ +struct page *generic_read_merkle_tree_page(struct inode *inode, pgoff_t index) +{ + struct folio *folio; + + folio = read_mapping_folio(inode->i_mapping, index, NULL); + if (IS_ERR(folio)) + return ERR_CAST(folio); + return folio_file_page(folio, index); +} +EXPORT_SYMBOL_GPL(generic_read_merkle_tree_page); + +/** + * generic_readahead_merkle_tree() - generic ->readahead_merkle_tree helper + * @inode: inode containing the Merkle tree + * @index: 0-based index of the first Merkle tree page to read ahead in the + * inode + * @nr_pages: the number of Merkle tree pages that should be read ahead + * + * The caller needs to adjust @index from the Merkle-tree relative index passed + * to ->read_merkle_tree_page to the actual index where the Merkle tree is + * stored in the page cache for @inode. + */ +void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index, + unsigned long nr_pages) +{ + struct folio *folio; + + lockdep_assert_held(&inode->i_mapping->invalidate_lock); + + folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0); + if (folio == ERR_PTR(-ENOENT) || + (!IS_ERR(folio) && !folio_test_uptodate(folio))) { + DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); + + page_cache_ra_unbounded(&ractl, nr_pages, 0); + } + if (!IS_ERR(folio)) + folio_put(folio); +} +EXPORT_SYMBOL_GPL(generic_readahead_merkle_tree); + +/** + * fsverity_fill_zerohash() - fill folio with hashes of zero data block + * @folio: folio to fill + * @offset: offset in the folio to start + * @len: length of the range to fill with hashes + * @vi: fsverity info + */ +void fsverity_fill_zerohash(struct folio *folio, size_t offset, size_t len, + struct fsverity_info *vi) +{ + size_t off = offset; + + WARN_ON_ONCE(!IS_ALIGNED(offset, vi->tree_params.digest_size)); + WARN_ON_ONCE(!IS_ALIGNED(len, vi->tree_params.digest_size)); + + for (; off < (offset + len); off += vi->tree_params.digest_size) + memcpy_to_folio(folio, off, vi->tree_params.zero_digest, + vi->tree_params.digest_size); +} +EXPORT_SYMBOL_GPL(fsverity_fill_zerohash); diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c new file mode 100644 index 000000000..b4c089243 --- /dev/null +++ b/fs/verity/read_metadata.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Ioctl to read verity metadata + * + * Copyright 2021 Google LLC + */ + +#include "fsverity_private.h" + +#include <linux/backing-dev.h> +#include <linux/export.h> +#include <linux/highmem.h> +#include <linux/sched/signal.h> +#include <linux/uaccess.h> + +static int fsverity_read_merkle_tree(struct inode *inode, + const struct fsverity_info *vi, + void __user *buf, u64 offset, int length) +{ + const struct fsverity_operations *vops = inode->i_sb->s_vop; + u64 end_offset; + unsigned int offs_in_page; + pgoff_t index, last_index; + int retval = 0; + int err = 0; + + end_offset = min(offset + length, vi->tree_params.tree_size); + if (offset >= end_offset) + return 0; + offs_in_page = offset_in_page(offset); + index = offset >> PAGE_SHIFT; + last_index = (end_offset - 1) >> PAGE_SHIFT; + + /* + * Kick off readahead for the range we are going to read to ensure a + * single large sequential read instead of lots of small ones. + */ + if (inode->i_sb->s_vop->readahead_merkle_tree) { + filemap_invalidate_lock_shared(inode->i_mapping); + inode->i_sb->s_vop->readahead_merkle_tree( + inode, index, last_index - index + 1); + filemap_invalidate_unlock_shared(inode->i_mapping); + } + + /* + * Iterate through each Merkle tree page in the requested range and copy + * the requested portion to userspace. Note that the Merkle tree block + * size isn't important here, as we are returning a byte stream; i.e., + * we can just work with pages even if the tree block size != PAGE_SIZE. + */ + for (; index <= last_index; index++) { + unsigned int bytes_to_copy = min_t(u64, end_offset - offset, + PAGE_SIZE - offs_in_page); + struct page *page; + const void *virt; + + page = vops->read_merkle_tree_page(inode, index); + if (IS_ERR(page)) { + err = PTR_ERR(page); + fsverity_err(inode, + "Error %d reading Merkle tree page %lu", + err, index); + break; + } + + virt = kmap_local_page(page); + if (copy_to_user(buf, virt + offs_in_page, bytes_to_copy)) { + kunmap_local(virt); + put_page(page); + err = -EFAULT; + break; + } + kunmap_local(virt); + put_page(page); + + retval += bytes_to_copy; + buf += bytes_to_copy; + offset += bytes_to_copy; + + if (fatal_signal_pending(current)) { + err = -EINTR; + break; + } + cond_resched(); + offs_in_page = 0; + } + return retval ? retval : err; +} + +/* Copy the requested portion of the buffer to userspace. */ +static int fsverity_read_buffer(void __user *dst, u64 offset, int length, + const void *src, size_t src_length) +{ + if (offset >= src_length) + return 0; + src += offset; + src_length -= offset; + + length = min_t(size_t, length, src_length); + + if (copy_to_user(dst, src, length)) + return -EFAULT; + + return length; +} + +static int fsverity_read_descriptor(struct inode *inode, + void __user *buf, u64 offset, int length) +{ + struct fsverity_descriptor *desc; + size_t desc_size; + int res; + + res = fsverity_get_descriptor(inode, &desc); + if (res) + return res; + + /* don't include the builtin signature */ + desc_size = offsetof(struct fsverity_descriptor, signature); + desc->sig_size = 0; + + res = fsverity_read_buffer(buf, offset, length, desc, desc_size); + + kfree(desc); + return res; +} + +static int fsverity_read_signature(struct inode *inode, + void __user *buf, u64 offset, int length) +{ + struct fsverity_descriptor *desc; + int res; + + res = fsverity_get_descriptor(inode, &desc); + if (res) + return res; + + if (desc->sig_size == 0) { + res = -ENODATA; + goto out; + } + + /* + * Include only the builtin signature. fsverity_get_descriptor() + * already verified that sig_size is in-bounds. + */ + res = fsverity_read_buffer(buf, offset, length, desc->signature, + le32_to_cpu(desc->sig_size)); +out: + kfree(desc); + return res; +} + +/** + * fsverity_ioctl_read_metadata() - read verity metadata from a file + * @filp: file to read the metadata from + * @uarg: user pointer to fsverity_read_metadata_arg + * + * Return: length read on success, 0 on EOF, -errno on failure + */ +int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg) +{ + struct inode *inode = file_inode(filp); + const struct fsverity_info *vi; + struct fsverity_read_metadata_arg arg; + int length; + void __user *buf; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + /* + * Note that we don't have to explicitly check that the file is open for + * reading, since verity files can only be opened for reading. + */ + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (arg.__reserved) + return -EINVAL; + + /* offset + length must not overflow. */ + if (arg.offset + arg.length < arg.offset) + return -EINVAL; + + /* Ensure that the return value will fit in INT_MAX. */ + length = min_t(u64, arg.length, INT_MAX); + + buf = u64_to_user_ptr(arg.buf_ptr); + + switch (arg.metadata_type) { + case FS_VERITY_METADATA_TYPE_MERKLE_TREE: + return fsverity_read_merkle_tree(inode, vi, buf, arg.offset, + length); + case FS_VERITY_METADATA_TYPE_DESCRIPTOR: + return fsverity_read_descriptor(inode, buf, arg.offset, length); + case FS_VERITY_METADATA_TYPE_SIGNATURE: + return fsverity_read_signature(inode, buf, arg.offset, length); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL_GPL(fsverity_ioctl_read_metadata); diff --git a/fs/verity/signature.c b/fs/verity/signature.c new file mode 100644 index 000000000..0302a4e50 --- /dev/null +++ b/fs/verity/signature.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Verification of builtin signatures + * + * Copyright 2019 Google LLC + */ + +/* + * This file implements verification of fs-verity builtin signatures. Please + * take great care before using this feature. It is not the only way to do + * signatures with fs-verity, and the alternatives (such as userspace signature + * verification, and IMA appraisal) can be much better. For details about the + * limitations of this feature, see Documentation/filesystems/fsverity.rst. + */ + +#include "fsverity_private.h" + +#include <linux/cred.h> +#include <linux/key.h> +#include <linux/security.h> +#include <linux/slab.h> +#include <linux/verification.h> + +/* + * /proc/sys/fs/verity/require_signatures + * If 1, all verity files must have a valid builtin signature. + */ +int fsverity_require_signatures; + +/* + * Keyring that contains the trusted X.509 certificates. + * + * Only root (kuid=0) can modify this. Also, root may use + * keyctl_restrict_keyring() to prevent any more additions. + */ +static struct key *fsverity_keyring; + +/** + * fsverity_verify_signature() - check a verity file's signature + * @vi: the file's fsverity_info + * @signature: the file's built-in signature + * @sig_size: size of signature in bytes, or 0 if no signature + * + * If the file includes a signature of its fs-verity file digest, verify it + * against the certificates in the fs-verity keyring. Note that signatures + * are verified regardless of the state of the 'fsverity_require_signatures' + * variable and the LSM subsystem relies on this behavior to help enforce + * file integrity policies. Please discuss changes with the LSM list + * (thank you!). + * + * Return: 0 on success (signature valid or not required); -errno on failure + */ +int fsverity_verify_signature(const struct fsverity_info *vi, + const u8 *signature, size_t sig_size) +{ + const struct inode *inode = vi->inode; + const struct fsverity_hash_alg *hash_alg = vi->tree_params.hash_alg; + struct fsverity_formatted_digest *d; + int err; + + if (sig_size == 0) { + if (fsverity_require_signatures) { + fsverity_err(inode, + "require_signatures=1, rejecting unsigned file!"); + return -EPERM; + } + return 0; + } + + if (fsverity_keyring->keys.nr_leaves_on_tree == 0) { + /* + * The ".fs-verity" keyring is empty, due to builtin signatures + * being supported by the kernel but not actually being used. + * In this case, verify_pkcs7_signature() would always return an + * error, usually ENOKEY. It could also be EBADMSG if the + * PKCS#7 is malformed, but that isn't very important to + * distinguish. So, just skip to ENOKEY to avoid the attack + * surface of the PKCS#7 parser, which would otherwise be + * reachable by any task able to execute FS_IOC_ENABLE_VERITY. + */ + fsverity_err(inode, + "fs-verity keyring is empty, rejecting signed file!"); + return -ENOKEY; + } + + d = kzalloc(sizeof(*d) + hash_alg->digest_size, GFP_KERNEL); + if (!d) + return -ENOMEM; + memcpy(d->magic, "FSVerity", 8); + d->digest_algorithm = cpu_to_le16(hash_alg - fsverity_hash_algs); + d->digest_size = cpu_to_le16(hash_alg->digest_size); + memcpy(d->digest, vi->file_digest, hash_alg->digest_size); + + err = verify_pkcs7_signature(d, sizeof(*d) + hash_alg->digest_size, + signature, sig_size, fsverity_keyring, + VERIFYING_UNSPECIFIED_SIGNATURE, + NULL, NULL); + kfree(d); + + if (err) { + if (err == -ENOKEY) + fsverity_err(inode, + "File's signing cert isn't in the fs-verity keyring"); + else if (err == -EKEYREJECTED) + fsverity_err(inode, "Incorrect file signature"); + else if (err == -EBADMSG) + fsverity_err(inode, "Malformed file signature"); + else + fsverity_err(inode, "Error %d verifying file signature", + err); + return err; + } + + err = security_inode_setintegrity(inode, + LSM_INT_FSVERITY_BUILTINSIG_VALID, + signature, + sig_size); + + if (err) { + fsverity_err(inode, "Error %d exposing file signature to LSMs", + err); + return err; + } + + return 0; +} + +void __init fsverity_init_signature(void) +{ + fsverity_keyring = + keyring_alloc(".fs-verity", KUIDT_INIT(0), KGIDT_INIT(0), + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | KEY_USR_WRITE | + KEY_USR_SEARCH | KEY_USR_SETATTR, + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + if (IS_ERR(fsverity_keyring)) + panic("failed to allocate \".fs-verity\" keyring"); +} diff --git a/fs/verity/verify.c b/fs/verity/verify.c new file mode 100644 index 000000000..4004a1d42 --- /dev/null +++ b/fs/verity/verify.c @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Data verification functions, i.e. hooks for ->readahead() + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +#include <linux/bio.h> +#include <linux/export.h> + +#define FS_VERITY_MAX_PENDING_BLOCKS 2 + +struct fsverity_pending_block { + const void *data; + u64 pos; + u8 real_hash[FS_VERITY_MAX_DIGEST_SIZE]; +}; + +struct fsverity_verification_context { + struct fsverity_info *vi; + + /* + * This is the queue of data blocks that are pending verification. When + * the crypto layer supports interleaved hashing, we allow multiple + * blocks to be queued up in order to utilize it. This can improve + * performance significantly vs. sequential hashing of each block. + */ + int num_pending; + int max_pending; + struct fsverity_pending_block + pending_blocks[FS_VERITY_MAX_PENDING_BLOCKS]; +}; + +static struct workqueue_struct *fsverity_read_workqueue; + +/** + * fsverity_readahead() - kick off readahead on fsverity hashes + * @vi: fsverity_info for the inode to be read + * @index: first file data page index that is being read + * @nr_pages: number of file data pages to be read + * + * Start readahead on the fsverity hashes that are needed to verify the file + * data in the range from @index to @index + @nr_pages (exclusive upper bound). + * + * To be called from the file systems' ->read_folio and ->readahead methods to + * ensure that the hashes are already cached on completion of the file data + * read if possible. + */ +void fsverity_readahead(struct fsverity_info *vi, pgoff_t index, + unsigned long nr_pages) +{ + struct inode *inode = vi->inode; + const struct merkle_tree_params *params = &vi->tree_params; + u64 start_hidx = (u64)index << params->log_blocks_per_page; + u64 end_hidx = + (((u64)index + nr_pages) << params->log_blocks_per_page) - 1; + int level; + + if (!inode->i_sb->s_vop->readahead_merkle_tree) + return; + + for (level = 0; level < params->num_levels; level++) { + unsigned long level_start = params->level_start[level]; + unsigned long next_start_hidx = start_hidx >> params->log_arity; + unsigned long next_end_hidx = end_hidx >> params->log_arity; + pgoff_t start_idx = (level_start + next_start_hidx) >> + params->log_blocks_per_page; + pgoff_t end_idx = (level_start + next_end_hidx) >> + params->log_blocks_per_page; + + inode->i_sb->s_vop->readahead_merkle_tree( + inode, start_idx, end_idx - start_idx + 1); + + start_hidx = next_start_hidx; + end_hidx = next_end_hidx; + } +} +EXPORT_SYMBOL_GPL(fsverity_readahead); + +/* + * Returns true if the hash block with index @hblock_idx in the tree, located in + * @hpage, has already been verified. + */ +static bool is_hash_block_verified(struct fsverity_info *vi, struct page *hpage, + unsigned long hblock_idx) +{ + unsigned int blocks_per_page; + unsigned int i; + + /* + * When the Merkle tree block size and page size are the same, then the + * ->hash_block_verified bitmap isn't allocated, and we use PG_checked + * to directly indicate whether the page's block has been verified. + * + * Using PG_checked also guarantees that we re-verify hash pages that + * get evicted and re-instantiated from the backing storage, as new + * pages always start out with PG_checked cleared. + */ + if (!vi->hash_block_verified) + return PageChecked(hpage); + + /* + * When the Merkle tree block size and page size differ, we use a bitmap + * to indicate whether each hash block has been verified. + * + * However, we still need to ensure that hash pages that get evicted and + * re-instantiated from the backing storage are re-verified. To do + * this, we use PG_checked again, but now it doesn't really mean + * "checked". Instead, now it just serves as an indicator for whether + * the hash page is newly instantiated or not. If the page is new, as + * indicated by PG_checked=0, we clear the bitmap bits for the page's + * blocks since they are untrustworthy, then set PG_checked=1. + * Otherwise we return the bitmap bit for the requested block. + * + * Multiple threads may execute this code concurrently on the same page. + * This is safe because we use memory barriers to ensure that if a + * thread sees PG_checked=1, then it also sees the associated bitmap + * clearing to have occurred. Also, all writes and their corresponding + * reads are atomic, and all writes are safe to repeat in the event that + * multiple threads get into the PG_checked=0 section. (Clearing a + * bitmap bit again at worst causes a hash block to be verified + * redundantly. That event should be very rare, so it's not worth using + * a lock to avoid. Setting PG_checked again has no effect.) + */ + if (PageChecked(hpage)) { + /* + * A read memory barrier is needed here to give ACQUIRE + * semantics to the above PageChecked() test. + */ + smp_rmb(); + return test_bit(hblock_idx, vi->hash_block_verified); + } + blocks_per_page = vi->tree_params.blocks_per_page; + hblock_idx = round_down(hblock_idx, blocks_per_page); + for (i = 0; i < blocks_per_page; i++) + clear_bit(hblock_idx + i, vi->hash_block_verified); + /* + * A write memory barrier is needed here to give RELEASE semantics to + * the below SetPageChecked() operation. + */ + smp_wmb(); + SetPageChecked(hpage); + return false; +} + +/* + * Verify the hash of a single data block against the file's Merkle tree. + * + * In principle, we need to verify the entire path to the root node. However, + * for efficiency the filesystem may cache the hash blocks. Therefore we need + * only ascend the tree until an already-verified hash block is seen, and then + * verify the path to that block. + * + * Return: %true if the data block is valid, else %false. + */ +static bool verify_data_block(struct fsverity_info *vi, + const struct fsverity_pending_block *dblock) +{ + struct inode *inode = vi->inode; + const u64 data_pos = dblock->pos; + const struct merkle_tree_params *params = &vi->tree_params; + const unsigned int hsize = params->digest_size; + int level; + u8 _want_hash[FS_VERITY_MAX_DIGEST_SIZE]; + const u8 *want_hash; + u8 real_hash[FS_VERITY_MAX_DIGEST_SIZE]; + /* The hash blocks that are traversed, indexed by level */ + struct { + /* Page containing the hash block */ + struct page *page; + /* Mapped address of the hash block (will be within @page) */ + const void *addr; + /* Index of the hash block in the tree overall */ + unsigned long index; + /* Byte offset of the wanted hash relative to @addr */ + unsigned int hoffset; + } hblocks[FS_VERITY_MAX_LEVELS]; + + trace_fsverity_verify_data_block(inode, params, data_pos); + + /* + * The index of the previous level's block within that level; also the + * index of that block's hash within the current level. + */ + u64 hidx = data_pos >> params->log_blocksize; + + /* + * Up to FS_VERITY_MAX_PENDING_BLOCKS + FS_VERITY_MAX_LEVELS pages may + * be mapped at once. + */ + static_assert(FS_VERITY_MAX_PENDING_BLOCKS + FS_VERITY_MAX_LEVELS <= + KM_MAX_IDX); + + if (unlikely(data_pos >= inode->i_size)) { + /* + * This can happen in the data page spanning EOF when the Merkle + * tree block size is less than the page size. The Merkle tree + * doesn't cover data blocks fully past EOF. But the entire + * page spanning EOF can be visible to userspace via a mmap, and + * any part past EOF should be all zeroes. Therefore, we need + * to verify that any data blocks fully past EOF are all zeroes. + */ + if (memchr_inv(dblock->data, 0, params->block_size)) { + fsverity_err(inode, + "FILE CORRUPTED! Data past EOF is not zeroed"); + return false; + } + return true; + } + + /* + * Starting at the leaf level, ascend the tree saving hash blocks along + * the way until we find a hash block that has already been verified, or + * until we reach the root. + */ + for (level = 0; level < params->num_levels; level++) { + unsigned long next_hidx; + unsigned long hblock_idx; + pgoff_t hpage_idx; + unsigned int hblock_offset_in_page; + unsigned int hoffset; + struct page *hpage; + const void *haddr; + + /* + * The index of the block in the current level; also the index + * of that block's hash within the next level. + */ + next_hidx = hidx >> params->log_arity; + + /* Index of the hash block in the tree overall */ + hblock_idx = params->level_start[level] + next_hidx; + + /* Index of the hash page in the tree overall */ + hpage_idx = hblock_idx >> params->log_blocks_per_page; + + /* Byte offset of the hash block within the page */ + hblock_offset_in_page = + (hblock_idx << params->log_blocksize) & ~PAGE_MASK; + + /* Byte offset of the hash within the block */ + hoffset = (hidx << params->log_digestsize) & + (params->block_size - 1); + + hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode, + hpage_idx); + if (IS_ERR(hpage)) { + fsverity_err(inode, + "Error %ld reading Merkle tree page %lu", + PTR_ERR(hpage), hpage_idx); + goto error; + } + haddr = kmap_local_page(hpage) + hblock_offset_in_page; + if (is_hash_block_verified(vi, hpage, hblock_idx)) { + memcpy(_want_hash, haddr + hoffset, hsize); + want_hash = _want_hash; + kunmap_local(haddr); + put_page(hpage); + trace_fsverity_merkle_hit(inode, data_pos, hblock_idx, + level, + hoffset >> params->log_digestsize); + goto descend; + } + hblocks[level].page = hpage; + hblocks[level].addr = haddr; + hblocks[level].index = hblock_idx; + hblocks[level].hoffset = hoffset; + hidx = next_hidx; + } + + want_hash = vi->root_hash; +descend: + /* Descend the tree verifying hash blocks. */ + for (; level > 0; level--) { + struct page *hpage = hblocks[level - 1].page; + const void *haddr = hblocks[level - 1].addr; + unsigned long hblock_idx = hblocks[level - 1].index; + unsigned int hoffset = hblocks[level - 1].hoffset; + + trace_fsverity_verify_merkle_block(inode, hblock_idx, + level, hoffset >> params->log_digestsize); + + fsverity_hash_block(params, haddr, real_hash); + if (memcmp(want_hash, real_hash, hsize) != 0) + goto corrupted; + /* + * Mark the hash block as verified. This must be atomic and + * idempotent, as the same hash block might be verified by + * multiple threads concurrently. + */ + if (vi->hash_block_verified) + set_bit(hblock_idx, vi->hash_block_verified); + else + SetPageChecked(hpage); + memcpy(_want_hash, haddr + hoffset, hsize); + want_hash = _want_hash; + kunmap_local(haddr); + put_page(hpage); + } + + /* Finally, verify the hash of the data block. */ + if (memcmp(want_hash, dblock->real_hash, hsize) != 0) + goto corrupted; + return true; + +corrupted: + fsverity_err( + inode, + "FILE CORRUPTED! pos=%llu, level=%d, want_hash=%s:%*phN, real_hash=%s:%*phN", + data_pos, level - 1, params->hash_alg->name, hsize, want_hash, + params->hash_alg->name, hsize, + level == 0 ? dblock->real_hash : real_hash); +error: + for (; level > 0; level--) { + kunmap_local(hblocks[level - 1].addr); + put_page(hblocks[level - 1].page); + } + return false; +} + +static void +fsverity_init_verification_context(struct fsverity_verification_context *ctx, + struct fsverity_info *vi) +{ + ctx->vi = vi; + ctx->num_pending = 0; + if (vi->tree_params.hash_alg->algo_id == HASH_ALGO_SHA256 && + sha256_finup_2x_is_optimized()) + ctx->max_pending = 2; + else + ctx->max_pending = 1; +} + +static void +fsverity_clear_pending_blocks(struct fsverity_verification_context *ctx) +{ + int i; + + for (i = ctx->num_pending - 1; i >= 0; i--) { + kunmap_local(ctx->pending_blocks[i].data); + ctx->pending_blocks[i].data = NULL; + } + ctx->num_pending = 0; +} + +static bool +fsverity_verify_pending_blocks(struct fsverity_verification_context *ctx) +{ + struct fsverity_info *vi = ctx->vi; + const struct merkle_tree_params *params = &vi->tree_params; + int i; + + if (ctx->num_pending == 2) { + /* num_pending == 2 implies that the algorithm is SHA-256 */ + sha256_finup_2x(params->hashstate ? ¶ms->hashstate->sha256 : + NULL, + ctx->pending_blocks[0].data, + ctx->pending_blocks[1].data, params->block_size, + ctx->pending_blocks[0].real_hash, + ctx->pending_blocks[1].real_hash); + } else { + for (i = 0; i < ctx->num_pending; i++) + fsverity_hash_block(params, ctx->pending_blocks[i].data, + ctx->pending_blocks[i].real_hash); + } + + for (i = 0; i < ctx->num_pending; i++) { + if (!verify_data_block(vi, &ctx->pending_blocks[i])) + return false; + } + fsverity_clear_pending_blocks(ctx); + return true; +} + +static bool fsverity_add_data_blocks(struct fsverity_verification_context *ctx, + struct folio *data_folio, size_t len, + size_t offset) +{ + struct fsverity_info *vi = ctx->vi; + const struct merkle_tree_params *params = &vi->tree_params; + const unsigned int block_size = params->block_size; + u64 pos = (u64)data_folio->index << PAGE_SHIFT; + + if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offset, block_size))) + return false; + if (WARN_ON_ONCE(!folio_test_locked(data_folio) || + folio_test_uptodate(data_folio))) + return false; + do { + ctx->pending_blocks[ctx->num_pending].data = + kmap_local_folio(data_folio, offset); + ctx->pending_blocks[ctx->num_pending].pos = pos + offset; + if (++ctx->num_pending == ctx->max_pending && + !fsverity_verify_pending_blocks(ctx)) + return false; + offset += block_size; + len -= block_size; + } while (len); + return true; +} + +/** + * fsverity_verify_blocks() - verify data in a folio + * @vi: fsverity_info for the inode to be read + * @folio: the folio containing the data to verify + * @len: the length of the data to verify in the folio + * @offset: the offset of the data to verify in the folio + * + * Verify data that has just been read from a verity file. The data must be + * located in a pagecache folio that is still locked and not yet uptodate. The + * length and offset of the data must be Merkle tree block size aligned. + * + * Return: %true if the data is valid, else %false. + */ +bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio, + size_t len, size_t offset) +{ + struct fsverity_verification_context ctx; + + fsverity_init_verification_context(&ctx, vi); + + if (fsverity_add_data_blocks(&ctx, folio, len, offset) && + fsverity_verify_pending_blocks(&ctx)) + return true; + fsverity_clear_pending_blocks(&ctx); + return false; +} +EXPORT_SYMBOL_GPL(fsverity_verify_blocks); + +#ifdef CONFIG_BLOCK +/** + * fsverity_verify_bio() - verify a 'read' bio that has just completed + * @vi: fsverity_info for the inode to be read + * @bio: the bio to verify + * + * Verify the bio's data against the file's Merkle tree. All bio data segments + * must be aligned to the file's Merkle tree block size. If any data fails + * verification, then bio->bi_status is set to an error status. + * + * This is a helper function for use by the ->readahead() method of filesystems + * that issue bios to read data directly into the page cache. Filesystems that + * populate the page cache without issuing bios (e.g. non block-based + * filesystems) must instead call fsverity_verify_blocks() directly. All + * filesystems must also call fsverity_verify_blocks() on holes. + */ +void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio) +{ + struct fsverity_verification_context ctx; + struct folio_iter fi; + + fsverity_init_verification_context(&ctx, vi); + + bio_for_each_folio_all(fi, bio) { + if (!fsverity_add_data_blocks(&ctx, fi.folio, fi.length, + fi.offset)) + goto ioerr; + } + + if (!fsverity_verify_pending_blocks(&ctx)) + goto ioerr; + return; + +ioerr: + fsverity_clear_pending_blocks(&ctx); + bio->bi_status = BLK_STS_IOERR; +} +EXPORT_SYMBOL_GPL(fsverity_verify_bio); +#endif /* CONFIG_BLOCK */ + +/** + * fsverity_enqueue_verify_work() - enqueue work on the fs-verity workqueue + * @work: the work to enqueue + * + * Enqueue verification work for asynchronous processing. + */ +void fsverity_enqueue_verify_work(struct work_struct *work) +{ + queue_work(fsverity_read_workqueue, work); +} +EXPORT_SYMBOL_GPL(fsverity_enqueue_verify_work); + +void __init fsverity_init_workqueue(void) +{ + /* + * Use a high-priority workqueue to prioritize verification work, which + * blocks reads from completing, over regular application tasks. + * + * For performance reasons, don't use an unbound workqueue. Using an + * unbound workqueue for crypto operations causes excessive scheduler + * latency on ARM64. + */ + fsverity_read_workqueue = alloc_workqueue("fsverity_read_queue", + WQ_HIGHPRI | WQ_PERCPU, + num_online_cpus()); + if (!fsverity_read_workqueue) + panic("failed to allocate fsverity_read_queue"); +} |