kbuild: rust: keep Rust objects out of Clang LTO with inline helpersgrafted

Under `CONFIG_LTO_CLANG` + `CONFIG_RUST_INLINE_HELPERS`, one may hit
`objtool` errors such as:

    vmlinux.o: error: objtool: _R..._3Gsp4boot+0xd6a:
    can't find jump dest instruction at .text._R..._3Gsp4boot+0x1dfd

The reason is that in such builds, the Clang invocation that compiles
the combined Rust plus helpers bitcode emits LLVM bitcode (again) --
the final code generation happens in the linker's LTO step, which the
`-mllvm` trap options passed to Clang do not reach.

This, in turn, means that unreachable traps are missing, and the
impossible paths do not merely fallthrough to the next symbol, but past
the end of their own section, since LTO builds place each function in
its own section.

Thus filter `CC_FLAGS_LTO` out of the Clang invocation, so that it always
emits machine code directly, with the traps in place.

Assisted-by: LLM
Cc: Gary Guo <gary@garyguo.net>
Cc: Boqun Feng <boqun@kernel.org>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Matthew Maurer <mmaurer@google.com>
Cc: Josh Poimboeuf <jpoimboe@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: stable@vger.kernel.org
Fixes: 3a2486cc1da5 ("kbuild: rust: provide an option to inline C helpers into Rust")
Acked-by: Gary Guo <gary@garyguo.net>
Link: https://patch.msgid.link/20260816133233.197500-2-ojeda@kernel.org
Signed-off-by: Miguel Ojeda <ojeda@kernel.org>

1 files changed, 476 insertions, 0 deletions

diff --git a/drivers/dax/device.c b/drivers/dax/device.c
new file mode 100644
index 000000000..d0c9b4e03
--- /dev/null
+++ b/drivers/dax/device.c
@@ -0,0 +1,476 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */
+#include <linux/memremap.h>
+#include <linux/pagemap.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/cdev.h>
+#include <linux/slab.h>
+#include <linux/dax.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include "dax-private.h"
+#include "bus.h"
+
+static int __check_vma(struct dev_dax *dev_dax, vma_flags_t flags,
+		       unsigned long start, unsigned long end, struct file *file,
+		       const char *func)
+{
+	struct device *dev = &dev_dax->dev;
+	unsigned long mask;
+
+	if (!dax_alive(dev_dax->dax_dev))
+		return -ENXIO;
+
+	/* prevent private mappings from being established */
+	if (!vma_flags_test_any(&flags, VMA_MAYSHARE_BIT)) {
+		dev_info_ratelimited(dev,
+				"%s: %s: fail, attempted private mapping\n",
+				current->comm, func);
+		return -EINVAL;
+	}
+
+	mask = dev_dax->align - 1;
+	if (start & mask || end & mask) {
+		dev_info_ratelimited(dev,
+				"%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
+				current->comm, func, start, end,
+				mask);
+		return -EINVAL;
+	}
+
+	if (!file_is_dax(file)) {
+		dev_info_ratelimited(dev,
+				"%s: %s: fail, vma is not DAX capable\n",
+				current->comm, func);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
+		     const char *func)
+{
+	return __check_vma(dev_dax, vma->flags, vma->vm_start, vma->vm_end,
+			   vma->vm_file, func);
+}
+
+static void dax_set_mapping(struct vm_fault *vmf, unsigned long pfn,
+			      unsigned long fault_size)
+{
+	unsigned long i, nr_pages = fault_size / PAGE_SIZE;
+	struct file *filp = vmf->vma->vm_file;
+	struct dev_dax *dev_dax = filp->private_data;
+	pgoff_t pgoff;
+
+	/* mapping is only set on the head */
+	if (dev_dax->pgmap->vmemmap_shift)
+		nr_pages = 1;
+
+	pgoff = linear_page_index(vmf->vma,
+			ALIGN_DOWN(vmf->address, fault_size));
+
+	for (i = 0; i < nr_pages; i++) {
+		struct folio *folio = pfn_folio(pfn + i);
+
+		if (folio->mapping)
+			continue;
+
+		folio->mapping = filp->f_mapping;
+		folio->index = pgoff + i;
+	}
+}
+
+static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
+				struct vm_fault *vmf)
+{
+	struct device *dev = &dev_dax->dev;
+	phys_addr_t phys;
+	unsigned long pfn;
+	unsigned int fault_size = PAGE_SIZE;
+
+	if (check_vma(dev_dax, vmf->vma, __func__))
+		return VM_FAULT_SIGBUS;
+
+	if (dev_dax->align > PAGE_SIZE) {
+		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
+			dev_dax->align, fault_size);
+		return VM_FAULT_SIGBUS;
+	}
+
+	if (fault_size != dev_dax->align)
+		return VM_FAULT_SIGBUS;
+
+	phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
+	if (phys == -1) {
+		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff);
+		return VM_FAULT_SIGBUS;
+	}
+
+	pfn = PHYS_PFN(phys);
+
+	dax_set_mapping(vmf, pfn, fault_size);
+
+	return vmf_insert_page_mkwrite(vmf, pfn_to_page(pfn),
+					vmf->flags & FAULT_FLAG_WRITE);
+}
+
+static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
+				struct vm_fault *vmf)
+{
+	unsigned long pmd_addr = vmf->address & PMD_MASK;
+	struct device *dev = &dev_dax->dev;
+	phys_addr_t phys;
+	pgoff_t pgoff;
+	unsigned long pfn;
+	unsigned int fault_size = PMD_SIZE;
+
+	if (check_vma(dev_dax, vmf->vma, __func__))
+		return VM_FAULT_SIGBUS;
+
+	if (dev_dax->align > PMD_SIZE) {
+		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
+			dev_dax->align, fault_size);
+		return VM_FAULT_SIGBUS;
+	}
+
+	if (fault_size < dev_dax->align)
+		return VM_FAULT_SIGBUS;
+	else if (fault_size > dev_dax->align)
+		return VM_FAULT_FALLBACK;
+
+	/* if we are outside of the VMA */
+	if (pmd_addr < vmf->vma->vm_start ||
+			(pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
+		return VM_FAULT_SIGBUS;
+
+	pgoff = linear_page_index(vmf->vma, pmd_addr);
+	phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
+	if (phys == -1) {
+		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
+		return VM_FAULT_SIGBUS;
+	}
+
+	pfn = PHYS_PFN(phys);
+
+	dax_set_mapping(vmf, pfn, fault_size);
+
+	return vmf_insert_folio_pmd(vmf, page_folio(pfn_to_page(pfn)),
+				vmf->flags & FAULT_FLAG_WRITE);
+}
+
+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
+static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
+				struct vm_fault *vmf)
+{
+	unsigned long pud_addr = vmf->address & PUD_MASK;
+	struct device *dev = &dev_dax->dev;
+	phys_addr_t phys;
+	pgoff_t pgoff;
+	unsigned long pfn;
+	unsigned int fault_size = PUD_SIZE;
+
+
+	if (check_vma(dev_dax, vmf->vma, __func__))
+		return VM_FAULT_SIGBUS;
+
+	if (dev_dax->align > PUD_SIZE) {
+		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
+			dev_dax->align, fault_size);
+		return VM_FAULT_SIGBUS;
+	}
+
+	if (fault_size < dev_dax->align)
+		return VM_FAULT_SIGBUS;
+	else if (fault_size > dev_dax->align)
+		return VM_FAULT_FALLBACK;
+
+	/* if we are outside of the VMA */
+	if (pud_addr < vmf->vma->vm_start ||
+			(pud_addr + PUD_SIZE) > vmf->vma->vm_end)
+		return VM_FAULT_SIGBUS;
+
+	pgoff = linear_page_index(vmf->vma, pud_addr);
+	phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
+	if (phys == -1) {
+		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
+		return VM_FAULT_SIGBUS;
+	}
+
+	pfn = PHYS_PFN(phys);
+
+	dax_set_mapping(vmf, pfn, fault_size);
+
+	return vmf_insert_folio_pud(vmf, page_folio(pfn_to_page(pfn)),
+				vmf->flags & FAULT_FLAG_WRITE);
+}
+#else
+static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
+				struct vm_fault *vmf)
+{
+	return VM_FAULT_FALLBACK;
+}
+#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
+
+static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, unsigned int order)
+{
+	struct file *filp = vmf->vma->vm_file;
+	vm_fault_t rc = VM_FAULT_SIGBUS;
+	int id;
+	struct dev_dax *dev_dax = filp->private_data;
+
+	dev_dbg(&dev_dax->dev, "%s: op=%s addr=%#lx order=%d\n", current->comm,
+		(vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",
+		vmf->address & ~((1UL << (order + PAGE_SHIFT)) - 1), order);
+
+	id = dax_read_lock();
+	if (order == 0)
+		rc = __dev_dax_pte_fault(dev_dax, vmf);
+	else if (order == PMD_ORDER)
+		rc = __dev_dax_pmd_fault(dev_dax, vmf);
+	else if (order == PUD_ORDER)
+		rc = __dev_dax_pud_fault(dev_dax, vmf);
+	else
+		rc = VM_FAULT_SIGBUS;
+
+	dax_read_unlock(id);
+
+	return rc;
+}
+
+static vm_fault_t dev_dax_fault(struct vm_fault *vmf)
+{
+	return dev_dax_huge_fault(vmf, 0);
+}
+
+static int dev_dax_may_split(struct vm_area_struct *vma, unsigned long addr)
+{
+	struct file *filp = vma->vm_file;
+	struct dev_dax *dev_dax = filp->private_data;
+
+	if (!IS_ALIGNED(addr, dev_dax->align))
+		return -EINVAL;
+	return 0;
+}
+
+static unsigned long dev_dax_pagesize(struct vm_area_struct *vma)
+{
+	struct file *filp = vma->vm_file;
+	struct dev_dax *dev_dax = filp->private_data;
+
+	return dev_dax->align;
+}
+
+static const struct vm_operations_struct dax_vm_ops = {
+	.fault = dev_dax_fault,
+	.huge_fault = dev_dax_huge_fault,
+	.may_split = dev_dax_may_split,
+	.pagesize = dev_dax_pagesize,
+};
+
+static int dax_mmap_prepare(struct vm_area_desc *desc)
+{
+	struct file *filp = desc->file;
+	struct dev_dax *dev_dax = filp->private_data;
+	int rc, id;
+
+	dev_dbg(&dev_dax->dev, "trace\n");
+
+	/*
+	 * We lock to check dax_dev liveness and will re-check at
+	 * fault time.
+	 */
+	id = dax_read_lock();
+	rc = __check_vma(dev_dax, desc->vma_flags, desc->start, desc->end, filp,
+			 __func__);
+	dax_read_unlock(id);
+	if (rc)
+		return rc;
+
+	desc->vm_ops = &dax_vm_ops;
+	vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT);
+	return 0;
+}
+
+/* return an unmapped area aligned to the dax region specified alignment */
+static unsigned long dax_get_unmapped_area(struct file *filp,
+		unsigned long addr, unsigned long len, unsigned long pgoff,
+		unsigned long flags)
+{
+	unsigned long off, off_end, off_align, len_align, addr_align, align;
+	struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
+
+	if (!dev_dax || addr)
+		goto out;
+
+	align = dev_dax->align;
+	off = pgoff << PAGE_SHIFT;
+	off_end = off + len;
+	off_align = round_up(off, align);
+
+	if ((off_end <= off_align) || ((off_end - off_align) < align))
+		goto out;
+
+	len_align = len + align;
+	if ((off + len_align) < off)
+		goto out;
+
+	addr_align = mm_get_unmapped_area(filp, addr, len_align, pgoff, flags);
+	if (!IS_ERR_VALUE(addr_align)) {
+		addr_align += (off - addr_align) & (align - 1);
+		return addr_align;
+	}
+ out:
+	return mm_get_unmapped_area(filp, addr, len, pgoff, flags);
+}
+
+static const struct address_space_operations dev_dax_aops = {
+	.dirty_folio	= noop_dirty_folio,
+};
+
+static int dax_open(struct inode *inode, struct file *filp)
+{
+	struct dax_device *dax_dev = inode_dax(inode);
+	struct inode *__dax_inode = dax_inode(dax_dev);
+	struct dev_dax *dev_dax = dax_get_private(dax_dev);
+
+	dev_dbg(&dev_dax->dev, "trace\n");
+	inode->i_mapping = __dax_inode->i_mapping;
+	inode->i_mapping->host = __dax_inode;
+	inode->i_mapping->a_ops = &dev_dax_aops;
+	filp->f_mapping = inode->i_mapping;
+	filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
+	filp->f_sb_err = file_sample_sb_err(filp);
+	filp->private_data = dev_dax;
+	inode->i_flags = S_DAX;
+
+	return 0;
+}
+
+static int dax_release(struct inode *inode, struct file *filp)
+{
+	struct dev_dax *dev_dax = filp->private_data;
+
+	dev_dbg(&dev_dax->dev, "trace\n");
+	return 0;
+}
+
+static const struct file_operations dax_fops = {
+	.llseek = noop_llseek,
+	.owner = THIS_MODULE,
+	.open = dax_open,
+	.release = dax_release,
+	.get_unmapped_area = dax_get_unmapped_area,
+	.mmap_prepare = dax_mmap_prepare,
+	.fop_flags = FOP_MMAP_SYNC,
+};
+
+static void dev_dax_cdev_del(void *cdev)
+{
+	cdev_del(cdev);
+}
+
+static void dev_dax_kill(void *dev_dax)
+{
+	kill_dev_dax(dev_dax);
+}
+
+static int dev_dax_probe(struct dev_dax *dev_dax)
+{
+	struct dax_device *dax_dev = dev_dax->dax_dev;
+	struct device *dev = &dev_dax->dev;
+	struct dev_pagemap *pgmap;
+	struct inode *inode;
+	struct cdev *cdev;
+	void *addr;
+	int rc, i;
+
+	if (static_dev_dax(dev_dax))  {
+		if (dev_dax->nr_range > 1) {
+			dev_warn(dev,
+				"static pgmap / multi-range device conflict\n");
+			return -EINVAL;
+		}
+
+		pgmap = dev_dax->pgmap;
+	} else {
+		if (dev_dax->pgmap) {
+			dev_warn(dev,
+				 "dynamic-dax with pre-populated page map\n");
+			return -EINVAL;
+		}
+
+		pgmap = devm_kzalloc(dev,
+                       struct_size(pgmap, ranges, dev_dax->nr_range - 1),
+                       GFP_KERNEL);
+		if (!pgmap)
+			return -ENOMEM;
+
+		pgmap->nr_range = dev_dax->nr_range;
+		dev_dax->pgmap = pgmap;
+
+		for (i = 0; i < dev_dax->nr_range; i++) {
+			struct range *range = &dev_dax->ranges[i].range;
+			pgmap->ranges[i] = *range;
+		}
+	}
+
+	for (i = 0; i < dev_dax->nr_range; i++) {
+		struct range *range = &dev_dax->ranges[i].range;
+
+		if (!devm_request_mem_region(dev, range->start,
+					range_len(range), dev_name(dev))) {
+			dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n",
+					i, range->start, range->end);
+			return -EBUSY;
+		}
+	}
+
+	pgmap->type = MEMORY_DEVICE_GENERIC;
+	if (dev_dax->align > PAGE_SIZE)
+		pgmap->vmemmap_shift =
+			order_base_2(dev_dax->align >> PAGE_SHIFT);
+	addr = devm_memremap_pages(dev, pgmap);
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
+
+	inode = dax_inode(dax_dev);
+	cdev = inode->i_cdev;
+	cdev_init(cdev, &dax_fops);
+	cdev->owner = dev->driver->owner;
+	cdev_set_parent(cdev, &dev->kobj);
+	rc = cdev_add(cdev, dev->devt, 1);
+	if (rc)
+		return rc;
+
+	rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev);
+	if (rc)
+		return rc;
+
+	run_dax(dax_dev);
+	return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
+}
+
+static struct dax_device_driver device_dax_driver = {
+	.probe = dev_dax_probe,
+	.type = DAXDRV_DEVICE_TYPE,
+};
+
+static int __init dax_init(void)
+{
+	return dax_driver_register(&device_dax_driver);
+}
+
+static void __exit dax_exit(void)
+{
+	dax_driver_unregister(&device_dax_driver);
+}
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_DESCRIPTION("Device DAX: direct access device driver");
+MODULE_LICENSE("GPL v2");
+module_init(dax_init);
+module_exit(dax_exit);
+MODULE_ALIAS_DAX_DEVICE(0);