Merge tag 'ipe-pr-20260925' of git://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipegrafted

Pull IPE fixes from Fan Wu:
 "Two fixes for use-after-free issues found by recent LLM-assisted code
  analysis.

   - move successful policy load auditing under the new policy
     directory's inode lock, preventing a concurrent policy deletion
     from freeing the policy while it is still being audited

   - protect the dm-verity root hash with RCU, preventing policy
     evaluation from racing with root hash replacement during preresume"

* tag 'ipe-pr-20260925' of git://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe:
  ipe: protect the dm-verity root hash with RCU
  ipe: fix use-after-free when auditing a newly loaded policy

10 files changed, 2096 insertions, 0 deletions

diff --git a/rust/macros/concat_idents.rs b/rust/macros/concat_idents.rs
new file mode 100644
index 000000000..47b6add37
--- /dev/null
+++ b/rust/macros/concat_idents.rs
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use proc_macro2::{
+    Ident,
+    TokenStream,
+    TokenTree, //
+};
+use syn::{
+    parse::{
+        Parse,
+        ParseStream, //
+    },
+    Result,
+    Token, //
+};
+
+pub(crate) struct Input {
+    a: Ident,
+    _comma: Token![,],
+    b: Ident,
+}
+
+impl Parse for Input {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        Ok(Self {
+            a: input.parse()?,
+            _comma: input.parse()?,
+            b: input.parse()?,
+        })
+    }
+}
+
+pub(crate) fn concat_idents(Input { a, b, .. }: Input) -> TokenStream {
+    let res = Ident::new(&format!("{a}{b}"), b.span());
+    TokenStream::from_iter([TokenTree::Ident(res)])
+}
diff --git a/rust/macros/export.rs b/rust/macros/export.rs
new file mode 100644
index 000000000..6d53521f6
--- /dev/null
+++ b/rust/macros/export.rs
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use proc_macro2::TokenStream;
+use quote::quote;
+
+/// Please see [`crate::export`] for documentation.
+pub(crate) fn export(f: syn::ItemFn) -> TokenStream {
+    let name = &f.sig.ident;
+
+    quote! {
+        // This verifies that the function has the same signature as the declaration generated by
+        // bindgen. It makes use of the fact that all branches of an if/else must have the same
+        // type.
+        const _: () = {
+            if true {
+                ::kernel::bindings::#name
+            } else {
+                #name
+            };
+        };
+
+        #[no_mangle]
+        #f
+    }
+}
diff --git a/rust/macros/fmt.rs b/rust/macros/fmt.rs
new file mode 100644
index 000000000..ce6c72493
--- /dev/null
+++ b/rust/macros/fmt.rs
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use std::collections::BTreeSet;
+
+use proc_macro2::{Ident, TokenStream, TokenTree};
+use quote::quote_spanned;
+
+/// Please see [`crate::fmt`] for documentation.
+pub(crate) fn fmt(input: TokenStream) -> TokenStream {
+    let mut input = input.into_iter();
+
+    let first_opt = input.next();
+    let first_owned_str;
+    let mut names = BTreeSet::new();
+    let first_span = {
+        let Some((mut first_str, first_span)) = (match first_opt.as_ref() {
+            Some(TokenTree::Literal(first_lit)) => {
+                first_owned_str = first_lit.to_string();
+                Some(first_owned_str.as_str()).and_then(|first| {
+                    let first = first.strip_prefix('"')?;
+                    let first = first.strip_suffix('"')?;
+                    Some((first, first_lit.span()))
+                })
+            }
+            _ => None,
+        }) else {
+            return first_opt.into_iter().chain(input).collect();
+        };
+
+        // Parse `identifier`s from the format string.
+        //
+        // See https://doc.rust-lang.org/std/fmt/index.html#syntax.
+        while let Some((_, rest)) = first_str.split_once('{') {
+            first_str = rest;
+            if let Some(rest) = first_str.strip_prefix('{') {
+                first_str = rest;
+                continue;
+            }
+            if let Some((name, rest)) = first_str.split_once('}') {
+                first_str = rest;
+                let name = name.split_once(':').map_or(name, |(name, _)| name);
+                if !name.is_empty() && !name.chars().all(|c| c.is_ascii_digit()) {
+                    names.insert(name);
+                }
+            }
+        }
+        first_span
+    };
+
+    let adapter = quote_spanned!(first_span => ::kernel::fmt::Adapter);
+
+    let mut args = TokenStream::from_iter(first_opt);
+    {
+        let mut flush = |args: &mut TokenStream, current: &mut TokenStream| {
+            let current = std::mem::take(current);
+            if !current.is_empty() {
+                let (lhs, rhs) = (|| {
+                    let mut current = current.into_iter();
+                    let mut acc = TokenStream::new();
+                    while let Some(tt) = current.next() {
+                        // Split on `=` only once to handle cases like `a = b = c`.
+                        if matches!(&tt, TokenTree::Punct(p) if p.as_char() == '=') {
+                            names.remove(acc.to_string().as_str());
+                            // Include the `=` itself to keep the handling below uniform.
+                            acc.extend([tt]);
+                            return (Some(acc), current.collect::<TokenStream>());
+                        }
+                        acc.extend([tt]);
+                    }
+                    (None, acc)
+                })();
+                args.extend(quote_spanned!(first_span => #lhs #adapter(&(#rhs))));
+            }
+        };
+
+        let mut current = TokenStream::new();
+        for tt in input {
+            match &tt {
+                TokenTree::Punct(p) if p.as_char() == ',' => {
+                    flush(&mut args, &mut current);
+                    &mut args
+                }
+                _ => &mut current,
+            }
+            .extend([tt]);
+        }
+        flush(&mut args, &mut current);
+    }
+
+    for name in names {
+        let name = Ident::new(name, first_span);
+        args.extend(quote_spanned!(first_span => , #name = #adapter(&#name)));
+    }
+
+    quote_spanned!(first_span => ::core::format_args!(#args))
+}
diff --git a/rust/macros/for_lt.rs b/rust/macros/for_lt.rs
new file mode 100644
index 000000000..4372cbad3
--- /dev/null
+++ b/rust/macros/for_lt.rs
@@ -0,0 +1,265 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use proc_macro2::{
+    Span,
+    TokenStream, //
+};
+use quote::{
+    format_ident,
+    quote, //
+};
+use syn::{
+    parse::{
+        Parse,
+        ParseStream, //
+    },
+    visit::Visit,
+    visit_mut::VisitMut,
+    Lifetime,
+    Result,
+    Token,
+    Type, //
+};
+
+pub(crate) enum HigherRankedType {
+    Explicit {
+        _for_token: Token![for],
+        _lt_token: Token![<],
+        lifetime: Lifetime,
+        _gt_token: Token![>],
+        ty: Type,
+    },
+    Implicit {
+        ty: Type,
+    },
+}
+
+impl Parse for HigherRankedType {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        if input.peek(Token![for]) {
+            Ok(Self::Explicit {
+                _for_token: input.parse()?,
+                _lt_token: input.parse()?,
+                lifetime: input.parse()?,
+                _gt_token: input.parse()?,
+                ty: input.parse()?,
+            })
+        } else {
+            Ok(Self::Implicit { ty: input.parse()? })
+        }
+    }
+}
+
+trait TypeExt {
+    fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type;
+    fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type;
+    fn has_lifetime(&self, lt: &Lifetime) -> bool;
+}
+
+impl TypeExt for Type {
+    fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type {
+        struct ElidedLifetimeExpander<'a>(&'a Lifetime);
+
+        impl VisitMut for ElidedLifetimeExpander<'_> {
+            fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) {
+                // Expand explicit `'_`
+                if lifetime.ident == "_" {
+                    *lifetime = self.0.clone();
+                }
+            }
+
+            fn visit_type_reference_mut(&mut self, reference: &mut syn::TypeReference) {
+                syn::visit_mut::visit_type_reference_mut(self, reference);
+
+                if reference.lifetime.is_none() {
+                    reference.lifetime = Some(self.0.clone());
+                }
+            }
+        }
+
+        let mut ret = self.clone();
+        ElidedLifetimeExpander(explicit_lt).visit_type_mut(&mut ret);
+        ret
+    }
+
+    fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type {
+        struct LifetimeReplacer<'a>(&'a Lifetime, &'a Lifetime);
+
+        impl VisitMut for LifetimeReplacer<'_> {
+            fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) {
+                if lifetime.ident == self.0.ident {
+                    *lifetime = self.1.clone();
+                }
+            }
+        }
+
+        let mut ret = self.clone();
+        LifetimeReplacer(src, dst).visit_type_mut(&mut ret);
+        ret
+    }
+
+    fn has_lifetime(&self, lt: &Lifetime) -> bool {
+        struct HasLifetime<'a>(&'a Lifetime, bool);
+
+        impl Visit<'_> for HasLifetime<'_> {
+            fn visit_lifetime(&mut self, lifetime: &Lifetime) {
+                if lifetime.ident == self.0.ident {
+                    self.1 = true;
+                }
+            }
+
+            // Macro invocations are opaque; conservatively assume they may
+            // reference the lifetime.
+            fn visit_macro(&mut self, _: &syn::Macro) {
+                self.1 = true;
+            }
+        }
+
+        let mut visitor = HasLifetime(lt, false);
+        visitor.visit_type(self);
+        visitor.1
+    }
+}
+
+struct Prover<'a>(&'a Lifetime, Vec<&'a Type>);
+
+impl<'a> Prover<'a> {
+    /// Prove that `ty` is covariant over `'lt`.
+    ///
+    /// This also needs to prove that it'll be wellformed for any instance of `'lt`.
+    /// It can be assumed that `ty` will be wellformed if `'lt` is substituted to `'static`.
+    fn prove(&mut self, ty: &'a Type) {
+        match ty {
+            Type::Paren(ty) => self.prove(&ty.elem),
+            Type::Group(ty) => self.prove(&ty.elem),
+
+            // No lifetime involved
+            Type::Never(_) => {}
+
+            // `[T; N]` and `[T]` is covariant over `T`.
+            Type::Array(ty) => self.prove(&ty.elem),
+            Type::Slice(ty) => self.prove(&ty.elem),
+
+            Type::Tuple(ty) => {
+                for elem in &ty.elems {
+                    self.prove(elem);
+                }
+            }
+
+            // `*const T` is covariant over `T`
+            Type::Ptr(ty) if ty.const_token.is_some() => self.prove(&ty.elem),
+
+            // `&T` is covariant over `T` and lifetime.
+            //
+            // Note that if we encounter `&'other_lt T`, then we still need to make sure the type
+            // is wellformed if `T` involves `&'lt`, so we defer to the compiler.
+            //
+            // This is to block cases like `CovariantForLt!(for<'a> &'static &'a u32)`, as the
+            // presence of the type implies `'a: 'static` but this is unsound.
+            Type::Reference(ty)
+                if ty.mutability.is_none() && ty.lifetime.as_ref() == Some(self.0) =>
+            {
+                self.prove(&ty.elem)
+            }
+
+            // `&[mut] T` is covariant over lifetime.
+            // In case we have `&[mut] NoLifetime`, we don't need to do additional checks.
+            Type::Reference(ty) if !ty.elem.has_lifetime(self.0) => (),
+
+            // No mention of lifetime at all, no need to perform compiler check.
+            ty if !ty.has_lifetime(self.0) => (),
+
+            // Otherwise, we need to emit checks so that compiler can determine if the types are
+            // actually covariant.
+            ty => self.1.push(ty),
+        }
+    }
+}
+
+/// Shared implementation for both `ForLt!` and `CovariantForLt!`.
+///
+/// Both macros run the prover and emit `ProveWf` structs to check well-formedness for all lifetime
+/// instances (workaround for <https://github.com/rust-lang/rust/issues/152489>). `CovariantForLt!`
+/// additionally emits covariance proof functions and sets `N = 1`.
+fn for_lt_inner(input: HigherRankedType, prove_covariance: bool) -> TokenStream {
+    let (ty, lifetime) = match input {
+        HigherRankedType::Explicit { lifetime, ty, .. } => (ty, lifetime),
+        HigherRankedType::Implicit { ty } => {
+            // If there's no explicit `for<'a>` binder, inject a synthetic `'__elided` lifetime
+            // and expand elided sites.
+            let lifetime = Lifetime {
+                apostrophe: Span::mixed_site(),
+                ident: format_ident!("__elided", span = Span::mixed_site()),
+            };
+            (ty.expand_elided_lifetime(&lifetime), lifetime)
+        }
+    };
+
+    let mut prover = Prover(&lifetime, Vec::new());
+    prover.prove(&ty);
+
+    let mut proof = Vec::new();
+
+    // Emit proofs for every type that requires additional compiler help in proving covariance.
+    for (idx, required_proof) in prover.1.into_iter().enumerate() {
+        // Insert a proof that the type is well-formed.
+        //
+        // This is intended to workaround a Rust compiler soundness bug related to HRTB.
+        // https://github.com/rust-lang/rust/issues/152489
+        //
+        // This needs to be a struct instead of fn to avoid the implied WF bounds.
+        let wf_proof_name = format_ident!("ProveWf{idx}");
+        proof.push(quote!(
+            struct #wf_proof_name<#lifetime>(
+                ::core::marker::PhantomData<&#lifetime ()>, #required_proof
+            );
+        ));
+
+        // Insert a proof that the type is covariant.
+        if prove_covariance {
+            let cov_proof_name = format_ident!("prove_covariant_{idx}");
+            proof.push(quote!(
+                fn #cov_proof_name<'__short, '__long: '__short>(
+                    long: #wf_proof_name<'__long>
+                ) -> #wf_proof_name<'__short> {
+                    long
+                }
+            ));
+        }
+    }
+
+    // Make sure that the type is wellformed when substituting lifetime with `'static`.
+    //
+    // Currently the Rust compiler doesn't check this, see the above `ProveWf` documentation.
+    //
+    // We prefer to use this way of proving WF-ness as it can work when generics are involved.
+    let ty_static = ty.replace_lifetime(
+        &lifetime,
+        &Lifetime {
+            apostrophe: Span::mixed_site(),
+            ident: format_ident!("static"),
+        },
+    );
+
+    let n: usize = prove_covariance.into();
+
+    quote!(
+        ::kernel::types::for_lt::UnsafeForLtImpl::<
+            dyn for<#lifetime> ::kernel::types::for_lt::WithLt<#lifetime, Of = #ty>,
+            #ty_static,
+            {
+                #(#proof)*
+
+                #n
+            }
+        >
+    )
+}
+
+pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream {
+    for_lt_inner(input, false)
+}
+
+pub(crate) fn covariant_for_lt(input: HigherRankedType) -> TokenStream {
+    for_lt_inner(input, true)
+}
diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs
new file mode 100644
index 000000000..d18fbf4da
--- /dev/null
+++ b/rust/macros/helpers.rs
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use proc_macro2::TokenStream;
+use quote::ToTokens;
+use syn::{
+    parse::{
+        Parse,
+        ParseStream, //
+    },
+    Attribute,
+    Error,
+    LitStr,
+    Result, //
+};
+
+/// A string literal that is required to have ASCII value only.
+pub(crate) struct AsciiLitStr(LitStr);
+
+impl Parse for AsciiLitStr {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        let s: LitStr = input.parse()?;
+        if !s.value().is_ascii() {
+            return Err(Error::new_spanned(s, "expected ASCII-only string literal"));
+        }
+        Ok(Self(s))
+    }
+}
+
+impl ToTokens for AsciiLitStr {
+    fn to_tokens(&self, ts: &mut TokenStream) {
+        self.0.to_tokens(ts);
+    }
+}
+
+impl AsciiLitStr {
+    pub(crate) fn value(&self) -> String {
+        self.0.value()
+    }
+}
+
+pub(crate) fn file() -> String {
+    #[cfg(not(CONFIG_RUSTC_HAS_SPAN_FILE))]
+    {
+        proc_macro::Span::call_site()
+            .source_file()
+            .path()
+            .to_string_lossy()
+            .into_owned()
+    }
+
+    #[cfg(CONFIG_RUSTC_HAS_SPAN_FILE)]
+    {
+        proc_macro::Span::call_site().file()
+    }
+}
+
+/// Obtain all `#[cfg]` attributes.
+pub(crate) fn gather_cfg_attrs(attr: &[Attribute]) -> impl Iterator<Item = &Attribute> + '_ {
+    attr.iter().filter(|a| a.path().is_ident("cfg"))
+}
diff --git a/rust/macros/kunit.rs b/rust/macros/kunit.rs
new file mode 100644
index 000000000..ae20ed676
--- /dev/null
+++ b/rust/macros/kunit.rs
@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Procedural macro to run KUnit tests using a user-space like syntax.
+//!
+//! Copyright (c) 2023 José Expósito <jose.exposito89@gmail.com>
+
+use std::ffi::CString;
+
+use proc_macro2::TokenStream;
+use quote::{
+    format_ident,
+    quote,
+    ToTokens, //
+};
+use syn::{
+    parse_quote,
+    Error,
+    Ident,
+    Item,
+    ItemMod,
+    LitCStr,
+    Result, //
+};
+
+pub(crate) fn kunit_tests(test_suite: Ident, mut module: ItemMod) -> Result<TokenStream> {
+    if test_suite.to_string().len() > 255 {
+        return Err(Error::new_spanned(
+            test_suite,
+            "test suite names cannot exceed the maximum length of 255 bytes",
+        ));
+    }
+
+    // We cannot handle modules that defer to another file (e.g. `mod foo;`).
+    let Some((module_brace, module_items)) = module.content.take() else {
+        Err(Error::new_spanned(
+            module,
+            "`#[kunit_tests(test_name)]` attribute should only be applied to inline modules",
+        ))?
+    };
+
+    // Make the entire module gated behind `CONFIG_KUNIT`.
+    module
+        .attrs
+        .insert(0, parse_quote!(#[cfg(CONFIG_KUNIT="y")]));
+
+    let mut processed_items = Vec::new();
+    let mut test_cases = Vec::new();
+
+    // Generate the test KUnit test suite and a test case for each `#[test]`.
+    //
+    // The code generated for the following test module:
+    //
+    // ```
+    // #[kunit_tests(kunit_test_suit_name)]
+    // mod tests {
+    //     #[test]
+    //     fn foo() {
+    //         assert_eq!(1, 1);
+    //     }
+    //
+    //     #[test]
+    //     fn bar() {
+    //         assert_eq!(2, 2);
+    //     }
+    // }
+    // ```
+    //
+    // Looks like:
+    //
+    // ```
+    // unsafe extern "C" fn kunit_rust_wrapper_foo(_test: *mut ::kernel::bindings::kunit) { foo(); }
+    // unsafe extern "C" fn kunit_rust_wrapper_bar(_test: *mut ::kernel::bindings::kunit) { bar(); }
+    //
+    // static mut TEST_CASES: [::kernel::bindings::kunit_case; 3] = [
+    //     ::kernel::kunit::kunit_case(c"foo", kunit_rust_wrapper_foo),
+    //     ::kernel::kunit::kunit_case(c"bar", kunit_rust_wrapper_bar),
+    //     ::pin_init::zeroed(),
+    // ];
+    //
+    // ::kernel::kunit_unsafe_test_suite!(kunit_test_suit_name, TEST_CASES);
+    // ```
+    //
+    // Non-function items (e.g. imports) are preserved.
+    for item in module_items {
+        let Item::Fn(mut f) = item else {
+            processed_items.push(item);
+            continue;
+        };
+
+        if f.attrs
+            .extract_if(.., |attr| attr.path().is_ident("test"))
+            .count()
+            == 0
+        {
+            processed_items.push(Item::Fn(f));
+            continue;
+        }
+
+        let test = f.sig.ident.clone();
+
+        // Retrieve `#[cfg]` applied on the function which needs to be present on derived items too.
+        let cfg_attrs: Vec<_> = f
+            .attrs
+            .iter()
+            .filter(|attr| attr.path().is_ident("cfg"))
+            .cloned()
+            .collect();
+
+        // Before the test, override usual `assert!` and `assert_eq!` macros with ones that call
+        // KUnit instead.
+        let test_str = test.to_string();
+        let path = CString::new(crate::helpers::file()).expect("file path cannot contain NUL");
+        processed_items.push(parse_quote! {
+            #[allow(unused)]
+            macro_rules! assert {
+                ($cond:expr $(,)?) => {{
+                    kernel::kunit_assert!(#test_str, #path, 0, $cond);
+                }}
+            }
+        });
+        processed_items.push(parse_quote! {
+            #[allow(unused)]
+            macro_rules! assert_eq {
+                ($left:expr, $right:expr $(,)?) => {{
+                    kernel::kunit_assert_eq!(#test_str, #path, 0, $left, $right);
+                }}
+            }
+        });
+
+        // Add back the test item.
+        processed_items.push(Item::Fn(f));
+
+        let kunit_wrapper_fn_name = format_ident!("kunit_rust_wrapper_{test}");
+        let test_cstr = LitCStr::new(
+            &CString::new(test_str.as_str()).expect("identifier cannot contain NUL"),
+            test.span(),
+        );
+        processed_items.push(parse_quote! {
+            unsafe extern "C" fn #kunit_wrapper_fn_name(_test: *mut ::kernel::bindings::kunit) {
+                (*_test).status = ::kernel::bindings::kunit_status_KUNIT_SKIPPED;
+
+                // Append any `cfg` attributes the user might have written on their tests so we
+                // don't attempt to call them when they are `cfg`'d out. An extra `use` is used
+                // here to reduce the length of the assert message.
+                #(#cfg_attrs)*
+                {
+                    (*_test).status = ::kernel::bindings::kunit_status_KUNIT_SUCCESS;
+                    use ::kernel::kunit::is_test_result_ok;
+                    assert!(is_test_result_ok(#test()));
+                }
+            }
+        });
+
+        test_cases.push(quote!(
+            ::kernel::kunit::kunit_case(#test_cstr, #kunit_wrapper_fn_name)
+        ));
+    }
+
+    let num_tests_plus_1 = test_cases.len() + 1;
+    processed_items.push(parse_quote! {
+        static mut TEST_CASES: [::kernel::bindings::kunit_case; #num_tests_plus_1] = [
+            #(#test_cases,)*
+            ::pin_init::zeroed(),
+        ];
+    });
+    processed_items.push(parse_quote! {
+        ::kernel::kunit_unsafe_test_suite!(#test_suite, TEST_CASES);
+    });
+
+    module.content = Some((module_brace, processed_items));
+    Ok(module.to_token_stream())
+}
diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs
new file mode 100644
index 000000000..24f96feae
--- /dev/null
+++ b/rust/macros/lib.rs
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Crate for all kernel procedural macros.
+
+// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT`
+// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is
+// touched by Kconfig when the version string from the compiler changes.
+
+// Stable since Rust 1.87.0.
+#![feature(extract_if)]
+//
+// Stable since Rust 1.88.0 under a different name, `proc_macro_span_file`,
+// which was added in Rust 1.88.0. This is why `cfg_attr` is used here, i.e.
+// to avoid depending on the full `proc_macro_span` on Rust >= 1.88.0.
+#![cfg_attr(not(CONFIG_RUSTC_HAS_SPAN_FILE), feature(proc_macro_span))]
+
+mod concat_idents;
+mod export;
+mod fmt;
+mod for_lt;
+mod helpers;
+mod kunit;
+mod module;
+mod paste;
+mod vtable;
+
+use proc_macro::TokenStream;
+
+use syn::parse_macro_input;
+
+/// Declares a kernel module.
+///
+/// The `type` argument should be a type which implements the [`Module`]
+/// trait. Also accepts various forms of kernel metadata.
+///
+/// The `params` field describe module parameters. Each entry has the form
+///
+/// ```ignore
+/// parameter_name: type {
+///     default: default_value,
+///     description: "Description",
+/// }
+/// ```
+///
+/// `type` may be one of
+///
+/// - [`i8`]
+/// - [`u8`]
+/// - [`i8`]
+/// - [`u8`]
+/// - [`i16`]
+/// - [`u16`]
+/// - [`i32`]
+/// - [`u32`]
+/// - [`i64`]
+/// - [`u64`]
+/// - [`isize`]
+/// - [`usize`]
+/// - [`bool`]
+///
+/// C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h)
+///
+/// [`Module`]: ../kernel/trait.Module.html
+///
+/// # Examples
+///
+/// ```ignore
+/// use kernel::prelude::*;
+///
+/// module!{
+///     type: MyModule,
+///     name: "my_kernel_module",
+///     authors: ["Rust for Linux Contributors"],
+///     description: "My very own kernel module!",
+///     license: "GPL",
+///     alias: ["alternate_module_name"],
+///     params: {
+///         my_parameter: i64 {
+///             default: 1,
+///             description: "This parameter has a default of 1",
+///         },
+///     },
+/// }
+///
+/// struct MyModule(i32);
+///
+/// impl kernel::Module for MyModule {
+///     fn init(_module: &'static ThisModule) -> Result<Self> {
+///         let foo: i32 = 42;
+///         pr_info!("I contain:  {}\n", foo);
+///         pr_info!("i32 param is:  {}\n", module_parameters::my_parameter.read());
+///         Ok(Self(foo))
+///     }
+/// }
+/// # fn main() {}
+/// ```
+///
+/// ## Firmware
+///
+/// The following example shows how to declare a kernel module that needs
+/// to load binary firmware files. You need to specify the file names of
+/// the firmware in the `firmware` field. The information is embedded
+/// in the `modinfo` section of the kernel module. For example, a tool to
+/// build an initramfs uses this information to put the firmware files into
+/// the initramfs image.
+///
+/// ```
+/// use kernel::prelude::*;
+///
+/// module!{
+///     type: MyDeviceDriverModule,
+///     name: "my_device_driver_module",
+///     authors: ["Rust for Linux Contributors"],
+///     description: "My device driver requires firmware",
+///     license: "GPL",
+///     firmware: ["my_device_firmware1.bin", "my_device_firmware2.bin"],
+/// }
+///
+/// struct MyDeviceDriverModule;
+///
+/// impl kernel::Module for MyDeviceDriverModule {
+///     fn init(_module: &'static ThisModule) -> Result<Self> {
+///         Ok(Self)
+///     }
+/// }
+/// # fn main() {}
+/// ```
+///
+/// # Supported argument types
+///   - `type`: type which implements the [`Module`] trait (required).
+///   - `name`: ASCII string literal of the name of the kernel module (required).
+///   - `authors`: array of ASCII string literals of the authors of the kernel module.
+///   - `description`: string literal of the description of the kernel module.
+///   - `license`: ASCII string literal of the license of the kernel module (required).
+///   - `alias`: array of ASCII string literals of the alias names of the kernel module.
+///   - `firmware`: array of ASCII string literals of the firmware files of
+///     the kernel module.
+#[proc_macro]
+pub fn module(input: TokenStream) -> TokenStream {
+    module::module(parse_macro_input!(input))
+        .unwrap_or_else(|e| e.into_compile_error())
+        .into()
+}
+
+/// Declares or implements a vtable trait.
+///
+/// Linux's use of pure vtables is very close to Rust traits, but they differ
+/// in how unimplemented functions are represented. In Rust, traits can provide
+/// default implementation for all non-required methods (and the default
+/// implementation could just return `Error::EINVAL`); Linux typically use C
+/// `NULL` pointers to represent these functions.
+///
+/// This attribute closes that gap. A trait can be annotated with the
+/// `#[vtable]` attribute. Implementers of the trait will then also have to
+/// annotate the trait with `#[vtable]`. This attribute generates a `HAS_*`
+/// associated constant bool for each method in the trait that is set to true if
+/// the implementer has overridden the associated method.
+///
+/// For a trait method to be optional, it must have a default implementation.
+/// This is also the case for traits annotated with `#[vtable]`, but in this
+/// case the default implementation will never be executed. The reason for this
+/// is that the functions will be called through function pointers installed in
+/// C side vtables. When an optional method is not implemented on a `#[vtable]`
+/// trait, a `NULL` entry is installed in the vtable. Thus the default
+/// implementation is never called. Since these traits are not designed to be
+/// used on the Rust side, it should not be possible to call the default
+/// implementation. This is done to ensure that we call the vtable methods
+/// through the C vtable, and not through the Rust vtable. Therefore, the
+/// default implementation should call `build_error!`, which prevents
+/// calls to this function at compile time:
+///
+/// ```compile_fail
+/// # // Intentionally missing `use`s to simplify `rusttest`.
+/// build_error!(VTABLE_DEFAULT_ERROR)
+/// ```
+///
+/// Note that you might need to import [`kernel::error::VTABLE_DEFAULT_ERROR`].
+///
+/// This macro should not be used when all functions are required.
+///
+/// Additionally, this macro automatically handles the `OwnerModule`
+/// associated type: on the trait side, `type OwnerModule: ModuleMetadata;`
+/// is added as a required associated type if not already defined; on the
+/// impl side, `type OwnerModule = LocalModule;` is automatically inserted
+/// if not explicitly defined.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::error::VTABLE_DEFAULT_ERROR;
+/// use kernel::prelude::*;
+///
+/// # struct LocalModule;
+/// # impl kernel::ModuleMetadata for LocalModule {
+/// #     const NAME: &'static kernel::str::CStr = c"vtable_doctest";
+/// #
+/// #     // SAFETY: This doctest runs on the host: there is no `THIS_MODULE`.
+/// #     const THIS_MODULE: kernel::ThisModule = unsafe {
+/// #         kernel::ThisModule::from_ptr(core::ptr::null_mut())
+/// #     };
+/// # }
+/// #
+/// # fn main() {
+/// // Declares a `#[vtable]` trait
+/// #[vtable]
+/// pub trait Operations: Send + Sync + Sized {
+///     fn foo(&self) -> Result<()> {
+///         build_error!(VTABLE_DEFAULT_ERROR)
+///     }
+///
+///     fn bar(&self) -> Result<()> {
+///         build_error!(VTABLE_DEFAULT_ERROR)
+///     }
+/// }
+///
+/// struct Foo;
+///
+/// // Implements the `#[vtable]` trait
+/// #[vtable]
+/// impl Operations for Foo {
+///     fn foo(&self) -> Result<()> {
+/// #        Err(EINVAL)
+///         // ...
+///     }
+/// }
+///
+/// assert_eq!(<Foo as Operations>::HAS_FOO, true);
+/// assert_eq!(<Foo as Operations>::HAS_BAR, false);
+/// # }
+/// ```
+///
+/// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html
+#[proc_macro_attribute]
+pub fn vtable(attr: TokenStream, input: TokenStream) -> TokenStream {
+    parse_macro_input!(attr as syn::parse::Nothing);
+    vtable::vtable(parse_macro_input!(input))
+        .unwrap_or_else(|e| e.into_compile_error())
+        .into()
+}
+
+/// Export a function so that C code can call it via a header file.
+///
+/// Functions exported using this macro can be called from C code using the declaration in the
+/// appropriate header file. It should only be used in cases where C calls the function through a
+/// header file; cases where C calls into Rust via a function pointer in a vtable (such as
+/// `file_operations`) should not use this macro.
+///
+/// This macro has the following effect:
+///
+/// * Disables name mangling for this function.
+/// * Verifies at compile-time that the function signature matches the declaration in the header
+///   file.
+///
+/// You must declare the signature of the Rust function in a header file that is included by
+/// `rust/bindings/bindings_helper.h`.
+///
+/// This macro is *not* the same as the C macros `EXPORT_SYMBOL_*`. All Rust symbols are currently
+/// automatically exported with `EXPORT_SYMBOL_GPL`.
+#[proc_macro_attribute]
+pub fn export(attr: TokenStream, input: TokenStream) -> TokenStream {
+    parse_macro_input!(attr as syn::parse::Nothing);
+    export::export(parse_macro_input!(input)).into()
+}
+
+/// Like [`core::format_args!`], but automatically wraps arguments in [`kernel::fmt::Adapter`].
+///
+/// This macro allows generating `fmt::Arguments` while ensuring that each argument is wrapped with
+/// `::kernel::fmt::Adapter`, which customizes formatting behavior for kernel logging.
+///
+/// Named arguments used in the format string (e.g. `{foo}`) are detected and resolved from local
+/// bindings. All positional and named arguments are automatically wrapped.
+///
+/// This macro is an implementation detail of other kernel logging macros like [`pr_info!`] and
+/// should not typically be used directly.
+///
+/// [`kernel::fmt::Adapter`]: ../kernel/fmt/struct.Adapter.html
+/// [`pr_info!`]: ../kernel/macro.pr_info.html
+#[proc_macro]
+pub fn fmt(input: TokenStream) -> TokenStream {
+    fmt::fmt(input.into()).into()
+}
+
+/// Concatenate two identifiers.
+///
+/// This is useful in macros that need to declare or reference items with names
+/// starting with a fixed prefix and ending in a user specified name. The resulting
+/// identifier has the span of the second argument.
+///
+/// # Examples
+///
+/// ```
+/// # const binder_driver_return_protocol_BR_OK: u32 = 0;
+/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1;
+/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2;
+/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3;
+/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4;
+/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5;
+/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6;
+/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7;
+/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8;
+/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9;
+/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10;
+/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11;
+/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12;
+/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13;
+/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14;
+/// use kernel::macros::concat_idents;
+///
+/// macro_rules! pub_no_prefix {
+///     ($prefix:ident, $($newname:ident),+) => {
+///         $(pub(crate) const $newname: u32 = concat_idents!($prefix, $newname);)+
+///     };
+/// }
+///
+/// pub_no_prefix!(
+///     binder_driver_return_protocol_,
+///     BR_OK,
+///     BR_ERROR,
+///     BR_TRANSACTION,
+///     BR_REPLY,
+///     BR_DEAD_REPLY,
+///     BR_TRANSACTION_COMPLETE,
+///     BR_INCREFS,
+///     BR_ACQUIRE,
+///     BR_RELEASE,
+///     BR_DECREFS,
+///     BR_NOOP,
+///     BR_SPAWN_LOOPER,
+///     BR_DEAD_BINDER,
+///     BR_CLEAR_DEATH_NOTIFICATION_DONE,
+///     BR_FAILED_REPLY
+/// );
+///
+/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK);
+/// ```
+#[proc_macro]
+pub fn concat_idents(input: TokenStream) -> TokenStream {
+    concat_idents::concat_idents(parse_macro_input!(input)).into()
+}
+
+/// Paste identifiers together.
+///
+/// Within the `paste!` macro, identifiers inside `[<` and `>]` are concatenated together to form a
+/// single identifier.
+///
+/// This is similar to the [`paste`] crate, but with pasting feature limited to identifiers and
+/// literals (lifetimes and documentation strings are not supported). There is a difference in
+/// supported modifiers as well.
+///
+/// # Examples
+///
+/// ```
+/// # const binder_driver_return_protocol_BR_OK: u32 = 0;
+/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1;
+/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2;
+/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3;
+/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4;
+/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5;
+/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6;
+/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7;
+/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8;
+/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9;
+/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10;
+/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11;
+/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12;
+/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13;
+/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14;
+/// macro_rules! pub_no_prefix {
+///     ($prefix:ident, $($newname:ident),+) => {
+///         ::kernel::macros::paste! {
+///             $(pub(crate) const $newname: u32 = [<$prefix $newname>];)+
+///         }
+///     };
+/// }
+///
+/// pub_no_prefix!(
+///     binder_driver_return_protocol_,
+///     BR_OK,
+///     BR_ERROR,
+///     BR_TRANSACTION,
+///     BR_REPLY,
+///     BR_DEAD_REPLY,
+///     BR_TRANSACTION_COMPLETE,
+///     BR_INCREFS,
+///     BR_ACQUIRE,
+///     BR_RELEASE,
+///     BR_DECREFS,
+///     BR_NOOP,
+///     BR_SPAWN_LOOPER,
+///     BR_DEAD_BINDER,
+///     BR_CLEAR_DEATH_NOTIFICATION_DONE,
+///     BR_FAILED_REPLY
+/// );
+///
+/// assert_eq!(BR_OK, binder_driver_return_protocol_BR_OK);
+/// ```
+///
+/// # Modifiers
+///
+/// For each identifier, it is possible to attach one or multiple modifiers to
+/// it.
+///
+/// Currently supported modifiers are:
+/// * `span`: change the span of concatenated identifier to the span of the specified token. By
+///   default the span of the `[< >]` group is used.
+/// * `lower`: change the identifier to lower case.
+/// * `upper`: change the identifier to upper case.
+///
+/// ```
+/// # const binder_driver_return_protocol_BR_OK: u32 = 0;
+/// # const binder_driver_return_protocol_BR_ERROR: u32 = 1;
+/// # const binder_driver_return_protocol_BR_TRANSACTION: u32 = 2;
+/// # const binder_driver_return_protocol_BR_REPLY: u32 = 3;
+/// # const binder_driver_return_protocol_BR_DEAD_REPLY: u32 = 4;
+/// # const binder_driver_return_protocol_BR_TRANSACTION_COMPLETE: u32 = 5;
+/// # const binder_driver_return_protocol_BR_INCREFS: u32 = 6;
+/// # const binder_driver_return_protocol_BR_ACQUIRE: u32 = 7;
+/// # const binder_driver_return_protocol_BR_RELEASE: u32 = 8;
+/// # const binder_driver_return_protocol_BR_DECREFS: u32 = 9;
+/// # const binder_driver_return_protocol_BR_NOOP: u32 = 10;
+/// # const binder_driver_return_protocol_BR_SPAWN_LOOPER: u32 = 11;
+/// # const binder_driver_return_protocol_BR_DEAD_BINDER: u32 = 12;
+/// # const binder_driver_return_protocol_BR_CLEAR_DEATH_NOTIFICATION_DONE: u32 = 13;
+/// # const binder_driver_return_protocol_BR_FAILED_REPLY: u32 = 14;
+/// macro_rules! pub_no_prefix {
+///     ($prefix:ident, $($newname:ident),+) => {
+///         ::kernel::macros::paste! {
+///             $(pub(crate) const fn [<$newname:lower:span>]() -> u32 { [<$prefix $newname:span>] })+
+///         }
+///     };
+/// }
+///
+/// pub_no_prefix!(
+///     binder_driver_return_protocol_,
+///     BR_OK,
+///     BR_ERROR,
+///     BR_TRANSACTION,
+///     BR_REPLY,
+///     BR_DEAD_REPLY,
+///     BR_TRANSACTION_COMPLETE,
+///     BR_INCREFS,
+///     BR_ACQUIRE,
+///     BR_RELEASE,
+///     BR_DECREFS,
+///     BR_NOOP,
+///     BR_SPAWN_LOOPER,
+///     BR_DEAD_BINDER,
+///     BR_CLEAR_DEATH_NOTIFICATION_DONE,
+///     BR_FAILED_REPLY
+/// );
+///
+/// assert_eq!(br_ok(), binder_driver_return_protocol_BR_OK);
+/// ```
+///
+/// # Literals
+///
+/// Literals can also be concatenated with other identifiers:
+///
+/// ```
+/// macro_rules! create_numbered_fn {
+///     ($name:literal, $val:literal) => {
+///         ::kernel::macros::paste! {
+///             fn [<some_ $name _fn $val>]() -> u32 { $val }
+///         }
+///     };
+/// }
+///
+/// create_numbered_fn!("foo", 100);
+///
+/// assert_eq!(some_foo_fn100(), 100)
+/// ```
+///
+/// [`paste`]: https://docs.rs/paste/
+#[proc_macro]
+pub fn paste(input: TokenStream) -> TokenStream {
+    let mut tokens = proc_macro2::TokenStream::from(input).into_iter().collect();
+    paste::expand(&mut tokens);
+    tokens
+        .into_iter()
+        .collect::<proc_macro2::TokenStream>()
+        .into()
+}
+
+/// Registers a KUnit test suite and its test cases using a user-space like syntax.
+///
+/// This macro should be used on modules. If `CONFIG_KUNIT` (in `.config`) is `n`, the target module
+/// is ignored.
+///
+/// # Examples
+///
+/// ```ignore
+/// # use kernel::prelude::*;
+/// #[kunit_tests(kunit_test_suit_name)]
+/// mod tests {
+///     #[test]
+///     fn foo() {
+///         assert_eq!(1, 1);
+///     }
+///
+///     #[test]
+///     fn bar() {
+///         assert_eq!(2, 2);
+///     }
+/// }
+/// ```
+#[proc_macro_attribute]
+pub fn kunit_tests(attr: TokenStream, input: TokenStream) -> TokenStream {
+    kunit::kunit_tests(parse_macro_input!(attr), parse_macro_input!(input))
+        .unwrap_or_else(|e| e.into_compile_error())
+        .into()
+}
+
+/// Obtain a type that implements [`ForLt`] for the given higher-ranked type.
+///
+/// Please refer to the documentation of the [`ForLt`] trait.
+///
+/// [`ForLt`]: trait.ForLt.html
+#[proc_macro]
+#[allow(non_snake_case)]
+pub fn ForLt(input: TokenStream) -> TokenStream {
+    for_lt::for_lt(parse_macro_input!(input)).into()
+}
+
+/// Obtain a type that implements [`CovariantForLt`] (and [`ForLt`]) for the given higher-ranked
+/// type.
+///
+/// Unlike [`ForLt!`], this macro additionally proves that the type is covariant over the lifetime,
+/// providing a safe [`CovariantForLt::cast_ref`] method.
+///
+/// Please refer to the documentation of the [`CovariantForLt`] trait.
+///
+/// [`CovariantForLt`]: trait.CovariantForLt.html
+/// [`CovariantForLt::cast_ref`]: trait.CovariantForLt.html#method.cast_ref
+/// [`ForLt`]: trait.ForLt.html
+#[proc_macro]
+#[allow(non_snake_case)]
+pub fn CovariantForLt(input: TokenStream) -> TokenStream {
+    for_lt::covariant_for_lt(parse_macro_input!(input)).into()
+}
diff --git a/rust/macros/module.rs b/rust/macros/module.rs
new file mode 100644
index 000000000..bc7027f8d
--- /dev/null
+++ b/rust/macros/module.rs
@@ -0,0 +1,654 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use std::ffi::CString;
+
+use proc_macro2::{
+    Literal,
+    TokenStream, //
+};
+use quote::{
+    format_ident,
+    quote, //
+};
+use syn::{
+    braced,
+    bracketed,
+    ext::IdentExt,
+    parse::{
+        Parse,
+        ParseStream, //
+    },
+    parse_quote,
+    punctuated::Punctuated,
+    Error,
+    Expr,
+    Ident,
+    LitStr,
+    Path,
+    Result,
+    Token,
+    Type, //
+};
+
+use crate::helpers::*;
+
+struct ModInfoBuilder<'a> {
+    module: &'a str,
+    counter: usize,
+    ts: TokenStream,
+    param_ts: TokenStream,
+}
+
+impl<'a> ModInfoBuilder<'a> {
+    fn new(module: &'a str) -> Self {
+        ModInfoBuilder {
+            module,
+            counter: 0,
+            ts: TokenStream::new(),
+            param_ts: TokenStream::new(),
+        }
+    }
+
+    fn emit_base(&mut self, field: &str, content: &str, builtin: bool, param: bool) {
+        let string = if builtin {
+            // Built-in modules prefix their modinfo strings by `module.`.
+            format!("{module}.{field}={content}\0", module = self.module)
+        } else {
+            // Loadable modules' modinfo strings go as-is.
+            format!("{field}={content}\0")
+        };
+        let length = string.len();
+        let string = Literal::byte_string(string.as_bytes());
+        let cfg = if builtin {
+            quote!(#[cfg(not(MODULE))])
+        } else {
+            quote!(#[cfg(MODULE)])
+        };
+
+        let counter = format_ident!(
+            "__{module}_{counter}",
+            module = self.module.to_uppercase(),
+            counter = self.counter
+        );
+        let item = quote! {
+            #cfg
+            #[cfg_attr(not(target_os = "macos"), link_section = ".modinfo")]
+            #[used(compiler)]
+            pub static #counter: [u8; #length] = *#string;
+        };
+
+        if param {
+            self.param_ts.extend(item);
+        } else {
+            self.ts.extend(item);
+        }
+
+        self.counter += 1;
+    }
+
+    fn emit_only_builtin(&mut self, field: &str, content: &str, param: bool) {
+        self.emit_base(field, content, true, param)
+    }
+
+    fn emit_only_loadable(&mut self, field: &str, content: &str, param: bool) {
+        self.emit_base(field, content, false, param)
+    }
+
+    fn emit(&mut self, field: &str, content: &str) {
+        self.emit_internal(field, content, false);
+    }
+
+    fn emit_internal(&mut self, field: &str, content: &str, param: bool) {
+        self.emit_only_builtin(field, content, param);
+        self.emit_only_loadable(field, content, param);
+    }
+
+    fn emit_param(&mut self, field: &str, param: &str, content: &str) {
+        let content = format!("{param}:{content}");
+        self.emit_internal(field, &content, true);
+    }
+
+    fn emit_params(&mut self, info: &ModuleInfo) {
+        let Some(params) = &info.params else {
+            return;
+        };
+
+        for param in params {
+            let param_name_str = param.name.to_string();
+            let param_type_str = param.ptype.to_string();
+
+            let ops = param_ops_path(&param_type_str);
+
+            // Note: The spelling of these fields is dictated by the user space
+            // tool `modinfo`.
+            self.emit_param("parmtype", &param_name_str, &param_type_str);
+            self.emit_param("parm", &param_name_str, &param.description.value());
+
+            let static_name = format_ident!("__{}_{}_struct", self.module, param.name);
+            let param_name_cstr =
+                CString::new(param_name_str).expect("name contains NUL-terminator");
+            let param_name_cstr_with_module =
+                CString::new(format!("{}.{}", self.module, param.name))
+                    .expect("name contains NUL-terminator");
+
+            let param_name = &param.name;
+            let param_type = &param.ptype;
+            let param_default = &param.default;
+
+            self.param_ts.extend(quote! {
+                #[allow(non_upper_case_globals)]
+                pub(crate) static #param_name:
+                    ::kernel::module_param::ModuleParamAccess<#param_type> =
+                        ::kernel::module_param::ModuleParamAccess::new(#param_default);
+
+                const _: () = {
+                    #[allow(non_upper_case_globals)]
+                    #[link_section = "__param"]
+                    #[used(compiler)]
+                    static #static_name:
+                        ::kernel::module_param::KernelParam =
+                        ::kernel::module_param::KernelParam::new(
+                            ::kernel::bindings::kernel_param {
+                                name: kernel::str::as_char_ptr_in_const_context(
+                                    if ::core::cfg!(MODULE) {
+                                        #param_name_cstr
+                                    } else {
+                                        #param_name_cstr_with_module
+                                    }
+                                ),
+                                // SAFETY: `__this_module` is constructed by the kernel at load
+                                // time and will not be freed until the module is unloaded.
+                                #[cfg(MODULE)]
+                                mod_: unsafe {
+                                    core::ptr::from_ref(&::kernel::bindings::__this_module)
+                                        .cast_mut()
+                                },
+                                #[cfg(not(MODULE))]
+                                mod_: ::core::ptr::null_mut(),
+                                ops: core::ptr::from_ref(&#ops),
+                                perm: 0, // Will not appear in sysfs
+                                level: -1,
+                                flags: 0,
+                                __bindgen_anon_1: ::kernel::bindings::kernel_param__bindgen_ty_1 {
+                                    arg: #param_name.as_void_ptr()
+                                },
+                            }
+                        );
+                };
+            });
+        }
+    }
+}
+
+fn param_ops_path(param_type: &str) -> Path {
+    match param_type {
+        "i8" => parse_quote!(::kernel::module_param::PARAM_OPS_I8),
+        "u8" => parse_quote!(::kernel::module_param::PARAM_OPS_U8),
+        "i16" => parse_quote!(::kernel::module_param::PARAM_OPS_I16),
+        "u16" => parse_quote!(::kernel::module_param::PARAM_OPS_U16),
+        "i32" => parse_quote!(::kernel::module_param::PARAM_OPS_I32),
+        "u32" => parse_quote!(::kernel::module_param::PARAM_OPS_U32),
+        "i64" => parse_quote!(::kernel::module_param::PARAM_OPS_I64),
+        "u64" => parse_quote!(::kernel::module_param::PARAM_OPS_U64),
+        "isize" => parse_quote!(::kernel::module_param::PARAM_OPS_ISIZE),
+        "usize" => parse_quote!(::kernel::module_param::PARAM_OPS_USIZE),
+        "bool" => parse_quote!(::kernel::module_param::PARAM_OPS_BOOL),
+        t => panic!("Unsupported parameter type {}", t),
+    }
+}
+
+/// Parse fields that are required to use a specific order.
+///
+/// As fields must follow a specific order, we *could* just parse fields one by one by peeking.
+/// However the error message generated when implementing that way is not very friendly.
+///
+/// So instead we parse fields in an arbitrary order, but only enforce the ordering after parsing,
+/// and if the wrong order is used, the proper order is communicated to the user with error message.
+///
+/// Usage looks like this:
+/// ```ignore
+/// parse_ordered_fields! {
+///     from input;
+///
+///     // This will extract "foo: <field>" into a variable named "foo".
+///     // The variable will have type `Option<_>`.
+///     foo => <expression that parses the field>,
+///
+///     // If you need the variable name to be different than the key name.
+///     // This extracts "baz: <field>" into a variable named "bar".
+///     // You might want this if "baz" is a keyword.
+///     baz as bar => <expression that parse the field>,
+///
+///     // You can mark a key as required, and the variable will no longer be `Option`.
+///     // foobar will be of type `Expr` instead of `Option<Expr>`.
+///     foobar [required] => input.parse::<Expr>()?,
+/// }
+/// ```
+macro_rules! parse_ordered_fields {
+    (@gen
+        [$input:expr]
+        [$([$name:ident; $key:ident; $parser:expr])*]
+        [$([$req_name:ident; $req_key:ident])*]
+    ) => {
+        $(let mut $name = None;)*
+
+        const EXPECTED_KEYS: &[&str] = &[$(stringify!($key),)*];
+        const REQUIRED_KEYS: &[&str] = &[$(stringify!($req_key),)*];
+
+        let span = $input.span();
+        let mut seen_keys = Vec::new();
+
+        while !$input.is_empty() {
+            let key = $input.call(Ident::parse_any)?;
+
+            if seen_keys.contains(&key) {
+                Err(Error::new_spanned(
+                    &key,
+                    format!(r#"duplicated key "{key}". Keys can only be specified once."#),
+                ))?
+            }
+
+            $input.parse::<Token![:]>()?;
+
+            match &*key.to_string() {
+                $(
+                    stringify!($key) => $name = Some($parser),
+                )*
+                _ => {
+                    Err(Error::new_spanned(
+                        &key,
+                        format!(r#"unknown key "{key}". Valid keys are: {EXPECTED_KEYS:?}."#),
+                    ))?
+                }
+            }
+
+            $input.parse::<Token![,]>()?;
+            seen_keys.push(key);
+        }
+
+        for key in REQUIRED_KEYS {
+            if !seen_keys.iter().any(|e| e == key) {
+                Err(Error::new(span, format!(r#"missing required key "{key}""#)))?
+            }
+        }
+
+        let mut ordered_keys: Vec<&str> = Vec::new();
+        for key in EXPECTED_KEYS {
+            if seen_keys.iter().any(|e| e == key) {
+                ordered_keys.push(key);
+            }
+        }
+
+        if seen_keys != ordered_keys {
+            Err(Error::new(
+                span,
+                format!(r#"keys are not ordered as expected. Order them like: {ordered_keys:?}."#),
+            ))?
+        }
+
+        $(let $req_name = $req_name.expect("required field");)*
+    };
+
+    // Handle required fields.
+    (@gen
+        [$input:expr] [$($tok:tt)*] [$($req:tt)*]
+        $key:ident as $name:ident [required] => $parser:expr,
+        $($rest:tt)*
+    ) => {
+        parse_ordered_fields!(
+            @gen [$input] [$($tok)* [$name; $key; $parser]] [$($req)* [$name; $key]] $($rest)*
+        )
+    };
+    (@gen
+        [$input:expr] [$($tok:tt)*] [$($req:tt)*]
+        $name:ident [required] => $parser:expr,
+        $($rest:tt)*
+    ) => {
+        parse_ordered_fields!(
+            @gen [$input] [$($tok)* [$name; $name; $parser]] [$($req)* [$name; $name]] $($rest)*
+        )
+    };
+
+    // Handle optional fields.
+    (@gen
+        [$input:expr] [$($tok:tt)*] [$($req:tt)*]
+        $key:ident as $name:ident => $parser:expr,
+        $($rest:tt)*
+    ) => {
+        parse_ordered_fields!(
+            @gen [$input] [$($tok)* [$name; $key; $parser]] [$($req)*] $($rest)*
+        )
+    };
+    (@gen
+        [$input:expr] [$($tok:tt)*] [$($req:tt)*]
+        $name:ident => $parser:expr,
+        $($rest:tt)*
+    ) => {
+        parse_ordered_fields!(
+            @gen [$input] [$($tok)* [$name; $name; $parser]] [$($req)*] $($rest)*
+        )
+    };
+
+    (from $input:expr; $($tok:tt)*) => {
+        parse_ordered_fields!(@gen [$input] [] [] $($tok)*)
+    }
+}
+
+struct Parameter {
+    name: Ident,
+    ptype: Ident,
+    default: Expr,
+    description: LitStr,
+}
+
+impl Parse for Parameter {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        let name = input.parse()?;
+        input.parse::<Token![:]>()?;
+        let ptype = input.parse()?;
+
+        let fields;
+        braced!(fields in input);
+
+        parse_ordered_fields! {
+            from fields;
+            default [required] => fields.parse()?,
+            description [required] => fields.parse()?,
+        }
+
+        Ok(Self {
+            name,
+            ptype,
+            default,
+            description,
+        })
+    }
+}
+
+pub(crate) struct ModuleInfo {
+    type_: Type,
+    license: AsciiLitStr,
+    name: AsciiLitStr,
+    authors: Option<Punctuated<AsciiLitStr, Token![,]>>,
+    description: Option<LitStr>,
+    alias: Option<Punctuated<AsciiLitStr, Token![,]>>,
+    firmware: Option<Punctuated<AsciiLitStr, Token![,]>>,
+    imports_ns: Option<Punctuated<AsciiLitStr, Token![,]>>,
+    params: Option<Punctuated<Parameter, Token![,]>>,
+}
+
+impl Parse for ModuleInfo {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        parse_ordered_fields!(
+            from input;
+            type as type_ [required] => input.parse()?,
+            name [required] => input.parse()?,
+            authors => {
+                let list;
+                bracketed!(list in input);
+                Punctuated::parse_terminated(&list)?
+            },
+            description => input.parse()?,
+            license [required] => input.parse()?,
+            alias => {
+                let list;
+                bracketed!(list in input);
+                Punctuated::parse_terminated(&list)?
+            },
+            firmware => {
+                let list;
+                bracketed!(list in input);
+                Punctuated::parse_terminated(&list)?
+            },
+            imports_ns => {
+                let list;
+                bracketed!(list in input);
+                Punctuated::parse_terminated(&list)?
+            },
+            params => {
+                let list;
+                braced!(list in input);
+                Punctuated::parse_terminated(&list)?
+            },
+        );
+
+        Ok(ModuleInfo {
+            type_,
+            license,
+            name,
+            authors,
+            description,
+            alias,
+            firmware,
+            imports_ns,
+            params,
+        })
+    }
+}
+
+pub(crate) fn module(info: ModuleInfo) -> Result<TokenStream> {
+    let ModuleInfo {
+        type_,
+        license,
+        name,
+        authors,
+        description,
+        alias,
+        firmware,
+        imports_ns,
+        params: _,
+    } = &info;
+
+    // Rust does not allow hyphens in identifiers, use underscore instead.
+    let ident = name.value().replace('-', "_");
+    let mut modinfo = ModInfoBuilder::new(ident.as_ref());
+    if let Some(authors) = authors {
+        for author in authors {
+            modinfo.emit("author", &author.value());
+        }
+    }
+    if let Some(description) = description {
+        modinfo.emit("description", &description.value());
+    }
+    modinfo.emit("license", &license.value());
+    if let Some(aliases) = alias {
+        for alias in aliases {
+            modinfo.emit("alias", &alias.value());
+        }
+    }
+    if let Some(firmware) = firmware {
+        for fw in firmware {
+            modinfo.emit("firmware", &fw.value());
+        }
+    }
+    if let Some(imports) = imports_ns {
+        for ns in imports {
+            modinfo.emit("import_ns", &ns.value());
+        }
+    }
+
+    // Built-in modules also export the `file` modinfo string.
+    let file =
+        std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable");
+    modinfo.emit_only_builtin("file", &file, false);
+
+    modinfo.emit_params(&info);
+
+    let modinfo_ts = modinfo.ts;
+    let params_ts = modinfo.param_ts;
+
+    let ident_init = format_ident!("__{ident}_init");
+    let ident_exit = format_ident!("__{ident}_exit");
+    let ident_initcall = format_ident!("__{ident}_initcall");
+    let initcall_section = ".initcall6.init";
+
+    let global_asm = format!(
+        r#".section "{initcall_section}", "a"
+        __{ident}_initcall:
+            .long   __{ident}_init - .
+            .previous
+        "#
+    );
+
+    let name_cstr = CString::new(name.value()).expect("name contains NUL-terminator");
+
+    Ok(quote! {
+        /// The module name.
+        ///
+        /// Used by the printing macros, e.g. [`info!`].
+        const __LOG_PREFIX: &[u8] = #name_cstr.to_bytes_with_nul();
+
+        /// The `LocalModule` type is the type of the module created by `module!`,
+        /// `module_pci_driver!`, `module_platform_driver!`, etc.
+        type LocalModule = #type_;
+
+        impl ::kernel::ModuleMetadata for #type_ {
+            const NAME: &'static ::kernel::str::CStr = #name_cstr;
+
+            #[cfg(MODULE)]
+            const THIS_MODULE: ::kernel::ThisModule = {
+                extern "C" {
+                    static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>;
+                }
+
+                // SAFETY: `__this_module` is constructed by the kernel at load time
+                // and lives until the module is unloaded.
+                unsafe { ::kernel::ThisModule::from_ptr(__this_module.get()) }
+            };
+
+            #[cfg(not(MODULE))]
+            const THIS_MODULE: ::kernel::ThisModule = unsafe {
+                ::kernel::ThisModule::from_ptr(::core::ptr::null_mut())
+            };
+        }
+
+        // Double nested modules, since then nobody can access the public items inside.
+        #[doc(hidden)]
+        mod __module_init {
+            mod __module_init {
+                use pin_init::PinInit;
+
+                /// The "Rust loadable module" mark.
+                //
+                // This may be best done another way later on, e.g. as a new modinfo
+                // key or a new section. For the moment, keep it simple.
+                #[cfg(MODULE)]
+                #[used(compiler)]
+                static __IS_RUST_MODULE: () = ();
+
+                static mut __MOD: ::core::mem::MaybeUninit<super::super::LocalModule> =
+                    ::core::mem::MaybeUninit::uninit();
+
+                // Loadable modules need to export the `{init,cleanup}_module` identifiers.
+                /// # Safety
+                ///
+                /// This function must not be called after module initialization, because it may be
+                /// freed after that completes.
+                #[cfg(MODULE)]
+                #[no_mangle]
+                #[link_section = ".init.text"]
+                pub unsafe extern "C" fn init_module() -> ::kernel::ffi::c_int {
+                    // SAFETY: This function is inaccessible to the outside due to the double
+                    // module wrapping it. It is called exactly once by the C side via its
+                    // unique name.
+                    unsafe { __init() }
+                }
+
+                #[cfg(MODULE)]
+                #[used(compiler)]
+                #[link_section = ".init.data"]
+                static __UNIQUE_ID___addressable_init_module: unsafe extern "C" fn() -> i32 =
+                    init_module;
+
+                #[cfg(MODULE)]
+                #[no_mangle]
+                #[link_section = ".exit.text"]
+                pub extern "C" fn cleanup_module() {
+                    // SAFETY:
+                    // - This function is inaccessible to the outside due to the double
+                    //   module wrapping it. It is called exactly once by the C side via its
+                    //   unique name,
+                    // - furthermore it is only called after `init_module` has returned `0`
+                    //   (which delegates to `__init`).
+                    unsafe { __exit() }
+                }
+
+                #[cfg(MODULE)]
+                #[used(compiler)]
+                #[link_section = ".exit.data"]
+                static __UNIQUE_ID___addressable_cleanup_module: extern "C" fn() = cleanup_module;
+
+                // Built-in modules are initialized through an initcall pointer
+                // and the identifiers need to be unique.
+                #[cfg(not(MODULE))]
+                #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))]
+                #[link_section = #initcall_section]
+                #[used(compiler)]
+                pub static #ident_initcall: extern "C" fn() ->
+                    ::kernel::ffi::c_int = #ident_init;
+
+                #[cfg(not(MODULE))]
+                #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)]
+                ::core::arch::global_asm!(#global_asm);
+
+                #[cfg(not(MODULE))]
+                #[no_mangle]
+                pub extern "C" fn #ident_init() -> ::kernel::ffi::c_int {
+                    // SAFETY: This function is inaccessible to the outside due to the double
+                    // module wrapping it. It is called exactly once by the C side via its
+                    // placement above in the initcall section.
+                    unsafe { __init() }
+                }
+
+                #[cfg(not(MODULE))]
+                #[no_mangle]
+                pub extern "C" fn #ident_exit() {
+                    // SAFETY:
+                    // - This function is inaccessible to the outside due to the double
+                    //   module wrapping it. It is called exactly once by the C side via its
+                    //   unique name,
+                    // - furthermore it is only called after `#ident_init` has
+                    //   returned `0` (which delegates to `__init`).
+                    unsafe { __exit() }
+                }
+
+                /// # Safety
+                ///
+                /// This function must only be called once.
+                unsafe fn __init() -> ::kernel::ffi::c_int {
+                    let initer = <super::super::LocalModule as ::kernel::InPlaceModule>::init(
+                        ::kernel::module::this_module::<super::super::LocalModule>()
+                    );
+                    // SAFETY: No data race, since `__MOD` can only be accessed by this module
+                    // and there only `__init` and `__exit` access it. These functions are only
+                    // called once and `__exit` cannot be called before or during `__init`.
+                    match unsafe { ::pin_init::raw_try_init(__MOD.as_mut_ptr(), initer) } {
+                        Ok(m) => 0,
+                        Err(e) => e.to_errno(),
+                    }
+                }
+
+                /// # Safety
+                ///
+                /// This function must
+                /// - only be called once,
+                /// - be called after `__init` has been called and returned `0`.
+                unsafe fn __exit() {
+                    // SAFETY: No data race, since `__MOD` can only be accessed by this module
+                    // and there only `__init` and `__exit` access it. These functions are only
+                    // called once and `__init` was already called.
+                    unsafe {
+                        // Invokes `drop()` on `__MOD`, which should be used for cleanup.
+                        __MOD.assume_init_drop();
+                    }
+                }
+
+                #modinfo_ts
+            }
+        }
+
+        mod module_parameters {
+            #params_ts
+        }
+    })
+}
diff --git a/rust/macros/paste.rs b/rust/macros/paste.rs
new file mode 100644
index 000000000..2181e312a
--- /dev/null
+++ b/rust/macros/paste.rs
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use proc_macro2::{Delimiter, Group, Ident, Spacing, Span, TokenTree};
+
+fn concat_helper(tokens: &[TokenTree]) -> Vec<(String, Span)> {
+    let mut tokens = tokens.iter();
+    let mut segments = Vec::new();
+    let mut span = None;
+    loop {
+        match tokens.next() {
+            None => break,
+            Some(TokenTree::Literal(lit)) => {
+                // Allow us to concat string literals by stripping quotes
+                let mut value = lit.to_string();
+                if value.starts_with('"') && value.ends_with('"') {
+                    value.remove(0);
+                    value.pop();
+                }
+                segments.push((value, lit.span()));
+            }
+            Some(TokenTree::Ident(ident)) => {
+                let mut value = ident.to_string();
+                if value.starts_with("r#") {
+                    value.replace_range(0..2, "");
+                }
+                segments.push((value, ident.span()));
+            }
+            Some(TokenTree::Punct(p)) if p.as_char() == ':' => {
+                let Some(TokenTree::Ident(ident)) = tokens.next() else {
+                    panic!("expected identifier as modifier");
+                };
+
+                let (mut value, sp) = segments.pop().expect("expected identifier before modifier");
+                match ident.to_string().as_str() {
+                    // Set the overall span of concatenated token as current span
+                    "span" => {
+                        assert!(
+                            span.is_none(),
+                            "span modifier should only appear at most once"
+                        );
+                        span = Some(sp);
+                    }
+                    "lower" => value = value.to_lowercase(),
+                    "upper" => value = value.to_uppercase(),
+                    v => panic!("unknown modifier `{v}`"),
+                };
+                segments.push((value, sp));
+            }
+            Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::None => {
+                let tokens = group.stream().into_iter().collect::<Vec<TokenTree>>();
+                segments.append(&mut concat_helper(tokens.as_slice()));
+            }
+            token => panic!("unexpected token in paste segments: {token:?}"),
+        };
+    }
+
+    segments
+}
+
+fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree {
+    let segments = concat_helper(tokens);
+    let pasted: String = segments.into_iter().map(|x| x.0).collect();
+    TokenTree::Ident(Ident::new(&pasted, group_span))
+}
+
+pub(crate) fn expand(tokens: &mut Vec<TokenTree>) {
+    for token in tokens.iter_mut() {
+        if let TokenTree::Group(group) = token {
+            let delimiter = group.delimiter();
+            let span = group.span();
+            let mut stream: Vec<_> = group.stream().into_iter().collect();
+            // Find groups that looks like `[< A B C D >]`
+            if delimiter == Delimiter::Bracket
+                && stream.len() >= 3
+                && matches!(&stream[0], TokenTree::Punct(p) if p.as_char() == '<')
+                && matches!(&stream[stream.len() - 1], TokenTree::Punct(p) if p.as_char() == '>')
+            {
+                // Replace the group with concatenated token
+                *token = concat(&stream[1..stream.len() - 1], span);
+            } else {
+                // Recursively expand tokens inside the group
+                expand(&mut stream);
+                let mut group = Group::new(delimiter, stream.into_iter().collect());
+                group.set_span(span);
+                *token = TokenTree::Group(group);
+            }
+        }
+    }
+
+    // Path segments cannot contain invisible delimiter group, so remove them if any.
+    for i in (0..tokens.len().saturating_sub(3)).rev() {
+        // Looking for a double colon
+        if matches!(
+            (&tokens[i + 1], &tokens[i + 2]),
+            (TokenTree::Punct(a), TokenTree::Punct(b))
+                if a.as_char() == ':' && a.spacing() == Spacing::Joint && b.as_char() == ':'
+        ) {
+            match &tokens[i + 3] {
+                TokenTree::Group(group) if group.delimiter() == Delimiter::None => {
+                    tokens.splice(i + 3..i + 4, group.stream());
+                }
+                _ => (),
+            }
+
+            match &tokens[i] {
+                TokenTree::Group(group) if group.delimiter() == Delimiter::None => {
+                    tokens.splice(i..i + 1, group.stream());
+                }
+                _ => (),
+            }
+        }
+    }
+}
diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs
new file mode 100644
index 000000000..be9a5ed8a
--- /dev/null
+++ b/rust/macros/vtable.rs
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use std::{
+    collections::HashSet,
+    iter::Extend, //
+};
+
+use proc_macro2::{
+    Ident,
+    TokenStream, //
+};
+use quote::ToTokens;
+use syn::{
+    parse_quote,
+    Error,
+    ImplItem,
+    Item,
+    ItemImpl,
+    ItemTrait,
+    Result,
+    TraitItem, //
+};
+
+fn handle_trait(mut item: ItemTrait) -> Result<ItemTrait> {
+    let mut gen_items = Vec::new();
+
+    gen_items.push(parse_quote! {
+         /// A marker to prevent implementors from forgetting to use [`#[vtable]`](vtable)
+         /// attribute when implementing this trait.
+         const USE_VTABLE_ATTR: ();
+    });
+
+    // Add `type OwnerModule: ModuleMetadata` as a required associated type if
+    // the trait does not already define it.
+    if !item
+        .items
+        .iter()
+        .any(|i| matches!(i, TraitItem::Type(t) if t.ident == "OwnerModule"))
+    {
+        gen_items.push(parse_quote! {
+            /// The module implementing this vtable trait.
+            ///
+            /// Automatically set to `crate::LocalModule` by the `#[vtable]`
+            /// impl macro.
+            type OwnerModule: ::kernel::ModuleMetadata;
+        });
+    }
+
+    for item in &item.items {
+        if let TraitItem::Fn(fn_item) = item {
+            let name = &fn_item.sig.ident;
+            let gen_const_name = Ident::new(
+                &format!("HAS_{}", name.to_string().to_uppercase()),
+                name.span(),
+            );
+
+            // We don't know on the implementation-site whether a method is required or provided
+            // so we have to generate a const for all methods.
+            let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs);
+            let comment =
+                format!("Indicates if the `{name}` method is overridden by the implementor.");
+            gen_items.push(parse_quote! {
+                #(#cfg_attrs)*
+                #[doc = #comment]
+                const #gen_const_name: bool = false;
+            });
+        }
+    }
+
+    item.items.extend(gen_items);
+    Ok(item)
+}
+
+fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> {
+    let mut gen_items = Vec::new();
+    let mut defined_items = HashSet::new();
+
+    // Iterate over all user-defined items to gather any possible explicit overrides.
+    for item in &item.items {
+        match item {
+            ImplItem::Const(const_item) => {
+                defined_items.insert(const_item.ident.clone());
+            }
+            ImplItem::Type(type_item) => {
+                defined_items.insert(type_item.ident.clone());
+            }
+            _ => {}
+        }
+    }
+
+    gen_items.push(parse_quote! {
+        const USE_VTABLE_ATTR: () = ();
+    });
+
+    // Auto-insert `type OwnerModule = crate::LocalModule` if not explicitly defined.
+    // `crate::LocalModule` resolves to the real module type (via `module!`) or a
+    // dummy fallback in non-module contexts (e.g., doctests).
+    if !defined_items.contains(&parse_quote!(OwnerModule)) {
+        gen_items.push(parse_quote! {
+            type OwnerModule = crate::LocalModule;
+        });
+    }
+
+    for item in &item.items {
+        if let ImplItem::Fn(fn_item) = item {
+            let name = &fn_item.sig.ident;
+            let gen_const_name = Ident::new(
+                &format!("HAS_{}", name.to_string().to_uppercase()),
+                name.span(),
+            );
+            // Skip if it's declared already -- this allows user override.
+            if defined_items.contains(&gen_const_name) {
+                continue;
+            }
+            let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs);
+            gen_items.push(parse_quote! {
+                #(#cfg_attrs)*
+                const #gen_const_name: bool = true;
+            });
+        }
+    }
+
+    item.items.extend(gen_items);
+    Ok(item)
+}
+
+pub(crate) fn vtable(input: Item) -> Result<TokenStream> {
+    match input {
+        Item::Trait(item) => Ok(handle_trait(item)?.into_token_stream()),
+        Item::Impl(item) => Ok(handle_impl(item)?.into_token_stream()),
+        _ => Err(Error::new_spanned(
+            input,
+            "`#[vtable]` attribute should only be applied to trait or impl block",
+        ))?,
+    }
+}