Merge tag 'io_uring-7.3-20261002' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linuxgrafted

Pull io_uring fixes from Jens Axboe:

 - Fix a task_work add use-after-free with SQPOLL.

   The sqpoll thread could pop and complete the last request while
   io_req_normal_work_add() was still looking at them after the mpscq
   push.

   Use the same approach as DEFER_TASKRUN to protect from that, holding
   an RCU read lock across the add, and have exit wait for an RCU grace
   period for SQPOLL rings as well.

 - CQE32 ring fixes: correct the free entry check for 32b CQEs, zero the
   big_cqe for aux CQEs, and only post the dummy skip CQE on CQE_MIXED
   rings

 - Mark the source filter table as COW when cloning bpf filters, so
   registering another filter on the source doesn't modify the shared
   table in place

 - Initialize the task context before running the BPF loop

 - Requeue zcrx multishot receives stopped by a local resource

 - End a TX_TIMESTAMP multishot cmd when the CQ is full (lollipopkit)

* tag 'io_uring-7.3-20261002' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux:
  io_uring: fix task_work add use-after-free with SQPOLL
  io_uring/cmd_net: end TX_TIMESTAMP multishot when the CQ is full
  io_uring/zcrx: requeue multishot receives stopped by a local resource
  io_uring: initialize task context before running the BPF loop
  io_uring: zero big_cqe for aux CQEs on CQE32 rings
  io_uring: fix free entry check for 32b CQEs on CQE32 rings
  io_uring: only post the dummy skip CQE on CQE_MIXED rings
  io_uring/bpf_filter: mark source as COW when cloning filters

6 files changed, 1261 insertions, 0 deletions

diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs
new file mode 100644
index 000000000..c7d9b3e62
--- /dev/null
+++ b/rust/pin-init/internal/src/diagnostics.rs
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use std::fmt::Display;
+
+use proc_macro2::TokenStream;
+use quote::quote_spanned;
+use syn::{spanned::Spanned, Error};
+
+pub(crate) struct DiagCtxt(TokenStream);
+pub(crate) struct ErrorGuaranteed(());
+
+impl DiagCtxt {
+    pub(crate) fn error(&mut self, span: impl Spanned, msg: impl Display) -> ErrorGuaranteed {
+        let error = Error::new(span.span(), msg);
+        self.0.extend(error.into_compile_error());
+        ErrorGuaranteed(())
+    }
+
+    pub(crate) fn warn(&mut self, span: impl Spanned, msg: impl Display) {
+        // Have the message start on a new line for visual clarity.
+        let msg = format!("\n{}", msg);
+        self.0.extend(quote_spanned!(span.span() =>
+            // Approximate using deprecated warning while `proc_macro_diagnostic` is unstable.
+            const _: () = {
+                #[deprecated = #msg]
+                const fn warn() {}
+                warn();
+            };
+        ));
+    }
+
+    pub(crate) fn with(
+        fun: impl FnOnce(&mut DiagCtxt) -> Result<TokenStream, ErrorGuaranteed>,
+    ) -> TokenStream {
+        let mut dcx = Self(TokenStream::new());
+        match fun(&mut dcx) {
+            Ok(mut stream) => {
+                stream.extend(dcx.0);
+                stream
+            }
+            Err(ErrorGuaranteed(())) => dcx.0,
+        }
+    }
+}
diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs
new file mode 100644
index 000000000..fd0b5ea4a
--- /dev/null
+++ b/rust/pin-init/internal/src/init.rs
@@ -0,0 +1,482 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use proc_macro2::{Span, TokenStream};
+use quote::{format_ident, quote};
+use syn::{
+    braced,
+    parse::{End, Parse},
+    parse_quote,
+    punctuated::Punctuated,
+    spanned::Spanned,
+    token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, Path, Token, Type,
+};
+
+use crate::diagnostics::{DiagCtxt, ErrorGuaranteed};
+
+pub(crate) struct Initializer {
+    attrs: Vec<InitializerAttribute>,
+    this: Option<This>,
+    path: Path,
+    brace_token: token::Brace,
+    fields: Punctuated<InitializerField, Token![,]>,
+    rest: Option<(Token![..], Expr)>,
+    error: Option<(Token![?], Type)>,
+}
+
+struct This {
+    _and_token: Token![&],
+    ident: Ident,
+    _in_token: Token![in],
+}
+
+struct InitializerField {
+    attrs: Vec<Attribute>,
+    kind: InitializerKind,
+}
+
+enum InitializerKind {
+    Value {
+        ident: Ident,
+        value: Option<(Token![:], Expr)>,
+    },
+    Init {
+        ident: Ident,
+        _left_arrow_token: Token![<-],
+        value: Expr,
+    },
+    Code {
+        _underscore_token: Token![_],
+        _colon_token: Token![:],
+        block: Block,
+    },
+}
+
+impl InitializerKind {
+    fn ident(&self) -> Option<&Ident> {
+        match self {
+            Self::Value { ident, .. } | Self::Init { ident, .. } => Some(ident),
+            Self::Code { .. } => None,
+        }
+    }
+}
+
+enum InitializerAttribute {
+    DefaultError(DefaultErrorAttribute),
+}
+
+struct DefaultErrorAttribute {
+    ty: Box<Type>,
+}
+
+pub(crate) fn expand(
+    Initializer {
+        attrs,
+        this,
+        path,
+        brace_token,
+        fields,
+        rest,
+        error,
+    }: Initializer,
+    default_error: Option<&'static str>,
+    pinned: bool,
+    dcx: &mut DiagCtxt,
+) -> Result<TokenStream, ErrorGuaranteed> {
+    let error = error.map_or_else(
+        || {
+            if let Some(default_error) = attrs.iter().fold(None, |acc, attr| {
+                #[expect(irrefutable_let_patterns)]
+                if let InitializerAttribute::DefaultError(DefaultErrorAttribute { ty }) = attr {
+                    Some(ty.clone())
+                } else {
+                    acc
+                }
+            }) {
+                default_error
+            } else if let Some(default_error) = default_error {
+                syn::parse_str(default_error).unwrap()
+            } else {
+                dcx.error(brace_token.span.close(), "expected `? <type>` after `}`");
+                parse_quote!(::core::convert::Infallible)
+            }
+        },
+        |(_, err)| Box::new(err),
+    );
+    let slot = format_ident!("slot");
+    let (has_data_trait, get_data, init_from_closure) = if pinned {
+        (
+            format_ident!("HasPinData"),
+            format_ident!("__pin_data"),
+            format_ident!("pin_init_from_closure"),
+        )
+    } else {
+        (
+            format_ident!("HasInitData"),
+            format_ident!("__init_data"),
+            format_ident!("init_from_closure"),
+        )
+    };
+    let init_kind = get_init_kind(rest, dcx);
+    let zeroable_check = match init_kind {
+        InitKind::Normal => quote!(),
+        InitKind::Zeroing => quote! {
+            // The user specified `..Zeroable::zeroed()` at the end of the list of fields.
+            // Therefore we check if the struct implements `Zeroable` and then zero the memory.
+            // This allows us to also remove the check that all fields are present (since we
+            // already set the memory to zero and that is a valid bit pattern).
+            fn assert_zeroable<T: ?::core::marker::Sized>(_: *mut T)
+            where T: ::pin_init::Zeroable
+            {}
+            // Ensure that the struct is indeed `Zeroable`.
+            assert_zeroable(#slot);
+            // SAFETY: The type implements `Zeroable` by the check above.
+            unsafe { ::core::ptr::write_bytes(#slot, 0, 1) };
+        },
+    };
+    let this = match this {
+        None => quote!(),
+        Some(This { ident, .. }) => quote! {
+            // Create the `this` so it can be referenced by the user inside of the
+            // expressions creating the individual fields.
+            let #ident = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };
+        },
+    };
+    // `mixed_site` ensures that the data is not accessible to the user-controlled code.
+    let data = Ident::new("__data", Span::mixed_site());
+    let init_fields = init_fields(&fields, pinned, &data, &slot);
+    let field_check = make_field_check(&fields, init_kind, &path);
+    Ok(quote! {{
+        // Get the data about fields from the supplied type.
+        // SAFETY: TODO
+        let #data = unsafe {
+            use ::pin_init::__internal::#has_data_trait;
+            // Can't use `<#path as #has_data_trait>::#get_data`, since the user is able to omit
+            // generics (which need to be present with that syntax).
+            #path::#get_data()
+        };
+        // Ensure that `#data` really is of type `#data` and help with type inference:
+        let init = #data.__make_closure::<_, #error>(
+            move |slot| {
+                #zeroable_check
+                #this
+                #init_fields
+                #field_check
+                // SAFETY: we are the `init!` macro that is allowed to call this.
+                Ok(unsafe { ::pin_init::__internal::InitOk::new() })
+            }
+        );
+        let init = move |slot| -> ::core::result::Result<(), #error> {
+            init(slot).map(|__InitOk| ())
+        };
+        // SAFETY: TODO
+        unsafe { ::pin_init::#init_from_closure::<_, #error>(init) }
+    }})
+}
+
+enum InitKind {
+    Normal,
+    Zeroing,
+}
+
+fn get_init_kind(rest: Option<(Token![..], Expr)>, dcx: &mut DiagCtxt) -> InitKind {
+    let Some((dotdot, expr)) = rest else {
+        return InitKind::Normal;
+    };
+    match &expr {
+        Expr::Call(ExprCall { func, args, .. }) if args.is_empty() => match &**func {
+            Expr::Path(ExprPath {
+                attrs,
+                qself: None,
+                path:
+                    Path {
+                        leading_colon: None,
+                        segments,
+                    },
+            }) if attrs.is_empty()
+                && segments.len() == 2
+                && segments[0].ident == "Zeroable"
+                && segments[0].arguments.is_none()
+                && segments[1].ident == "init_zeroed"
+                && segments[1].arguments.is_none() =>
+            {
+                return InitKind::Zeroing;
+            }
+            _ => {}
+        },
+        _ => {}
+    }
+    dcx.error(
+        dotdot.span().join(expr.span()).unwrap_or(expr.span()),
+        "expected nothing or `..Zeroable::init_zeroed()`.",
+    );
+    InitKind::Normal
+}
+
+/// Generate the code that initializes the fields of the struct using the initializers in `field`.
+fn init_fields(
+    fields: &Punctuated<InitializerField, Token![,]>,
+    pinned: bool,
+    data: &Ident,
+    slot: &Ident,
+) -> TokenStream {
+    let mut guards = vec![];
+    let mut guard_attrs = vec![];
+    let mut res = TokenStream::new();
+    for InitializerField { attrs, kind } in fields {
+        let cfgs = {
+            let mut cfgs = attrs.clone();
+            cfgs.retain(|attr| attr.path().is_ident("cfg"));
+            cfgs
+        };
+
+        let ident = match kind {
+            InitializerKind::Value { ident, .. } => ident,
+            InitializerKind::Init { ident, .. } => ident,
+            InitializerKind::Code { block, .. } => {
+                let stmt = &block.stmts;
+                res.extend(quote! {
+                    #(#attrs)*
+                    {
+                        #(#stmt)*
+                    }
+                });
+                continue;
+            }
+        };
+
+        let slot = if pinned {
+            quote! {
+                // SAFETY:
+                // - `slot` is valid and properly aligned.
+                // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned.
+                // - `make_field_check` prevents `#ident` from being used twice, therefore
+                //   `(*slot).#ident` is exclusively accessed and has not been initialized.
+                (unsafe { #data.#ident(#slot) })
+            }
+        } else {
+            quote! {
+                // For `init!()` macro, everything is unpinned.
+                // SAFETY:
+                // - `&raw mut (*slot).#ident` is valid.
+                // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned.
+                // - `make_field_check` prevents `#ident` from being used twice, therefore
+                //   `(*slot).#ident` is exclusively accessed and has not been initialized.
+                (unsafe {
+                    ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new(
+                        &raw mut (*#slot).#ident
+                    )
+                })
+            }
+        };
+
+        // `mixed_site` ensures that the guard is not accessible to the user-controlled code.
+        let guard = format_ident!("__{ident}_guard", span = Span::mixed_site());
+
+        let init = match kind {
+            InitializerKind::Value { ident, value } => {
+                let value = value
+                    .as_ref()
+                    .map(|(_, value)| quote!(#value))
+                    .unwrap_or_else(|| quote!(#ident));
+
+                quote! {
+                    #(#attrs)*
+                    let mut #guard = #slot.write(#value);
+
+                }
+            }
+            InitializerKind::Init { value, .. } => {
+                quote! {
+                    #(#attrs)*
+                    let mut #guard = #slot.init(#value)?;
+                }
+            }
+            InitializerKind::Code { .. } => unreachable!(),
+        };
+
+        res.extend(quote! {
+            #init
+
+            #(#cfgs)*
+            // Allow `non_snake_case` since the same warning is going to be reported for the struct
+            // field.
+            #[allow(unused_variables, non_snake_case)]
+            let #ident = #guard.let_binding();
+        });
+
+        guards.push(guard);
+        guard_attrs.push(cfgs);
+    }
+    quote! {
+        #res
+        // If execution reaches this point, all fields have been initialized. Therefore we can now
+        // dismiss the guards by forgetting them.
+        #(
+            #(#guard_attrs)*
+            ::core::mem::forget(#guards);
+        )*
+    }
+}
+
+/// Generate the check for ensuring that every field has been initialized and aligned.
+fn make_field_check(
+    fields: &Punctuated<InitializerField, Token![,]>,
+    init_kind: InitKind,
+    path: &Path,
+) -> TokenStream {
+    let field_attrs: Vec<_> = fields
+        .iter()
+        .filter_map(|f| f.kind.ident().map(|_| &f.attrs))
+        .collect();
+    let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect();
+    let zeroing_trailer = match init_kind {
+        InitKind::Normal => None,
+        InitKind::Zeroing => Some(quote! {
+            ..::core::mem::zeroed()
+        }),
+    };
+    quote! {
+        #[allow(unreachable_code)]
+        // We use unreachable code to perform field checks. They're still checked by the compiler.
+        // SAFETY: this code is never executed.
+        let _ = || unsafe {
+            // Create references to ensure that the initialized field is properly aligned.
+            // Unaligned fields will cause the compiler to emit E0793. We do not support
+            // unaligned fields since `Init::__init` requires an aligned pointer; the call to
+            // `ptr::write` for value-initialization case has the same requirement.
+            #(
+                #(#field_attrs)*
+                let _ = &(*slot).#field_name;
+            )*
+
+            // If the zeroing trailer is not present, this checks that all fields have been
+            // mentioned exactly once. If the zeroing trailer is present, all missing fields will be
+            // zeroed, so this checks that all fields have been mentioned at most once. The use of
+            // struct initializer will still generate very natural error messages for any misuse.
+            ::core::ptr::write(slot, #path {
+                #(
+                    #(#field_attrs)*
+                    #field_name: loop {},
+                )*
+                #zeroing_trailer
+            })
+        };
+    }
+}
+
+impl Parse for Initializer {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        let attrs = input.call(Attribute::parse_outer)?;
+        let this = input.peek(Token![&]).then(|| input.parse()).transpose()?;
+        let path = input.parse()?;
+        let content;
+        let brace_token = braced!(content in input);
+        let mut fields = Punctuated::new();
+        loop {
+            let lh = content.lookahead1();
+            if lh.peek(End) || lh.peek(Token![..]) {
+                break;
+            } else if lh.peek(Ident) || lh.peek(Token![_]) || lh.peek(Token![#]) {
+                fields.push_value(content.parse()?);
+                let lh = content.lookahead1();
+                if lh.peek(End) {
+                    break;
+                } else if lh.peek(Token![,]) {
+                    fields.push_punct(content.parse()?);
+                } else {
+                    return Err(lh.error());
+                }
+            } else {
+                return Err(lh.error());
+            }
+        }
+        let rest = content
+            .peek(Token![..])
+            .then(|| Ok::<_, syn::Error>((content.parse()?, content.parse()?)))
+            .transpose()?;
+        let error = input
+            .peek(Token![?])
+            .then(|| Ok::<_, syn::Error>((input.parse()?, input.parse()?)))
+            .transpose()?;
+        let attrs = attrs
+            .into_iter()
+            .map(|a| {
+                if a.path().is_ident("default_error") {
+                    a.parse_args::<DefaultErrorAttribute>()
+                        .map(InitializerAttribute::DefaultError)
+                } else {
+                    Err(syn::Error::new_spanned(a, "unknown initializer attribute"))
+                }
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        Ok(Self {
+            attrs,
+            this,
+            path,
+            brace_token,
+            fields,
+            rest,
+            error,
+        })
+    }
+}
+
+impl Parse for DefaultErrorAttribute {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        Ok(Self { ty: input.parse()? })
+    }
+}
+
+impl Parse for This {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        Ok(Self {
+            _and_token: input.parse()?,
+            ident: input.parse()?,
+            _in_token: input.parse()?,
+        })
+    }
+}
+
+impl Parse for InitializerField {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        let attrs = input.call(Attribute::parse_outer)?;
+        Ok(Self {
+            attrs,
+            kind: input.parse()?,
+        })
+    }
+}
+
+impl Parse for InitializerKind {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        let lh = input.lookahead1();
+        if lh.peek(Token![_]) {
+            Ok(Self::Code {
+                _underscore_token: input.parse()?,
+                _colon_token: input.parse()?,
+                block: input.parse()?,
+            })
+        } else if lh.peek(Ident) {
+            let ident = input.parse()?;
+            let lh = input.lookahead1();
+            if lh.peek(Token![<-]) {
+                Ok(Self::Init {
+                    ident,
+                    _left_arrow_token: input.parse()?,
+                    value: input.parse()?,
+                })
+            } else if lh.peek(Token![:]) {
+                Ok(Self::Value {
+                    ident,
+                    value: Some((input.parse()?, input.parse()?)),
+                })
+            } else if lh.peek(Token![,]) || lh.peek(End) {
+                Ok(Self::Value { ident, value: None })
+            } else {
+                Err(lh.error())
+            }
+        } else {
+            Err(lh.error())
+        }
+    }
+}
diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs
new file mode 100644
index 000000000..60d5093f3
--- /dev/null
+++ b/rust/pin-init/internal/src/lib.rs
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT`
+// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is
+// touched by Kconfig when the version string from the compiler changes.
+
+//! `pin-init` proc macros.
+
+// Documentation is done in the pin-init crate instead.
+#![allow(missing_docs)]
+
+use proc_macro::TokenStream;
+use syn::parse_macro_input;
+
+use crate::diagnostics::DiagCtxt;
+
+mod diagnostics;
+mod init;
+mod pin_data;
+mod pinned_drop;
+mod zeroable;
+
+#[proc_macro_attribute]
+pub fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream {
+    let args = parse_macro_input!(args);
+    let input = parse_macro_input!(input);
+    DiagCtxt::with(|dcx| pin_data::pin_data(args, input, dcx)).into()
+}
+
+#[proc_macro_attribute]
+pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream {
+    let args = parse_macro_input!(args);
+    let input = parse_macro_input!(input);
+    DiagCtxt::with(|dcx| pinned_drop::pinned_drop(args, input, dcx)).into()
+}
+
+#[proc_macro_derive(Zeroable)]
+pub fn derive_zeroable(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input);
+    DiagCtxt::with(|dcx| zeroable::derive(input, dcx)).into()
+}
+
+#[proc_macro_derive(MaybeZeroable)]
+pub fn maybe_derive_zeroable(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input);
+    DiagCtxt::with(|dcx| zeroable::maybe_derive(input, dcx)).into()
+}
+#[proc_macro]
+pub fn init(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input);
+    DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), false, dcx))
+        .into()
+}
+
+#[proc_macro]
+pub fn pin_init(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input);
+    DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), true, dcx)).into()
+}
diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs
new file mode 100644
index 000000000..ff194d275
--- /dev/null
+++ b/rust/pin-init/internal/src/pin_data.rs
@@ -0,0 +1,537 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use proc_macro2::TokenStream;
+use quote::{format_ident, quote, ToTokens};
+use syn::{
+    parse::{End, Nothing, Parse},
+    parse_quote, parse_quote_spanned,
+    spanned::Spanned,
+    visit_mut::VisitMut,
+    Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause,
+};
+
+use crate::diagnostics::{DiagCtxt, ErrorGuaranteed};
+
+pub(crate) mod kw {
+    syn::custom_keyword!(PinnedDrop);
+}
+
+pub(crate) enum Args {
+    Nothing(Nothing),
+    #[allow(dead_code)]
+    PinnedDrop(kw::PinnedDrop),
+}
+
+impl Parse for Args {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        let lh = input.lookahead1();
+        if lh.peek(End) {
+            input.parse().map(Self::Nothing)
+        } else if lh.peek(kw::PinnedDrop) {
+            input.parse().map(Self::PinnedDrop)
+        } else {
+            Err(lh.error())
+        }
+    }
+}
+
+impl ToTokens for Args {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        match self {
+            Self::Nothing(_) => (),
+            Self::PinnedDrop(kw) => kw.to_tokens(tokens),
+        }
+    }
+}
+
+struct FieldInfo<'a> {
+    field: &'a Field,
+    pinned: bool,
+}
+
+pub(crate) fn pin_data(
+    args: Args,
+    input: Item,
+    dcx: &mut DiagCtxt,
+) -> Result<TokenStream, ErrorGuaranteed> {
+    let mut struct_ = match input {
+        Item::Struct(struct_) => struct_,
+        Item::Enum(enum_) => {
+            return Err(dcx.error(
+                enum_.enum_token,
+                "`#[pin_data]` only supports structs for now",
+            ));
+        }
+        Item::Union(union) => {
+            return Err(dcx.error(
+                union.union_token,
+                "`#[pin_data]` only supports structs for now",
+            ));
+        }
+        rest => {
+            return Err(dcx.error(
+                rest,
+                "`#[pin_data]` can only be applied to struct, enum and union definitions",
+            ));
+        }
+    };
+
+    // Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all
+    // field cfgs first before continuing.
+    //
+    // We need to perform this after parsing so we can reliably detect field cfgs.
+    for (field_idx, field) in struct_.fields.iter_mut().enumerate() {
+        let cfg: Vec<_> = field
+            .attrs
+            .iter()
+            .filter(|a| a.path().is_ident("cfg"))
+            .map(|a| {
+                a.parse_args::<TokenStream>()
+                    .expect("parse as token stream cannot fail")
+            })
+            .collect();
+
+        if cfg.is_empty() {
+            continue;
+        }
+
+        field.attrs.retain(|a| !a.path().is_ident("cfg"));
+        let cfg_true_struct = quote!(#struct_);
+
+        let punctuated = match &mut struct_.fields {
+            Fields::Named(fields) => &mut fields.named,
+            Fields::Unnamed(fields) => &mut fields.unnamed,
+            Fields::Unit => unreachable!(),
+        };
+        *punctuated = std::mem::take(punctuated)
+            .into_pairs()
+            .enumerate()
+            .filter(|&(i, _)| i != field_idx)
+            .map(|(_, p)| p)
+            .collect();
+        let cfg_false_struct = quote!(#struct_);
+
+        // Resolve one field at a time until we've got no more field cfgs.
+        //
+        // This is linear time because macro invocations with false cfg will not be expanded.
+        return Ok(quote!(
+            #[cfg(all(#(#cfg,)*))]
+            #[::pin_init::pin_data(#args)]
+            #cfg_true_struct
+
+            #[cfg(not(all(#(#cfg,)*)))]
+            #[::pin_init::pin_data(#args)]
+            #cfg_false_struct
+        ));
+    }
+
+    // The generics might contain the `Self` type. Since this macro will define a new type with the
+    // same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed
+    // to this struct definition. Therefore we have to replace `Self` with the concrete name.
+    let mut replacer = {
+        let name = &struct_.ident;
+        let (_, ty_generics, _) = struct_.generics.split_for_impl();
+        SelfReplacer(parse_quote!(#name #ty_generics))
+    };
+    replacer.visit_generics_mut(&mut struct_.generics);
+    replacer.visit_fields_mut(&mut struct_.fields);
+
+    let fields: Vec<FieldInfo<'_>> = struct_
+        .fields
+        .iter_mut()
+        .map(|field| {
+            let len = field.attrs.len();
+            field.attrs.retain(|a| !a.path().is_ident("pin"));
+            let pinned_count = len - field.attrs.len();
+            if pinned_count > 1 {
+                dcx.error(&field, "#[pin] attribute specified more than once");
+            }
+
+            assert!(
+                !field.attrs.iter().any(|a| a.path().is_ident("cfg")),
+                "cfgs should be all resolved at this point"
+            );
+
+            FieldInfo {
+                field: &*field,
+                pinned: pinned_count != 0,
+            }
+        })
+        .collect();
+
+    for field in &fields {
+        let ident = field.field.ident.as_ref().unwrap();
+
+        if !field.pinned && is_phantom_pinned(&field.field.ty) {
+            dcx.warn(
+                field.field,
+                format!(
+                    "The field `{ident}` of type `PhantomPinned` only has an effect \
+                    if it has the `#[pin]` attribute",
+                ),
+            );
+        }
+    }
+
+    let unpin_impl = generate_unpin_impl(&struct_.ident, &struct_.generics, &fields);
+    let drop_impl = generate_drop_impl(&struct_.ident, &struct_.generics, args);
+    let projections =
+        generate_projections(&struct_.vis, &struct_.ident, &struct_.generics, &fields);
+    let the_pin_data =
+        generate_the_pin_data(&struct_.vis, &struct_.ident, &struct_.generics, &fields);
+
+    Ok(quote! {
+        #struct_
+        #projections
+        // We put the rest into this const item, because it then will not be accessible to anything
+        // outside.
+        const _: () = {
+            #the_pin_data
+            #unpin_impl
+            #drop_impl
+        };
+    })
+}
+
+fn is_phantom_pinned(ty: &Type) -> bool {
+    match ty {
+        Type::Path(TypePath { qself: None, path }) => {
+            // Cannot possibly refer to `PhantomPinned` (except alias, but that's on the user).
+            if path.segments.len() > 3 {
+                return false;
+            }
+            // If there is a `::`, then the path needs to be `::core::marker::PhantomPinned` or
+            // `::std::marker::PhantomPinned`.
+            if path.leading_colon.is_some() && path.segments.len() != 3 {
+                return false;
+            }
+            let expected: Vec<&[&str]> = vec![&["PhantomPinned"], &["marker"], &["core", "std"]];
+            for (actual, expected) in path.segments.iter().rev().zip(expected) {
+                if !actual.arguments.is_empty() || expected.iter().all(|e| actual.ident != e) {
+                    return false;
+                }
+            }
+            true
+        }
+        _ => false,
+    }
+}
+
+fn generate_unpin_impl(
+    ident: &Ident,
+    generics: &Generics,
+    fields: &[FieldInfo<'_>],
+) -> TokenStream {
+    let (_, ty_generics, _) = generics.split_for_impl();
+    let mut generics_with_pin_lt = generics.clone();
+    generics_with_pin_lt.params.insert(0, parse_quote!('__pin));
+    generics_with_pin_lt.make_where_clause();
+    let (
+        impl_generics_with_pin_lt,
+        ty_generics_with_pin_lt,
+        Some(WhereClause {
+            where_token,
+            predicates,
+        }),
+    ) = generics_with_pin_lt.split_for_impl()
+    else {
+        unreachable!()
+    };
+    let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| {
+        let ident = f.field.ident.as_ref().unwrap();
+        let ty = &f.field.ty;
+        quote!(
+            #ident: #ty
+        )
+    });
+    quote! {
+        // This struct will be used for the unpin analysis. It is needed, because only structurally
+        // pinned fields are relevant whether the struct should implement `Unpin`.
+        #[allow(
+            dead_code, // The fields below are never used.
+            non_snake_case // The warning will be emitted on the struct definition.
+        )]
+        struct __Unpin #generics_with_pin_lt
+        #where_token
+            #predicates
+        {
+            __phantom_pin: ::pin_init::__internal::PhantomInvariantLifetime<'__pin>,
+            __phantom: ::pin_init::__internal::PhantomInvariant<#ident #ty_generics>,
+            #(#pinned_fields),*
+        }
+
+        #[doc(hidden)]
+        impl #impl_generics_with_pin_lt ::core::marker::Unpin for #ident #ty_generics
+        #where_token
+            __Unpin #ty_generics_with_pin_lt: ::core::marker::Unpin,
+            #predicates
+        {}
+    }
+}
+
+fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenStream {
+    let (impl_generics, ty_generics, whr) = generics.split_for_impl();
+    let has_pinned_drop = matches!(args, Args::PinnedDrop(_));
+    // We need to disallow normal `Drop` implementation, the exact behavior depends on whether
+    // `PinnedDrop` was specified in `args`.
+    if has_pinned_drop {
+        // When `PinnedDrop` was specified we just implement `Drop` and delegate.
+        quote! {
+            impl #impl_generics ::core::ops::Drop for #ident #ty_generics
+                #whr
+            {
+                fn drop(&mut self) {
+                    // SAFETY: Since this is a destructor, `self` will not move after this function
+                    // terminates, since it is inaccessible.
+                    let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
+                    // SAFETY: Since this is a drop function, we can create this token to call the
+                    // pinned destructor of this type.
+                    let token = unsafe { ::pin_init::__internal::OnlyCallFromDrop::new() };
+                    ::pin_init::PinnedDrop::drop(pinned, token);
+                }
+            }
+        }
+    } else {
+        // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
+        quote! {
+            // We prevent this by creating a trait that will be implemented for all types implementing
+            // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
+            // if it also implements `Drop`
+            trait MustNotImplDrop {}
+            impl<T: ::core::ops::Drop + ?::core::marker::Sized> MustNotImplDrop for T {}
+            impl #impl_generics MustNotImplDrop for #ident #ty_generics
+                #whr
+            {}
+            // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
+            // They might implement `PinnedDrop` correctly for the struct, but forget to give
+            // `PinnedDrop` as the parameter to `#[pin_data]`.
+            trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
+            impl<T: ::pin_init::PinnedDrop + ?::core::marker::Sized>
+                UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
+            impl #impl_generics
+                UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for #ident #ty_generics
+                #whr
+            {}
+        }
+    }
+}
+
+fn generate_projections(
+    vis: &Visibility,
+    ident: &Ident,
+    generics: &Generics,
+    fields: &[FieldInfo<'_>],
+) -> TokenStream {
+    let (impl_generics, ty_generics, _) = generics.split_for_impl();
+    let mut generics_with_pin_lt = generics.clone();
+    generics_with_pin_lt.params.insert(0, parse_quote!('__pin));
+    let (_, ty_generics_with_pin_lt, whr) = generics_with_pin_lt.split_for_impl();
+    let projection = format_ident!("{ident}Projection");
+    let this = format_ident!("this");
+
+    let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields
+        .iter()
+        .map(|field| {
+            let Field { vis, ident, ty, .. } = &field.field;
+
+            let ident = ident
+                .as_ref()
+                .expect("only structs with named fields are supported");
+            if field.pinned {
+                (
+                    quote!(
+                        #vis #ident: ::core::pin::Pin<&'__pin mut #ty>,
+                    ),
+                    quote!(
+                        // SAFETY: this field is structurally pinned.
+                        #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) },
+                    ),
+                )
+            } else {
+                (
+                    quote!(
+                        #vis #ident: &'__pin mut #ty,
+                    ),
+                    quote!(
+                        #ident: &mut #this.#ident,
+                    ),
+                )
+            }
+        })
+        .collect();
+    let structurally_pinned_fields_docs = fields
+        .iter()
+        .filter(|f| f.pinned)
+        .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap()));
+    let not_structurally_pinned_fields_docs = fields
+        .iter()
+        .filter(|f| !f.pinned)
+        .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap()));
+    let docs = format!(" Pin-projections of [`{ident}`]");
+    quote! {
+        #[doc = #docs]
+        // Allow `non_snake_case` since the same warning will be emitted on
+        // the struct definition.
+        #[allow(dead_code, non_snake_case)]
+        #[doc(hidden)]
+        #vis struct #projection #generics_with_pin_lt
+            #whr
+        {
+            #(#fields_decl)*
+            ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>,
+        }
+
+        impl #impl_generics #ident #ty_generics
+            #whr
+        {
+            /// Pin-projects all fields of `Self`.
+            ///
+            /// These fields are structurally pinned:
+            #(#[doc = #structurally_pinned_fields_docs])*
+            ///
+            /// These fields are **not** structurally pinned:
+            #(#[doc = #not_structurally_pinned_fields_docs])*
+            #[inline]
+            #vis fn project<'__pin>(
+                self: ::core::pin::Pin<&'__pin mut Self>,
+            ) -> #projection #ty_generics_with_pin_lt {
+                // SAFETY: we only give access to `&mut` for fields not structurally pinned.
+                let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) };
+                #projection {
+                    #(#fields_proj)*
+                    ___pin_phantom_data: ::core::marker::PhantomData,
+                }
+            }
+        }
+    }
+}
+
+fn generate_the_pin_data(
+    vis: &Visibility,
+    struct_name: &Ident,
+    generics: &Generics,
+    fields: &[FieldInfo<'_>],
+) -> TokenStream {
+    let (impl_generics, ty_generics, whr) = generics.split_for_impl();
+
+    // For every field, we create an initializing projection function according to its projection
+    // type. If a field is structurally pinned, we create a `Slot` with `Pinned` which must be
+    // initialized via `PinInit`; if it is not structurally pinned, then we create a `Slot` with
+    // `Unpinned` which allows initialization via `Init`.
+    let field_accessors = fields
+        .iter()
+        .map(|f| {
+            let Field { vis, ident, ty, .. } = f.field;
+
+            let field_name = ident
+                .as_ref()
+                .expect("only structs with named fields are supported");
+            let pin_marker = if f.pinned {
+                quote!(Pinned)
+            } else {
+                quote!(Unpinned)
+            };
+            quote! {
+                /// # Safety
+                ///
+                /// - `slot` is valid and properly aligned.
+                /// - `(*slot).#field_name` is properly aligned.
+                /// - `(*slot).#field_name` points to uninitialized and exclusively accessed
+                ///   memory.
+                // Allow `non_snake_case` since the same warning will be emitted on
+                // the struct definition.
+                #[allow(non_snake_case)]
+                #[inline(always)]
+                #vis unsafe fn #field_name(
+                    self,
+                    slot: *mut #struct_name #ty_generics,
+                ) -> ::pin_init::__internal::Slot<::pin_init::__internal::#pin_marker, #ty> {
+                    // SAFETY:
+                    // - If `#pin_marker` is `Pinned`, the corresponding field is structurally
+                    //   pinned.
+                    // - Other safety requirements follows the safety requirement.
+                    unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#field_name) }
+                }
+            }
+        })
+        .collect::<TokenStream>();
+    quote! {
+        // We declare this struct which will host all of the projection function for our type. It
+        // will be invariant over all generic parameters which are inherited from the struct.
+        #[doc(hidden)]
+        #vis struct __ThePinData #generics
+            #whr
+        {
+            __phantom: ::pin_init::__internal::PhantomInvariant<#struct_name #ty_generics>,
+        }
+
+        impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics
+            #whr
+        {
+            #[inline]
+            fn clone(&self) -> Self { *self }
+        }
+
+        impl #impl_generics ::core::marker::Copy for __ThePinData #ty_generics
+            #whr
+        {}
+
+        #[allow(dead_code)] // Some functions might never be used and private.
+        impl #impl_generics __ThePinData #ty_generics
+            #whr
+        {
+            /// Type inference helper function.
+            #[inline(always)]
+            #vis fn __make_closure<__F, __E>(self, f: __F) -> __F
+            where
+                __F: FnOnce(*mut #struct_name #ty_generics) ->
+                    ::core::result::Result<::pin_init::__internal::InitOk, __E>,
+            {
+                f
+            }
+
+            #field_accessors
+        }
+
+        // SAFETY: We have added the correct projection functions above to `__ThePinData` and
+        // we also use the least restrictive generics possible.
+        unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name #ty_generics
+            #whr
+        {
+            type PinData = __ThePinData #ty_generics;
+
+            #[inline]
+            unsafe fn __pin_data() -> Self::PinData {
+                __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() }
+            }
+        }
+    }
+}
+
+struct SelfReplacer(PathSegment);
+
+impl VisitMut for SelfReplacer {
+    fn visit_path_mut(&mut self, i: &mut syn::Path) {
+        if i.is_ident("Self") {
+            let span = i.span();
+            let seg = &self.0;
+            *i = parse_quote_spanned!(span=> #seg);
+        } else {
+            syn::visit_mut::visit_path_mut(self, i);
+        }
+    }
+
+    fn visit_path_segment_mut(&mut self, seg: &mut PathSegment) {
+        if seg.ident == "Self" {
+            let span = seg.span();
+            let this = &self.0;
+            *seg = parse_quote_spanned!(span=> #this);
+        } else {
+            syn::visit_mut::visit_path_segment_mut(self, seg);
+        }
+    }
+
+    fn visit_item_mut(&mut self, _: &mut Item) {
+        // Do not descend into items, since items reset/change what `Self` refers to.
+    }
+}
diff --git a/rust/pin-init/internal/src/pinned_drop.rs b/rust/pin-init/internal/src/pinned_drop.rs
new file mode 100644
index 000000000..a20ac314c
--- /dev/null
+++ b/rust/pin-init/internal/src/pinned_drop.rs
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::{parse::Nothing, parse_quote, spanned::Spanned, ImplItem, ItemImpl, Token};
+
+use crate::diagnostics::{DiagCtxt, ErrorGuaranteed};
+
+pub(crate) fn pinned_drop(
+    _args: Nothing,
+    mut input: ItemImpl,
+    dcx: &mut DiagCtxt,
+) -> Result<TokenStream, ErrorGuaranteed> {
+    if let Some(unsafety) = input.unsafety {
+        dcx.error(unsafety, "implementing `PinnedDrop` is safe");
+    }
+    input.unsafety = Some(Token![unsafe](input.impl_token.span));
+    match &mut input.trait_ {
+        Some((not, path, _for)) => {
+            if let Some(not) = not {
+                dcx.error(not, "cannot implement `!PinnedDrop`");
+            }
+            for (seg, expected) in path
+                .segments
+                .iter()
+                .rev()
+                .zip(["PinnedDrop", "pin_init", ""])
+            {
+                if expected.is_empty() || seg.ident != expected {
+                    dcx.error(seg, "bad import path for `PinnedDrop`");
+                }
+                if !seg.arguments.is_none() {
+                    dcx.error(&seg.arguments, "unexpected arguments for `PinnedDrop` path");
+                }
+            }
+            *path = parse_quote!(::pin_init::PinnedDrop);
+        }
+        None => {
+            let span = input
+                .impl_token
+                .span
+                .join(input.self_ty.span())
+                .unwrap_or(input.impl_token.span);
+            dcx.error(
+                span,
+                "expected `impl ... PinnedDrop for ...`, got inherent impl",
+            );
+        }
+    }
+    for item in &mut input.items {
+        if let ImplItem::Fn(fn_item) = item {
+            if fn_item.sig.ident == "drop" {
+                fn_item
+                    .sig
+                    .inputs
+                    .push(parse_quote!(_: ::pin_init::__internal::OnlyCallFromDrop));
+            }
+        }
+    }
+    Ok(quote!(#input))
+}
diff --git a/rust/pin-init/internal/src/zeroable.rs b/rust/pin-init/internal/src/zeroable.rs
new file mode 100644
index 000000000..b11feaeb1
--- /dev/null
+++ b/rust/pin-init/internal/src/zeroable.rs
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::{parse_quote, Data, DeriveInput, Field, Fields};
+
+use crate::{diagnostics::ErrorGuaranteed, DiagCtxt};
+
+pub(crate) fn derive(
+    input: DeriveInput,
+    dcx: &mut DiagCtxt,
+) -> Result<TokenStream, ErrorGuaranteed> {
+    let fields = match input.data {
+        Data::Struct(data_struct) => data_struct.fields,
+        Data::Union(data_union) => Fields::Named(data_union.fields),
+        Data::Enum(data_enum) => {
+            return Err(dcx.error(data_enum.enum_token, "cannot derive `Zeroable` for an enum"));
+        }
+    };
+    let name = input.ident;
+    let mut generics = input.generics;
+    for param in generics.type_params_mut() {
+        param.bounds.insert(0, parse_quote!(::pin_init::Zeroable));
+    }
+    let (impl_gen, ty_gen, whr) = generics.split_for_impl();
+    let field_type = fields.iter().map(|field| &field.ty);
+    Ok(quote! {
+        // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
+        #[automatically_derived]
+        unsafe impl #impl_gen ::pin_init::Zeroable for #name #ty_gen
+            #whr
+        {}
+        const _: () = {
+            fn assert_zeroable<T: ?::core::marker::Sized + ::pin_init::Zeroable>() {}
+            fn ensure_zeroable #impl_gen ()
+                #whr
+            {
+                #(
+                    assert_zeroable::<#field_type>();
+                )*
+            }
+        };
+    })
+}
+
+pub(crate) fn maybe_derive(
+    input: DeriveInput,
+    dcx: &mut DiagCtxt,
+) -> Result<TokenStream, ErrorGuaranteed> {
+    let fields = match input.data {
+        Data::Struct(data_struct) => data_struct.fields,
+        Data::Union(data_union) => Fields::Named(data_union.fields),
+        Data::Enum(data_enum) => {
+            return Err(dcx.error(data_enum.enum_token, "cannot derive `Zeroable` for an enum"));
+        }
+    };
+    let name = input.ident;
+    let mut generics = input.generics;
+    for param in generics.type_params_mut() {
+        param.bounds.insert(0, parse_quote!(::pin_init::Zeroable));
+    }
+    for Field { ty, .. } in fields {
+        generics
+            .make_where_clause()
+            .predicates
+            // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds`
+            // feature <https://github.com/rust-lang/rust/issues/48214#issuecomment-2557829956>.
+            .push(parse_quote!(#ty: for<'__dummy> ::pin_init::Zeroable));
+    }
+    let (impl_gen, ty_gen, whr) = generics.split_for_impl();
+    Ok(quote! {
+        // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
+        #[automatically_derived]
+        unsafe impl #impl_gen ::pin_init::Zeroable for #name #ty_gen
+            #whr
+        {}
+    })
+}