1401 lines · rust
1// SPDX-License-Identifier: Apache-2.0 OR MIT2 3//! This module provides the macros that actually implement the proc-macros `pin_data` and4//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`5//! macros.6//!7//! These macros should never be called directly, since they expect their input to be8//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in9//! safe code! Use the public facing macros instead.10//!11//! This architecture has been chosen because the kernel does not yet have access to `syn` which12//! would make matters a lot easier for implementing these as proc-macros.13//!14//! # Macro expansion example15//!16//! This section is intended for readers trying to understand the macros in this module and the17//! `pin_init!` macros from `init.rs`.18//!19//! We will look at the following example:20//!21//! ```rust,ignore22//! # use kernel::init::*;23//! # use core::pin::Pin;24//! #[pin_data]25//! #[repr(C)]26//! struct Bar<T> {27//! #[pin]28//! t: T,29//! pub x: usize,30//! }31//!32//! impl<T> Bar<T> {33//! fn new(t: T) -> impl PinInit<Self> {34//! pin_init!(Self { t, x: 0 })35//! }36//! }37//!38//! #[pin_data(PinnedDrop)]39//! struct Foo {40//! a: usize,41//! #[pin]42//! b: Bar<u32>,43//! }44//!45//! #[pinned_drop]46//! impl PinnedDrop for Foo {47//! fn drop(self: Pin<&mut Self>) {48//! pr_info!("{self:p} is getting dropped.");49//! }50//! }51//!52//! let a = 42;53//! let initializer = pin_init!(Foo {54//! a,55//! b <- Bar::new(36),56//! });57//! ```58//!59//! This example includes the most common and important features of the pin-init API.60//!61//! Below you can find individual section about the different macro invocations. Here are some62//! general things we need to take into account when designing macros:63//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`64//! this ensures that the correct item is used, since users could define their own `mod core {}`65//! and then their own `panic!` inside to execute arbitrary code inside of our macro.66//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied67//! expressions inside of an `unsafe` block in the macro, because this would allow users to do68//! `unsafe` operations without an associated `unsafe` block.69//!70//! ## `#[pin_data]` on `Bar`71//!72//! This macro is used to specify which fields are structurally pinned and which fields are not. It73//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.74//!75//! Here is the definition of `Bar` from our example:76//!77//! ```rust,ignore78//! # use kernel::init::*;79//! #[pin_data]80//! #[repr(C)]81//! struct Bar<T> {82//! #[pin]83//! t: T,84//! pub x: usize,85//! }86//! ```87//!88//! This expands to the following code:89//!90//! ```rust,ignore91//! // Firstly the normal definition of the struct, attributes are preserved:92//! #[repr(C)]93//! struct Bar<T> {94//! t: T,95//! pub x: usize,96//! }97//! // Then an anonymous constant is defined, this is because we do not want any code to access the98//! // types that we define inside:99//! const _: () = {100//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,101//! // since we need to implement access functions for each field and thus need to know its102//! // type.103//! struct __ThePinData<T> {104//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,105//! }106//! // We implement `Copy` for the pin-data struct, since all functions it defines will take107//! // `self` by value.108//! impl<T> ::core::clone::Clone for __ThePinData<T> {109//! fn clone(&self) -> Self {110//! *self111//! }112//! }113//! impl<T> ::core::marker::Copy for __ThePinData<T> {}114//! // For every field of `Bar`, the pin-data struct will define a function with the same name115//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the116//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field117//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).118//! #[allow(dead_code)]119//! impl<T> __ThePinData<T> {120//! unsafe fn t<E>(121//! self,122//! slot: *mut T,123//! // Since `t` is `#[pin]`, this is `PinInit`.124//! init: impl ::kernel::init::PinInit<T, E>,125//! ) -> ::core::result::Result<(), E> {126//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }127//! }128//! pub unsafe fn x<E>(129//! self,130//! slot: *mut usize,131//! // Since `x` is not `#[pin]`, this is `Init`.132//! init: impl ::kernel::init::Init<usize, E>,133//! ) -> ::core::result::Result<(), E> {134//! unsafe { ::kernel::init::Init::__init(init, slot) }135//! }136//! }137//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct138//! // that we constructed above.139//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {140//! type PinData = __ThePinData<T>;141//! unsafe fn __pin_data() -> Self::PinData {142//! __ThePinData {143//! __phantom: ::core::marker::PhantomData,144//! }145//! }146//! }147//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data148//! // struct. This is important to ensure that no user can implement a rogue `__pin_data`149//! // function without using `unsafe`.150//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {151//! type Datee = Bar<T>;152//! }153//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is154//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned155//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our156//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist157//! // for two reasons:158//! // - `__phantom`: every generic must be used, since we cannot really know which generics159//! // are used, we declare all and then use everything here once.160//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant161//! // over it. The lifetime is needed to work around the limitation that trait bounds must162//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is163//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler164//! // into accepting these bounds regardless.165//! #[allow(dead_code)]166//! struct __Unpin<'__pin, T> {167//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,168//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,169//! // Our only `#[pin]` field is `t`.170//! t: T,171//! }172//! #[doc(hidden)]173//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>174//! where175//! __Unpin<'__pin, T>: ::core::marker::Unpin,176//! {}177//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users178//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to179//! // UB with only safe code, so we disallow this by giving a trait implementation error using180//! // a direct impl and a blanket implementation.181//! trait MustNotImplDrop {}182//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do183//! // (normally people want to know if a type has any kind of drop glue at all, here we want184//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).185//! #[allow(drop_bounds)]186//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}187//! impl<T> MustNotImplDrop for Bar<T> {}188//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to189//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed190//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.191//! #[allow(non_camel_case_types)]192//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}193//! impl<194//! T: ::kernel::init::PinnedDrop,195//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}196//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}197//! };198//! ```199//!200//! ## `pin_init!` in `impl Bar`201//!202//! This macro creates an pin-initializer for the given struct. It requires that the struct is203//! annotated by `#[pin_data]`.204//!205//! Here is the impl on `Bar` defining the new function:206//!207//! ```rust,ignore208//! impl<T> Bar<T> {209//! fn new(t: T) -> impl PinInit<Self> {210//! pin_init!(Self { t, x: 0 })211//! }212//! }213//! ```214//!215//! This expands to the following code:216//!217//! ```rust,ignore218//! impl<T> Bar<T> {219//! fn new(t: T) -> impl PinInit<Self> {220//! {221//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`222//! // return type and shadow it later when we insert the arbitrary user code. That way223//! // there will be no possibility of returning without `unsafe`.224//! struct __InitOk;225//! // Get the data about fields from the supplied type.226//! // - the function is unsafe, hence the unsafe block227//! // - we `use` the `HasPinData` trait in the block, it is only available in that228//! // scope.229//! let data = unsafe {230//! use ::kernel::init::__internal::HasPinData;231//! Self::__pin_data()232//! };233//! // Ensure that `data` really is of type `PinData` and help with type inference:234//! let init = ::kernel::init::__internal::PinData::make_closure::<235//! _,236//! __InitOk,237//! ::core::convert::Infallible,238//! >(data, move |slot| {239//! {240//! // Shadow the structure so it cannot be used to return early. If a user241//! // tries to write `return Ok(__InitOk)`, then they get a type error,242//! // since that will refer to this struct instead of the one defined243//! // above.244//! struct __InitOk;245//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.246//! {247//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };248//! }249//! // Since initialization could fail later (not in this case, since the250//! // error type is `Infallible`) we will need to drop this field if there251//! // is an error later. This `DropGuard` will drop the field when it gets252//! // dropped and has not yet been forgotten.253//! let __t_guard = unsafe {254//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))255//! };256//! // Expansion of `x: 0,`:257//! // Since this can be an arbitrary expression we cannot place it inside258//! // of the `unsafe` block, so we bind it here.259//! {260//! let x = 0;261//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };262//! }263//! // We again create a `DropGuard`.264//! let __x_guard = unsafe {265//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))266//! };267//! // Since initialization has successfully completed, we can now forget268//! // the guards. This is not `mem::forget`, since we only have269//! // `&DropGuard`.270//! ::core::mem::forget(__x_guard);271//! ::core::mem::forget(__t_guard);272//! // Here we use the type checker to ensure that every field has been273//! // initialized exactly once, since this is `if false` it will never get274//! // executed, but still type-checked.275//! // Additionally we abuse `slot` to automatically infer the correct type276//! // for the struct. This is also another check that every field is277//! // accessible from this scope.278//! #[allow(unreachable_code, clippy::diverging_sub_expression)]279//! let _ = || {280//! unsafe {281//! ::core::ptr::write(282//! slot,283//! Self {284//! // We only care about typecheck finding every field285//! // here, the expression does not matter, just conjure286//! // one using `panic!()`:287//! t: ::core::panic!(),288//! x: ::core::panic!(),289//! },290//! );291//! };292//! };293//! }294//! // We leave the scope above and gain access to the previously shadowed295//! // `__InitOk` that we need to return.296//! Ok(__InitOk)297//! });298//! // Change the return type from `__InitOk` to `()`.299//! let init = move |300//! slot,301//! | -> ::core::result::Result<(), ::core::convert::Infallible> {302//! init(slot).map(|__InitOk| ())303//! };304//! // Construct the initializer.305//! let init = unsafe {306//! ::kernel::init::pin_init_from_closure::<307//! _,308//! ::core::convert::Infallible,309//! >(init)310//! };311//! init312//! }313//! }314//! }315//! ```316//!317//! ## `#[pin_data]` on `Foo`318//!319//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the320//! differences/new things in the expansion of the `Foo` definition:321//!322//! ```rust,ignore323//! #[pin_data(PinnedDrop)]324//! struct Foo {325//! a: usize,326//! #[pin]327//! b: Bar<u32>,328//! }329//! ```330//!331//! This expands to the following code:332//!333//! ```rust,ignore334//! struct Foo {335//! a: usize,336//! b: Bar<u32>,337//! }338//! const _: () = {339//! struct __ThePinData {340//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,341//! }342//! impl ::core::clone::Clone for __ThePinData {343//! fn clone(&self) -> Self {344//! *self345//! }346//! }347//! impl ::core::marker::Copy for __ThePinData {}348//! #[allow(dead_code)]349//! impl __ThePinData {350//! unsafe fn b<E>(351//! self,352//! slot: *mut Bar<u32>,353//! init: impl ::kernel::init::PinInit<Bar<u32>, E>,354//! ) -> ::core::result::Result<(), E> {355//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }356//! }357//! unsafe fn a<E>(358//! self,359//! slot: *mut usize,360//! init: impl ::kernel::init::Init<usize, E>,361//! ) -> ::core::result::Result<(), E> {362//! unsafe { ::kernel::init::Init::__init(init, slot) }363//! }364//! }365//! unsafe impl ::kernel::init::__internal::HasPinData for Foo {366//! type PinData = __ThePinData;367//! unsafe fn __pin_data() -> Self::PinData {368//! __ThePinData {369//! __phantom: ::core::marker::PhantomData,370//! }371//! }372//! }373//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData {374//! type Datee = Foo;375//! }376//! #[allow(dead_code)]377//! struct __Unpin<'__pin> {378//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,379//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,380//! b: Bar<u32>,381//! }382//! #[doc(hidden)]383//! impl<'__pin> ::core::marker::Unpin for Foo384//! where385//! __Unpin<'__pin>: ::core::marker::Unpin,386//! {}387//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to388//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like389//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.390//! impl ::core::ops::Drop for Foo {391//! fn drop(&mut self) {392//! // Since we are getting dropped, no one else has a reference to `self` and thus we393//! // can assume that we never move.394//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };395//! // Create the unsafe token that proves that we are inside of a destructor, this396//! // type is only allowed to be created in a destructor.397//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };398//! ::kernel::init::PinnedDrop::drop(pinned, token);399//! }400//! }401//! };402//! ```403//!404//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`405//!406//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an407//! extra parameter that should not be used at all. The macro hides that parameter.408//!409//! Here is the `PinnedDrop` impl for `Foo`:410//!411//! ```rust,ignore412//! #[pinned_drop]413//! impl PinnedDrop for Foo {414//! fn drop(self: Pin<&mut Self>) {415//! pr_info!("{self:p} is getting dropped.");416//! }417//! }418//! ```419//!420//! This expands to the following code:421//!422//! ```rust,ignore423//! // `unsafe`, full path and the token parameter are added, everything else stays the same.424//! unsafe impl ::kernel::init::PinnedDrop for Foo {425//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {426//! pr_info!("{self:p} is getting dropped.");427//! }428//! }429//! ```430//!431//! ## `pin_init!` on `Foo`432//!433//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion434//! of `pin_init!` on `Foo`:435//!436//! ```rust,ignore437//! let a = 42;438//! let initializer = pin_init!(Foo {439//! a,440//! b <- Bar::new(36),441//! });442//! ```443//!444//! This expands to the following code:445//!446//! ```rust,ignore447//! let a = 42;448//! let initializer = {449//! struct __InitOk;450//! let data = unsafe {451//! use ::kernel::init::__internal::HasPinData;452//! Foo::__pin_data()453//! };454//! let init = ::kernel::init::__internal::PinData::make_closure::<455//! _,456//! __InitOk,457//! ::core::convert::Infallible,458//! >(data, move |slot| {459//! {460//! struct __InitOk;461//! {462//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };463//! }464//! let __a_guard = unsafe {465//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))466//! };467//! let init = Bar::new(36);468//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };469//! let __b_guard = unsafe {470//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))471//! };472//! ::core::mem::forget(__b_guard);473//! ::core::mem::forget(__a_guard);474//! #[allow(unreachable_code, clippy::diverging_sub_expression)]475//! let _ = || {476//! unsafe {477//! ::core::ptr::write(478//! slot,479//! Foo {480//! a: ::core::panic!(),481//! b: ::core::panic!(),482//! },483//! );484//! };485//! };486//! }487//! Ok(__InitOk)488//! });489//! let init = move |490//! slot,491//! | -> ::core::result::Result<(), ::core::convert::Infallible> {492//! init(slot).map(|__InitOk| ())493//! };494//! let init = unsafe {495//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)496//! };497//! init498//! };499//! ```500 501/// Creates a `unsafe impl<...> PinnedDrop for $type` block.502///503/// See [`PinnedDrop`] for more information.504#[doc(hidden)]505#[macro_export]506macro_rules! __pinned_drop {507 (508 @impl_sig($($impl_sig:tt)*),509 @impl_body(510 $(#[$($attr:tt)*])*511 fn drop($($sig:tt)*) {512 $($inner:tt)*513 }514 ),515 ) => {516 unsafe $($impl_sig)* {517 // Inherit all attributes and the type/ident tokens for the signature.518 $(#[$($attr)*])*519 fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {520 $($inner)*521 }522 }523 }524}525 526/// This macro first parses the struct definition such that it separates pinned and not pinned527/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.528#[doc(hidden)]529#[macro_export]530macro_rules! __pin_data {531 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.532 (parse_input:533 @args($($pinned_drop:ident)?),534 @sig(535 $(#[$($struct_attr:tt)*])*536 $vis:vis struct $name:ident537 $(where $($whr:tt)*)?538 ),539 @impl_generics($($impl_generics:tt)*),540 @ty_generics($($ty_generics:tt)*),541 @decl_generics($($decl_generics:tt)*),542 @body({ $($fields:tt)* }),543 ) => {544 // We now use token munching to iterate through all of the fields. While doing this we545 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user546 // wants these to be structurally pinned. The rest of the fields are the547 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right548 // order to declare the struct.549 //550 // In this call we also put some explaining comments for the parameters.551 $crate::__pin_data!(find_pinned_fields:552 // Attributes on the struct itself, these will just be propagated to be put onto the553 // struct definition.554 @struct_attrs($(#[$($struct_attr)*])*),555 // The visibility of the struct.556 @vis($vis),557 // The name of the struct.558 @name($name),559 // The 'impl generics', the generics that will need to be specified on the struct inside560 // of an `impl<$ty_generics>` block.561 @impl_generics($($impl_generics)*),562 // The 'ty generics', the generics that will need to be specified on the impl blocks.563 @ty_generics($($ty_generics)*),564 // The 'decl generics', the generics that need to be specified on the struct565 // definition.566 @decl_generics($($decl_generics)*),567 // The where clause of any impl block and the declaration.568 @where($($($whr)*)?),569 // The remaining fields tokens that need to be processed.570 // We add a `,` at the end to ensure correct parsing.571 @fields_munch($($fields)* ,),572 // The pinned fields.573 @pinned(),574 // The not pinned fields.575 @not_pinned(),576 // All fields.577 @fields(),578 // The accumulator containing all attributes already parsed.579 @accum(),580 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.581 @is_pinned(),582 // The proc-macro argument, this should be `PinnedDrop` or ``.583 @pinned_drop($($pinned_drop)?),584 );585 };586 (find_pinned_fields:587 @struct_attrs($($struct_attrs:tt)*),588 @vis($vis:vis),589 @name($name:ident),590 @impl_generics($($impl_generics:tt)*),591 @ty_generics($($ty_generics:tt)*),592 @decl_generics($($decl_generics:tt)*),593 @where($($whr:tt)*),594 // We found a PhantomPinned field, this should generally be pinned!595 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),596 @pinned($($pinned:tt)*),597 @not_pinned($($not_pinned:tt)*),598 @fields($($fields:tt)*),599 @accum($($accum:tt)*),600 // This field is not pinned.601 @is_pinned(),602 @pinned_drop($($pinned_drop:ident)?),603 ) => {604 ::core::compile_error!(concat!(605 "The field `",606 stringify!($field),607 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",608 ));609 $crate::__pin_data!(find_pinned_fields:610 @struct_attrs($($struct_attrs)*),611 @vis($vis),612 @name($name),613 @impl_generics($($impl_generics)*),614 @ty_generics($($ty_generics)*),615 @decl_generics($($decl_generics)*),616 @where($($whr)*),617 @fields_munch($($rest)*),618 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),619 @not_pinned($($not_pinned)*),620 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),621 @accum(),622 @is_pinned(),623 @pinned_drop($($pinned_drop)?),624 );625 };626 (find_pinned_fields:627 @struct_attrs($($struct_attrs:tt)*),628 @vis($vis:vis),629 @name($name:ident),630 @impl_generics($($impl_generics:tt)*),631 @ty_generics($($ty_generics:tt)*),632 @decl_generics($($decl_generics:tt)*),633 @where($($whr:tt)*),634 // We reached the field declaration.635 @fields_munch($field:ident : $type:ty, $($rest:tt)*),636 @pinned($($pinned:tt)*),637 @not_pinned($($not_pinned:tt)*),638 @fields($($fields:tt)*),639 @accum($($accum:tt)*),640 // This field is pinned.641 @is_pinned(yes),642 @pinned_drop($($pinned_drop:ident)?),643 ) => {644 $crate::__pin_data!(find_pinned_fields:645 @struct_attrs($($struct_attrs)*),646 @vis($vis),647 @name($name),648 @impl_generics($($impl_generics)*),649 @ty_generics($($ty_generics)*),650 @decl_generics($($decl_generics)*),651 @where($($whr)*),652 @fields_munch($($rest)*),653 @pinned($($pinned)* $($accum)* $field: $type,),654 @not_pinned($($not_pinned)*),655 @fields($($fields)* $($accum)* $field: $type,),656 @accum(),657 @is_pinned(),658 @pinned_drop($($pinned_drop)?),659 );660 };661 (find_pinned_fields:662 @struct_attrs($($struct_attrs:tt)*),663 @vis($vis:vis),664 @name($name:ident),665 @impl_generics($($impl_generics:tt)*),666 @ty_generics($($ty_generics:tt)*),667 @decl_generics($($decl_generics:tt)*),668 @where($($whr:tt)*),669 // We reached the field declaration.670 @fields_munch($field:ident : $type:ty, $($rest:tt)*),671 @pinned($($pinned:tt)*),672 @not_pinned($($not_pinned:tt)*),673 @fields($($fields:tt)*),674 @accum($($accum:tt)*),675 // This field is not pinned.676 @is_pinned(),677 @pinned_drop($($pinned_drop:ident)?),678 ) => {679 $crate::__pin_data!(find_pinned_fields:680 @struct_attrs($($struct_attrs)*),681 @vis($vis),682 @name($name),683 @impl_generics($($impl_generics)*),684 @ty_generics($($ty_generics)*),685 @decl_generics($($decl_generics)*),686 @where($($whr)*),687 @fields_munch($($rest)*),688 @pinned($($pinned)*),689 @not_pinned($($not_pinned)* $($accum)* $field: $type,),690 @fields($($fields)* $($accum)* $field: $type,),691 @accum(),692 @is_pinned(),693 @pinned_drop($($pinned_drop)?),694 );695 };696 (find_pinned_fields:697 @struct_attrs($($struct_attrs:tt)*),698 @vis($vis:vis),699 @name($name:ident),700 @impl_generics($($impl_generics:tt)*),701 @ty_generics($($ty_generics:tt)*),702 @decl_generics($($decl_generics:tt)*),703 @where($($whr:tt)*),704 // We found the `#[pin]` attr.705 @fields_munch(#[pin] $($rest:tt)*),706 @pinned($($pinned:tt)*),707 @not_pinned($($not_pinned:tt)*),708 @fields($($fields:tt)*),709 @accum($($accum:tt)*),710 @is_pinned($($is_pinned:ident)?),711 @pinned_drop($($pinned_drop:ident)?),712 ) => {713 $crate::__pin_data!(find_pinned_fields:714 @struct_attrs($($struct_attrs)*),715 @vis($vis),716 @name($name),717 @impl_generics($($impl_generics)*),718 @ty_generics($($ty_generics)*),719 @decl_generics($($decl_generics)*),720 @where($($whr)*),721 @fields_munch($($rest)*),722 // We do not include `#[pin]` in the list of attributes, since it is not actually an723 // attribute that is defined somewhere.724 @pinned($($pinned)*),725 @not_pinned($($not_pinned)*),726 @fields($($fields)*),727 @accum($($accum)*),728 // Set this to `yes`.729 @is_pinned(yes),730 @pinned_drop($($pinned_drop)?),731 );732 };733 (find_pinned_fields:734 @struct_attrs($($struct_attrs:tt)*),735 @vis($vis:vis),736 @name($name:ident),737 @impl_generics($($impl_generics:tt)*),738 @ty_generics($($ty_generics:tt)*),739 @decl_generics($($decl_generics:tt)*),740 @where($($whr:tt)*),741 // We reached the field declaration with visibility, for simplicity we only munch the742 // visibility and put it into `$accum`.743 @fields_munch($fvis:vis $field:ident $($rest:tt)*),744 @pinned($($pinned:tt)*),745 @not_pinned($($not_pinned:tt)*),746 @fields($($fields:tt)*),747 @accum($($accum:tt)*),748 @is_pinned($($is_pinned:ident)?),749 @pinned_drop($($pinned_drop:ident)?),750 ) => {751 $crate::__pin_data!(find_pinned_fields:752 @struct_attrs($($struct_attrs)*),753 @vis($vis),754 @name($name),755 @impl_generics($($impl_generics)*),756 @ty_generics($($ty_generics)*),757 @decl_generics($($decl_generics)*),758 @where($($whr)*),759 @fields_munch($field $($rest)*),760 @pinned($($pinned)*),761 @not_pinned($($not_pinned)*),762 @fields($($fields)*),763 @accum($($accum)* $fvis),764 @is_pinned($($is_pinned)?),765 @pinned_drop($($pinned_drop)?),766 );767 };768 (find_pinned_fields:769 @struct_attrs($($struct_attrs:tt)*),770 @vis($vis:vis),771 @name($name:ident),772 @impl_generics($($impl_generics:tt)*),773 @ty_generics($($ty_generics:tt)*),774 @decl_generics($($decl_generics:tt)*),775 @where($($whr:tt)*),776 // Some other attribute, just put it into `$accum`.777 @fields_munch(#[$($attr:tt)*] $($rest:tt)*),778 @pinned($($pinned:tt)*),779 @not_pinned($($not_pinned:tt)*),780 @fields($($fields:tt)*),781 @accum($($accum:tt)*),782 @is_pinned($($is_pinned:ident)?),783 @pinned_drop($($pinned_drop:ident)?),784 ) => {785 $crate::__pin_data!(find_pinned_fields:786 @struct_attrs($($struct_attrs)*),787 @vis($vis),788 @name($name),789 @impl_generics($($impl_generics)*),790 @ty_generics($($ty_generics)*),791 @decl_generics($($decl_generics)*),792 @where($($whr)*),793 @fields_munch($($rest)*),794 @pinned($($pinned)*),795 @not_pinned($($not_pinned)*),796 @fields($($fields)*),797 @accum($($accum)* #[$($attr)*]),798 @is_pinned($($is_pinned)?),799 @pinned_drop($($pinned_drop)?),800 );801 };802 (find_pinned_fields:803 @struct_attrs($($struct_attrs:tt)*),804 @vis($vis:vis),805 @name($name:ident),806 @impl_generics($($impl_generics:tt)*),807 @ty_generics($($ty_generics:tt)*),808 @decl_generics($($decl_generics:tt)*),809 @where($($whr:tt)*),810 // We reached the end of the fields, plus an optional additional comma, since we added one811 // before and the user is also allowed to put a trailing comma.812 @fields_munch($(,)?),813 @pinned($($pinned:tt)*),814 @not_pinned($($not_pinned:tt)*),815 @fields($($fields:tt)*),816 @accum(),817 @is_pinned(),818 @pinned_drop($($pinned_drop:ident)?),819 ) => {820 // Declare the struct with all fields in the correct order.821 $($struct_attrs)*822 $vis struct $name <$($decl_generics)*>823 where $($whr)*824 {825 $($fields)*826 }827 828 // We put the rest into this const item, because it then will not be accessible to anything829 // outside.830 const _: () = {831 // We declare this struct which will host all of the projection function for our type.832 // it will be invariant over all generic parameters which are inherited from the833 // struct.834 $vis struct __ThePinData<$($impl_generics)*>835 where $($whr)*836 {837 __phantom: ::core::marker::PhantomData<838 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>839 >,840 }841 842 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>843 where $($whr)*844 {845 fn clone(&self) -> Self { *self }846 }847 848 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>849 where $($whr)*850 {}851 852 // Make all projection functions.853 $crate::__pin_data!(make_pin_data:854 @pin_data(__ThePinData),855 @impl_generics($($impl_generics)*),856 @ty_generics($($ty_generics)*),857 @where($($whr)*),858 @pinned($($pinned)*),859 @not_pinned($($not_pinned)*),860 );861 862 // SAFETY: We have added the correct projection functions above to `__ThePinData` and863 // we also use the least restrictive generics possible.864 unsafe impl<$($impl_generics)*>865 $crate::init::__internal::HasPinData for $name<$($ty_generics)*>866 where $($whr)*867 {868 type PinData = __ThePinData<$($ty_generics)*>;869 870 unsafe fn __pin_data() -> Self::PinData {871 __ThePinData { __phantom: ::core::marker::PhantomData }872 }873 }874 875 unsafe impl<$($impl_generics)*>876 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>877 where $($whr)*878 {879 type Datee = $name<$($ty_generics)*>;880 }881 882 // This struct will be used for the unpin analysis. Since only structurally pinned883 // fields are relevant whether the struct should implement `Unpin`.884 #[allow(dead_code)]885 struct __Unpin <'__pin, $($impl_generics)*>886 where $($whr)*887 {888 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,889 __phantom: ::core::marker::PhantomData<890 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>891 >,892 // Only the pinned fields.893 $($pinned)*894 }895 896 #[doc(hidden)]897 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>898 where899 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,900 $($whr)*901 {}902 903 // We need to disallow normal `Drop` implementation, the exact behavior depends on904 // whether `PinnedDrop` was specified as the parameter.905 $crate::__pin_data!(drop_prevention:906 @name($name),907 @impl_generics($($impl_generics)*),908 @ty_generics($($ty_generics)*),909 @where($($whr)*),910 @pinned_drop($($pinned_drop)?),911 );912 };913 };914 // When no `PinnedDrop` was specified, then we have to prevent implementing drop.915 (drop_prevention:916 @name($name:ident),917 @impl_generics($($impl_generics:tt)*),918 @ty_generics($($ty_generics:tt)*),919 @where($($whr:tt)*),920 @pinned_drop(),921 ) => {922 // We prevent this by creating a trait that will be implemented for all types implementing923 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,924 // if it also implements `Drop`925 trait MustNotImplDrop {}926 #[allow(drop_bounds)]927 impl<T: ::core::ops::Drop> MustNotImplDrop for T {}928 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>929 where $($whr)* {}930 // We also take care to prevent users from writing a useless `PinnedDrop` implementation.931 // They might implement `PinnedDrop` correctly for the struct, but forget to give932 // `PinnedDrop` as the parameter to `#[pin_data]`.933 #[allow(non_camel_case_types)]934 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}935 impl<T: $crate::init::PinnedDrop>936 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}937 impl<$($impl_generics)*>938 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>939 where $($whr)* {}940 };941 // When `PinnedDrop` was specified we just implement `Drop` and delegate.942 (drop_prevention:943 @name($name:ident),944 @impl_generics($($impl_generics:tt)*),945 @ty_generics($($ty_generics:tt)*),946 @where($($whr:tt)*),947 @pinned_drop(PinnedDrop),948 ) => {949 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>950 where $($whr)*951 {952 fn drop(&mut self) {953 // SAFETY: Since this is a destructor, `self` will not move after this function954 // terminates, since it is inaccessible.955 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };956 // SAFETY: Since this is a drop function, we can create this token to call the957 // pinned destructor of this type.958 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };959 $crate::init::PinnedDrop::drop(pinned, token);960 }961 }962 };963 // If some other parameter was specified, we emit a readable error.964 (drop_prevention:965 @name($name:ident),966 @impl_generics($($impl_generics:tt)*),967 @ty_generics($($ty_generics:tt)*),968 @where($($whr:tt)*),969 @pinned_drop($($rest:tt)*),970 ) => {971 compile_error!(972 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",973 stringify!($($rest)*),974 );975 };976 (make_pin_data:977 @pin_data($pin_data:ident),978 @impl_generics($($impl_generics:tt)*),979 @ty_generics($($ty_generics:tt)*),980 @where($($whr:tt)*),981 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),982 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),983 ) => {984 // For every field, we create a projection function according to its projection type. If a985 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not986 // structurally pinned, then it can be initialized via `Init`.987 //988 // The functions are `unsafe` to prevent accidentally calling them.989 #[allow(dead_code)]990 impl<$($impl_generics)*> $pin_data<$($ty_generics)*>991 where $($whr)*992 {993 $(994 $(#[$($p_attr)*])*995 $pvis unsafe fn $p_field<E>(996 self,997 slot: *mut $p_type,998 init: impl $crate::init::PinInit<$p_type, E>,999 ) -> ::core::result::Result<(), E> {1000 unsafe { $crate::init::PinInit::__pinned_init(init, slot) }1001 }1002 )*1003 $(1004 $(#[$($attr)*])*1005 $fvis unsafe fn $field<E>(1006 self,1007 slot: *mut $type,1008 init: impl $crate::init::Init<$type, E>,1009 ) -> ::core::result::Result<(), E> {1010 unsafe { $crate::init::Init::__init(init, slot) }1011 }1012 )*1013 }1014 };1015}1016 1017/// The internal init macro. Do not call manually!1018///1019/// This is called by the `{try_}{pin_}init!` macros with various inputs.1020///1021/// This macro has multiple internal call configurations, these are always the very first ident:1022/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.1023/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.1024/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.1025/// - `make_initializer`: recursively create the struct initializer that guarantees that every1026/// field has been initialized exactly once.1027#[doc(hidden)]1028#[macro_export]1029macro_rules! __init_internal {1030 (1031 @this($($this:ident)?),1032 @typ($t:path),1033 @fields($($fields:tt)*),1034 @error($err:ty),1035 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`1036 // case.1037 @data($data:ident, $($use_data:ident)?),1038 // `HasPinData` or `HasInitData`.1039 @has_data($has_data:ident, $get_data:ident),1040 // `pin_init_from_closure` or `init_from_closure`.1041 @construct_closure($construct_closure:ident),1042 @munch_fields(),1043 ) => {1044 $crate::__init_internal!(with_update_parsed:1045 @this($($this)?),1046 @typ($t),1047 @fields($($fields)*),1048 @error($err),1049 @data($data, $($use_data)?),1050 @has_data($has_data, $get_data),1051 @construct_closure($construct_closure),1052 @zeroed(), // Nothing means default behavior.1053 )1054 };1055 (1056 @this($($this:ident)?),1057 @typ($t:path),1058 @fields($($fields:tt)*),1059 @error($err:ty),1060 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`1061 // case.1062 @data($data:ident, $($use_data:ident)?),1063 // `HasPinData` or `HasInitData`.1064 @has_data($has_data:ident, $get_data:ident),1065 // `pin_init_from_closure` or `init_from_closure`.1066 @construct_closure($construct_closure:ident),1067 @munch_fields(..Zeroable::zeroed()),1068 ) => {1069 $crate::__init_internal!(with_update_parsed:1070 @this($($this)?),1071 @typ($t),1072 @fields($($fields)*),1073 @error($err),1074 @data($data, $($use_data)?),1075 @has_data($has_data, $get_data),1076 @construct_closure($construct_closure),1077 @zeroed(()), // `()` means zero all fields not mentioned.1078 )1079 };1080 (1081 @this($($this:ident)?),1082 @typ($t:path),1083 @fields($($fields:tt)*),1084 @error($err:ty),1085 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`1086 // case.1087 @data($data:ident, $($use_data:ident)?),1088 // `HasPinData` or `HasInitData`.1089 @has_data($has_data:ident, $get_data:ident),1090 // `pin_init_from_closure` or `init_from_closure`.1091 @construct_closure($construct_closure:ident),1092 @munch_fields($ignore:tt $($rest:tt)*),1093 ) => {1094 $crate::__init_internal!(1095 @this($($this)?),1096 @typ($t),1097 @fields($($fields)*),1098 @error($err),1099 @data($data, $($use_data)?),1100 @has_data($has_data, $get_data),1101 @construct_closure($construct_closure),1102 @munch_fields($($rest)*),1103 )1104 };1105 (with_update_parsed:1106 @this($($this:ident)?),1107 @typ($t:path),1108 @fields($($fields:tt)*),1109 @error($err:ty),1110 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`1111 // case.1112 @data($data:ident, $($use_data:ident)?),1113 // `HasPinData` or `HasInitData`.1114 @has_data($has_data:ident, $get_data:ident),1115 // `pin_init_from_closure` or `init_from_closure`.1116 @construct_closure($construct_closure:ident),1117 @zeroed($($init_zeroed:expr)?),1118 ) => {{1119 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return1120 // type and shadow it later when we insert the arbitrary user code. That way there will be1121 // no possibility of returning without `unsafe`.1122 struct __InitOk;1123 // Get the data about fields from the supplied type.1124 let data = unsafe {1125 use $crate::init::__internal::$has_data;1126 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal1127 // information that is associated to already parsed fragments, so a path fragment1128 // cannot be used in this position. Doing the retokenization results in valid rust1129 // code.1130 ::kernel::macros::paste!($t::$get_data())1131 };1132 // Ensure that `data` really is of type `$data` and help with type inference:1133 let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(1134 data,1135 move |slot| {1136 {1137 // Shadow the structure so it cannot be used to return early.1138 struct __InitOk;1139 // If `$init_zeroed` is present we should zero the slot now and not emit an1140 // error when fields are missing (since they will be zeroed). We also have to1141 // check that the type actually implements `Zeroable`.1142 $({1143 fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}1144 // Ensure that the struct is indeed `Zeroable`.1145 assert_zeroable(slot);1146 // SAFETY: The type implements `Zeroable` by the check above.1147 unsafe { ::core::ptr::write_bytes(slot, 0, 1) };1148 $init_zeroed // This will be `()` if set.1149 })?1150 // Create the `this` so it can be referenced by the user inside of the1151 // expressions creating the individual fields.1152 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?1153 // Initialize every field.1154 $crate::__init_internal!(init_slot($($use_data)?):1155 @data(data),1156 @slot(slot),1157 @guards(),1158 @munch_fields($($fields)*,),1159 );1160 // We use unreachable code to ensure that all fields have been mentioned exactly1161 // once, this struct initializer will still be type-checked and complain with a1162 // very natural error message if a field is forgotten/mentioned more than once.1163 #[allow(unreachable_code, clippy::diverging_sub_expression)]1164 let _ = || {1165 $crate::__init_internal!(make_initializer:1166 @slot(slot),1167 @type_name($t),1168 @munch_fields($($fields)*,),1169 @acc(),1170 );1171 };1172 }1173 Ok(__InitOk)1174 }1175 );1176 let init = move |slot| -> ::core::result::Result<(), $err> {1177 init(slot).map(|__InitOk| ())1178 };1179 let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };1180 init1181 }};1182 (init_slot($($use_data:ident)?):1183 @data($data:ident),1184 @slot($slot:ident),1185 @guards($($guards:ident,)*),1186 @munch_fields($(..Zeroable::zeroed())? $(,)?),1187 ) => {1188 // Endpoint of munching, no fields are left. If execution reaches this point, all fields1189 // have been initialized. Therefore we can now dismiss the guards by forgetting them.1190 $(::core::mem::forget($guards);)*1191 };1192 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.1193 @data($data:ident),1194 @slot($slot:ident),1195 @guards($($guards:ident,)*),1196 // In-place initialization syntax.1197 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),1198 ) => {1199 let init = $val;1200 // Call the initializer.1201 //1202 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we1203 // return when an error/panic occurs.1204 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.1205 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };1206 // Create the drop guard:1207 //1208 // We rely on macro hygiene to make it impossible for users to access this local variable.1209 // We use `paste!` to create new hygiene for `$field`.1210 ::kernel::macros::paste! {1211 // SAFETY: We forget the guard later when initialization has succeeded.1212 let [< __ $field _guard >] = unsafe {1213 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))1214 };1215 1216 $crate::__init_internal!(init_slot($use_data):1217 @data($data),1218 @slot($slot),1219 @guards([< __ $field _guard >], $($guards,)*),1220 @munch_fields($($rest)*),1221 );1222 }1223 };1224 (init_slot(): // No `use_data`, so we use `Init::__init` directly.1225 @data($data:ident),1226 @slot($slot:ident),1227 @guards($($guards:ident,)*),1228 // In-place initialization syntax.1229 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),1230 ) => {1231 let init = $val;1232 // Call the initializer.1233 //1234 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we1235 // return when an error/panic occurs.1236 unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };1237 // Create the drop guard:1238 //1239 // We rely on macro hygiene to make it impossible for users to access this local variable.1240 // We use `paste!` to create new hygiene for `$field`.1241 ::kernel::macros::paste! {1242 // SAFETY: We forget the guard later when initialization has succeeded.1243 let [< __ $field _guard >] = unsafe {1244 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))1245 };1246 1247 $crate::__init_internal!(init_slot():1248 @data($data),1249 @slot($slot),1250 @guards([< __ $field _guard >], $($guards,)*),1251 @munch_fields($($rest)*),1252 );1253 }1254 };1255 (init_slot($($use_data:ident)?):1256 @data($data:ident),1257 @slot($slot:ident),1258 @guards($($guards:ident,)*),1259 // Init by-value.1260 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),1261 ) => {1262 {1263 $(let $field = $val;)?1264 // Initialize the field.1265 //1266 // SAFETY: The memory at `slot` is uninitialized.1267 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };1268 }1269 // Create the drop guard:1270 //1271 // We rely on macro hygiene to make it impossible for users to access this local variable.1272 // We use `paste!` to create new hygiene for `$field`.1273 ::kernel::macros::paste! {1274 // SAFETY: We forget the guard later when initialization has succeeded.1275 let [< __ $field _guard >] = unsafe {1276 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))1277 };1278 1279 $crate::__init_internal!(init_slot($($use_data)?):1280 @data($data),1281 @slot($slot),1282 @guards([< __ $field _guard >], $($guards,)*),1283 @munch_fields($($rest)*),1284 );1285 }1286 };1287 (make_initializer:1288 @slot($slot:ident),1289 @type_name($t:path),1290 @munch_fields(..Zeroable::zeroed() $(,)?),1291 @acc($($acc:tt)*),1292 ) => {1293 // Endpoint, nothing more to munch, create the initializer. Since the users specified1294 // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have1295 // not been overwritten are thus zero and initialized. We still check that all fields are1296 // actually accessible by using the struct update syntax ourselves.1297 // We are inside of a closure that is never executed and thus we can abuse `slot` to1298 // get the correct type inference here:1299 #[allow(unused_assignments)]1300 unsafe {1301 let mut zeroed = ::core::mem::zeroed();1302 // We have to use type inference here to make zeroed have the correct type. This does1303 // not get executed, so it has no effect.1304 ::core::ptr::write($slot, zeroed);1305 zeroed = ::core::mem::zeroed();1306 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal1307 // information that is associated to already parsed fragments, so a path fragment1308 // cannot be used in this position. Doing the retokenization results in valid rust1309 // code.1310 ::kernel::macros::paste!(1311 ::core::ptr::write($slot, $t {1312 $($acc)*1313 ..zeroed1314 });1315 );1316 }1317 };1318 (make_initializer:1319 @slot($slot:ident),1320 @type_name($t:path),1321 @munch_fields($(,)?),1322 @acc($($acc:tt)*),1323 ) => {1324 // Endpoint, nothing more to munch, create the initializer.1325 // Since we are in the closure that is never called, this will never get executed.1326 // We abuse `slot` to get the correct type inference here:1327 unsafe {1328 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal1329 // information that is associated to already parsed fragments, so a path fragment1330 // cannot be used in this position. Doing the retokenization results in valid rust1331 // code.1332 ::kernel::macros::paste!(1333 ::core::ptr::write($slot, $t {1334 $($acc)*1335 });1336 );1337 }1338 };1339 (make_initializer:1340 @slot($slot:ident),1341 @type_name($t:path),1342 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),1343 @acc($($acc:tt)*),1344 ) => {1345 $crate::__init_internal!(make_initializer:1346 @slot($slot),1347 @type_name($t),1348 @munch_fields($($rest)*),1349 @acc($($acc)* $field: ::core::panic!(),),1350 );1351 };1352 (make_initializer:1353 @slot($slot:ident),1354 @type_name($t:path),1355 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),1356 @acc($($acc:tt)*),1357 ) => {1358 $crate::__init_internal!(make_initializer:1359 @slot($slot),1360 @type_name($t),1361 @munch_fields($($rest)*),1362 @acc($($acc)* $field: ::core::panic!(),),1363 );1364 };1365}1366 1367#[doc(hidden)]1368#[macro_export]1369macro_rules! __derive_zeroable {1370 (parse_input:1371 @sig(1372 $(#[$($struct_attr:tt)*])*1373 $vis:vis struct $name:ident1374 $(where $($whr:tt)*)?1375 ),1376 @impl_generics($($impl_generics:tt)*),1377 @ty_generics($($ty_generics:tt)*),1378 @body({1379 $(1380 $(#[$($field_attr:tt)*])*1381 $field:ident : $field_ty:ty1382 ),* $(,)?1383 }),1384 ) => {1385 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.1386 #[automatically_derived]1387 unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>1388 where1389 $($($whr)*)?1390 {}1391 const _: () = {1392 fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}1393 fn ensure_zeroable<$($impl_generics)*>()1394 where $($($whr)*)?1395 {1396 $(assert_zeroable::<$field_ty>();)*1397 }1398 };1399 };1400}1401