kernel/init/macros.rs
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! This module provides the macros that actually implement the proc-macros `pin_data` and
4//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
5//! macros.
6//!
7//! These macros should never be called directly, since they expect their input to be
8//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
9//! safe code! Use the public facing macros instead.
10//!
11//! This architecture has been chosen because the kernel does not yet have access to `syn` which
12//! would make matters a lot easier for implementing these as proc-macros.
13//!
14//! # Macro expansion example
15//!
16//! This section is intended for readers trying to understand the macros in this module and the
17//! `pin_init!` macros from `init.rs`.
18//!
19//! We will look at the following example:
20//!
21//! ```rust,ignore
22//! # use kernel::init::*;
23//! # use core::pin::Pin;
24//! #[pin_data]
25//! #[repr(C)]
26//! struct Bar<T> {
27//! #[pin]
28//! t: T,
29//! pub x: usize,
30//! }
31//!
32//! impl<T> Bar<T> {
33//! fn new(t: T) -> impl PinInit<Self> {
34//! pin_init!(Self { t, x: 0 })
35//! }
36//! }
37//!
38//! #[pin_data(PinnedDrop)]
39//! struct Foo {
40//! a: usize,
41//! #[pin]
42//! b: Bar<u32>,
43//! }
44//!
45//! #[pinned_drop]
46//! impl PinnedDrop for Foo {
47//! fn drop(self: Pin<&mut Self>) {
48//! pr_info!("{self:p} is getting dropped.\n");
49//! }
50//! }
51//!
52//! let a = 42;
53//! let initializer = pin_init!(Foo {
54//! a,
55//! b <- Bar::new(36),
56//! });
57//! ```
58//!
59//! This example includes the most common and important features of the pin-init API.
60//!
61//! Below you can find individual section about the different macro invocations. Here are some
62//! general things we need to take into account when designing macros:
63//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
64//! this ensures that the correct item is used, since users could define their own `mod core {}`
65//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
66//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
67//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
68//! `unsafe` operations without an associated `unsafe` block.
69//!
70//! ## `#[pin_data]` on `Bar`
71//!
72//! This macro is used to specify which fields are structurally pinned and which fields are not. It
73//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
74//!
75//! Here is the definition of `Bar` from our example:
76//!
77//! ```rust,ignore
78//! # use kernel::init::*;
79//! #[pin_data]
80//! #[repr(C)]
81//! struct Bar<T> {
82//! #[pin]
83//! t: T,
84//! pub x: usize,
85//! }
86//! ```
87//!
88//! This expands to the following code:
89//!
90//! ```rust,ignore
91//! // Firstly the normal definition of the struct, attributes are preserved:
92//! #[repr(C)]
93//! struct Bar<T> {
94//! t: T,
95//! pub x: usize,
96//! }
97//! // Then an anonymous constant is defined, this is because we do not want any code to access the
98//! // types that we define inside:
99//! const _: () = {
100//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
101//! // since we need to implement access functions for each field and thus need to know its
102//! // type.
103//! struct __ThePinData<T> {
104//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
105//! }
106//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
107//! // `self` by value.
108//! impl<T> ::core::clone::Clone for __ThePinData<T> {
109//! fn clone(&self) -> Self {
110//! *self
111//! }
112//! }
113//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
114//! // For every field of `Bar`, the pin-data struct will define a function with the same name
115//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
116//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
117//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
118//! #[allow(dead_code)]
119//! impl<T> __ThePinData<T> {
120//! unsafe fn t<E>(
121//! self,
122//! slot: *mut T,
123//! // Since `t` is `#[pin]`, this is `PinInit`.
124//! init: impl ::kernel::init::PinInit<T, E>,
125//! ) -> ::core::result::Result<(), E> {
126//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
127//! }
128//! pub unsafe fn x<E>(
129//! self,
130//! slot: *mut usize,
131//! // Since `x` is not `#[pin]`, this is `Init`.
132//! init: impl ::kernel::init::Init<usize, E>,
133//! ) -> ::core::result::Result<(), E> {
134//! unsafe { ::kernel::init::Init::__init(init, slot) }
135//! }
136//! }
137//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
138//! // that we constructed above.
139//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
140//! type PinData = __ThePinData<T>;
141//! unsafe fn __pin_data() -> Self::PinData {
142//! __ThePinData {
143//! __phantom: ::core::marker::PhantomData,
144//! }
145//! }
146//! }
147//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
148//! // struct. This is important to ensure that no user can implement a rogue `__pin_data`
149//! // function without using `unsafe`.
150//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
151//! type Datee = Bar<T>;
152//! }
153//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
154//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
155//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
156//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
157//! // for two reasons:
158//! // - `__phantom`: every generic must be used, since we cannot really know which generics
159//! // are used, we declare all and then use everything here once.
160//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
161//! // over it. The lifetime is needed to work around the limitation that trait bounds must
162//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
163//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
164//! // into accepting these bounds regardless.
165//! #[allow(dead_code)]
166//! struct __Unpin<'__pin, T> {
167//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
168//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
169//! // Our only `#[pin]` field is `t`.
170//! t: T,
171//! }
172//! #[doc(hidden)]
173//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
174//! where
175//! __Unpin<'__pin, T>: ::core::marker::Unpin,
176//! {}
177//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
178//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
179//! // UB with only safe code, so we disallow this by giving a trait implementation error using
180//! // a direct impl and a blanket implementation.
181//! trait MustNotImplDrop {}
182//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
183//! // (normally people want to know if a type has any kind of drop glue at all, here we want
184//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
185//! #[expect(drop_bounds)]
186//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
187//! impl<T> MustNotImplDrop for Bar<T> {}
188//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
189//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
190//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
191//! #[expect(non_camel_case_types)]
192//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
193//! impl<
194//! T: ::kernel::init::PinnedDrop,
195//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
196//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
197//! };
198//! ```
199//!
200//! ## `pin_init!` in `impl Bar`
201//!
202//! This macro creates an pin-initializer for the given struct. It requires that the struct is
203//! annotated by `#[pin_data]`.
204//!
205//! Here is the impl on `Bar` defining the new function:
206//!
207//! ```rust,ignore
208//! impl<T> Bar<T> {
209//! fn new(t: T) -> impl PinInit<Self> {
210//! pin_init!(Self { t, x: 0 })
211//! }
212//! }
213//! ```
214//!
215//! This expands to the following code:
216//!
217//! ```rust,ignore
218//! impl<T> Bar<T> {
219//! fn new(t: T) -> impl PinInit<Self> {
220//! {
221//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
222//! // return type and shadow it later when we insert the arbitrary user code. That way
223//! // there will be no possibility of returning without `unsafe`.
224//! struct __InitOk;
225//! // Get the data about fields from the supplied type.
226//! // - the function is unsafe, hence the unsafe block
227//! // - we `use` the `HasPinData` trait in the block, it is only available in that
228//! // scope.
229//! let data = unsafe {
230//! use ::kernel::init::__internal::HasPinData;
231//! Self::__pin_data()
232//! };
233//! // Ensure that `data` really is of type `PinData` and help with type inference:
234//! let init = ::kernel::init::__internal::PinData::make_closure::<
235//! _,
236//! __InitOk,
237//! ::core::convert::Infallible,
238//! >(data, move |slot| {
239//! {
240//! // Shadow the structure so it cannot be used to return early. If a user
241//! // tries to write `return Ok(__InitOk)`, then they get a type error,
242//! // since that will refer to this struct instead of the one defined
243//! // above.
244//! struct __InitOk;
245//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
246//! {
247//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
248//! }
249//! // Since initialization could fail later (not in this case, since the
250//! // error type is `Infallible`) we will need to drop this field if there
251//! // is an error later. This `DropGuard` will drop the field when it gets
252//! // dropped and has not yet been forgotten.
253//! let __t_guard = unsafe {
254//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
255//! };
256//! // Expansion of `x: 0,`:
257//! // Since this can be an arbitrary expression we cannot place it inside
258//! // of the `unsafe` block, so we bind it here.
259//! {
260//! let x = 0;
261//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
262//! }
263//! // We again create a `DropGuard`.
264//! let __x_guard = unsafe {
265//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
266//! };
267//! // Since initialization has successfully completed, we can now forget
268//! // the guards. This is not `mem::forget`, since we only have
269//! // `&DropGuard`.
270//! ::core::mem::forget(__x_guard);
271//! ::core::mem::forget(__t_guard);
272//! // Here we use the type checker to ensure that every field has been
273//! // initialized exactly once, since this is `if false` it will never get
274//! // executed, but still type-checked.
275//! // Additionally we abuse `slot` to automatically infer the correct type
276//! // for the struct. This is also another check that every field is
277//! // accessible from this scope.
278//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
279//! let _ = || {
280//! unsafe {
281//! ::core::ptr::write(
282//! slot,
283//! Self {
284//! // We only care about typecheck finding every field
285//! // here, the expression does not matter, just conjure
286//! // one using `panic!()`:
287//! t: ::core::panic!(),
288//! x: ::core::panic!(),
289//! },
290//! );
291//! };
292//! };
293//! }
294//! // We leave the scope above and gain access to the previously shadowed
295//! // `__InitOk` that we need to return.
296//! Ok(__InitOk)
297//! });
298//! // Change the return type from `__InitOk` to `()`.
299//! let init = move |
300//! slot,
301//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
302//! init(slot).map(|__InitOk| ())
303//! };
304//! // Construct the initializer.
305//! let init = unsafe {
306//! ::kernel::init::pin_init_from_closure::<
307//! _,
308//! ::core::convert::Infallible,
309//! >(init)
310//! };
311//! init
312//! }
313//! }
314//! }
315//! ```
316//!
317//! ## `#[pin_data]` on `Foo`
318//!
319//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
320//! differences/new things in the expansion of the `Foo` definition:
321//!
322//! ```rust,ignore
323//! #[pin_data(PinnedDrop)]
324//! struct Foo {
325//! a: usize,
326//! #[pin]
327//! b: Bar<u32>,
328//! }
329//! ```
330//!
331//! This expands to the following code:
332//!
333//! ```rust,ignore
334//! struct Foo {
335//! a: usize,
336//! b: Bar<u32>,
337//! }
338//! const _: () = {
339//! struct __ThePinData {
340//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
341//! }
342//! impl ::core::clone::Clone for __ThePinData {
343//! fn clone(&self) -> Self {
344//! *self
345//! }
346//! }
347//! impl ::core::marker::Copy for __ThePinData {}
348//! #[allow(dead_code)]
349//! impl __ThePinData {
350//! unsafe fn b<E>(
351//! self,
352//! slot: *mut Bar<u32>,
353//! init: impl ::kernel::init::PinInit<Bar<u32>, E>,
354//! ) -> ::core::result::Result<(), E> {
355//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
356//! }
357//! unsafe fn a<E>(
358//! self,
359//! slot: *mut usize,
360//! init: impl ::kernel::init::Init<usize, E>,
361//! ) -> ::core::result::Result<(), E> {
362//! unsafe { ::kernel::init::Init::__init(init, slot) }
363//! }
364//! }
365//! unsafe impl ::kernel::init::__internal::HasPinData for Foo {
366//! type PinData = __ThePinData;
367//! unsafe fn __pin_data() -> Self::PinData {
368//! __ThePinData {
369//! __phantom: ::core::marker::PhantomData,
370//! }
371//! }
372//! }
373//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
374//! type Datee = Foo;
375//! }
376//! #[allow(dead_code)]
377//! struct __Unpin<'__pin> {
378//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
379//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
380//! b: Bar<u32>,
381//! }
382//! #[doc(hidden)]
383//! impl<'__pin> ::core::marker::Unpin for Foo
384//! where
385//! __Unpin<'__pin>: ::core::marker::Unpin,
386//! {}
387//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
388//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
389//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
390//! impl ::core::ops::Drop for Foo {
391//! fn drop(&mut self) {
392//! // Since we are getting dropped, no one else has a reference to `self` and thus we
393//! // can assume that we never move.
394//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
395//! // Create the unsafe token that proves that we are inside of a destructor, this
396//! // type is only allowed to be created in a destructor.
397//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
398//! ::kernel::init::PinnedDrop::drop(pinned, token);
399//! }
400//! }
401//! };
402//! ```
403//!
404//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
405//!
406//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
407//! extra parameter that should not be used at all. The macro hides that parameter.
408//!
409//! Here is the `PinnedDrop` impl for `Foo`:
410//!
411//! ```rust,ignore
412//! #[pinned_drop]
413//! impl PinnedDrop for Foo {
414//! fn drop(self: Pin<&mut Self>) {
415//! pr_info!("{self:p} is getting dropped.\n");
416//! }
417//! }
418//! ```
419//!
420//! This expands to the following code:
421//!
422//! ```rust,ignore
423//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
424//! unsafe impl ::kernel::init::PinnedDrop for Foo {
425//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
426//! pr_info!("{self:p} is getting dropped.\n");
427//! }
428//! }
429//! ```
430//!
431//! ## `pin_init!` on `Foo`
432//!
433//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
434//! of `pin_init!` on `Foo`:
435//!
436//! ```rust,ignore
437//! let a = 42;
438//! let initializer = pin_init!(Foo {
439//! a,
440//! b <- Bar::new(36),
441//! });
442//! ```
443//!
444//! This expands to the following code:
445//!
446//! ```rust,ignore
447//! let a = 42;
448//! let initializer = {
449//! struct __InitOk;
450//! let data = unsafe {
451//! use ::kernel::init::__internal::HasPinData;
452//! Foo::__pin_data()
453//! };
454//! let init = ::kernel::init::__internal::PinData::make_closure::<
455//! _,
456//! __InitOk,
457//! ::core::convert::Infallible,
458//! >(data, move |slot| {
459//! {
460//! struct __InitOk;
461//! {
462//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
463//! }
464//! let __a_guard = unsafe {
465//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
466//! };
467//! let init = Bar::new(36);
468//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
469//! let __b_guard = unsafe {
470//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
471//! };
472//! ::core::mem::forget(__b_guard);
473//! ::core::mem::forget(__a_guard);
474//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
475//! let _ = || {
476//! unsafe {
477//! ::core::ptr::write(
478//! slot,
479//! Foo {
480//! a: ::core::panic!(),
481//! b: ::core::panic!(),
482//! },
483//! );
484//! };
485//! };
486//! }
487//! Ok(__InitOk)
488//! });
489//! let init = move |
490//! slot,
491//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
492//! init(slot).map(|__InitOk| ())
493//! };
494//! let init = unsafe {
495//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
496//! };
497//! init
498//! };
499//! ```
500
501/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
502///
503/// See [`PinnedDrop`] for more information.
504#[doc(hidden)]
505#[macro_export]
506macro_rules! __pinned_drop {
507 (
508 @impl_sig($($impl_sig:tt)*),
509 @impl_body(
510 $(#[$($attr:tt)*])*
511 fn drop($($sig:tt)*) {
512 $($inner:tt)*
513 }
514 ),
515 ) => {
516 // SAFETY: TODO.
517 unsafe $($impl_sig)* {
518 // Inherit all attributes and the type/ident tokens for the signature.
519 $(#[$($attr)*])*
520 fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
521 $($inner)*
522 }
523 }
524 }
525}
526
527/// This macro first parses the struct definition such that it separates pinned and not pinned
528/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
529#[doc(hidden)]
530#[macro_export]
531macro_rules! __pin_data {
532 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
533 (parse_input:
534 @args($($pinned_drop:ident)?),
535 @sig(
536 $(#[$($struct_attr:tt)*])*
537 $vis:vis struct $name:ident
538 $(where $($whr:tt)*)?
539 ),
540 @impl_generics($($impl_generics:tt)*),
541 @ty_generics($($ty_generics:tt)*),
542 @decl_generics($($decl_generics:tt)*),
543 @body({ $($fields:tt)* }),
544 ) => {
545 // We now use token munching to iterate through all of the fields. While doing this we
546 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
547 // wants these to be structurally pinned. The rest of the fields are the
548 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
549 // order to declare the struct.
550 //
551 // In this call we also put some explaining comments for the parameters.
552 $crate::__pin_data!(find_pinned_fields:
553 // Attributes on the struct itself, these will just be propagated to be put onto the
554 // struct definition.
555 @struct_attrs($(#[$($struct_attr)*])*),
556 // The visibility of the struct.
557 @vis($vis),
558 // The name of the struct.
559 @name($name),
560 // The 'impl generics', the generics that will need to be specified on the struct inside
561 // of an `impl<$ty_generics>` block.
562 @impl_generics($($impl_generics)*),
563 // The 'ty generics', the generics that will need to be specified on the impl blocks.
564 @ty_generics($($ty_generics)*),
565 // The 'decl generics', the generics that need to be specified on the struct
566 // definition.
567 @decl_generics($($decl_generics)*),
568 // The where clause of any impl block and the declaration.
569 @where($($($whr)*)?),
570 // The remaining fields tokens that need to be processed.
571 // We add a `,` at the end to ensure correct parsing.
572 @fields_munch($($fields)* ,),
573 // The pinned fields.
574 @pinned(),
575 // The not pinned fields.
576 @not_pinned(),
577 // All fields.
578 @fields(),
579 // The accumulator containing all attributes already parsed.
580 @accum(),
581 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
582 @is_pinned(),
583 // The proc-macro argument, this should be `PinnedDrop` or ``.
584 @pinned_drop($($pinned_drop)?),
585 );
586 };
587 (find_pinned_fields:
588 @struct_attrs($($struct_attrs:tt)*),
589 @vis($vis:vis),
590 @name($name:ident),
591 @impl_generics($($impl_generics:tt)*),
592 @ty_generics($($ty_generics:tt)*),
593 @decl_generics($($decl_generics:tt)*),
594 @where($($whr:tt)*),
595 // We found a PhantomPinned field, this should generally be pinned!
596 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
597 @pinned($($pinned:tt)*),
598 @not_pinned($($not_pinned:tt)*),
599 @fields($($fields:tt)*),
600 @accum($($accum:tt)*),
601 // This field is not pinned.
602 @is_pinned(),
603 @pinned_drop($($pinned_drop:ident)?),
604 ) => {
605 ::core::compile_error!(concat!(
606 "The field `",
607 stringify!($field),
608 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
609 ));
610 $crate::__pin_data!(find_pinned_fields:
611 @struct_attrs($($struct_attrs)*),
612 @vis($vis),
613 @name($name),
614 @impl_generics($($impl_generics)*),
615 @ty_generics($($ty_generics)*),
616 @decl_generics($($decl_generics)*),
617 @where($($whr)*),
618 @fields_munch($($rest)*),
619 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
620 @not_pinned($($not_pinned)*),
621 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
622 @accum(),
623 @is_pinned(),
624 @pinned_drop($($pinned_drop)?),
625 );
626 };
627 (find_pinned_fields:
628 @struct_attrs($($struct_attrs:tt)*),
629 @vis($vis:vis),
630 @name($name:ident),
631 @impl_generics($($impl_generics:tt)*),
632 @ty_generics($($ty_generics:tt)*),
633 @decl_generics($($decl_generics:tt)*),
634 @where($($whr:tt)*),
635 // We reached the field declaration.
636 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
637 @pinned($($pinned:tt)*),
638 @not_pinned($($not_pinned:tt)*),
639 @fields($($fields:tt)*),
640 @accum($($accum:tt)*),
641 // This field is pinned.
642 @is_pinned(yes),
643 @pinned_drop($($pinned_drop:ident)?),
644 ) => {
645 $crate::__pin_data!(find_pinned_fields:
646 @struct_attrs($($struct_attrs)*),
647 @vis($vis),
648 @name($name),
649 @impl_generics($($impl_generics)*),
650 @ty_generics($($ty_generics)*),
651 @decl_generics($($decl_generics)*),
652 @where($($whr)*),
653 @fields_munch($($rest)*),
654 @pinned($($pinned)* $($accum)* $field: $type,),
655 @not_pinned($($not_pinned)*),
656 @fields($($fields)* $($accum)* $field: $type,),
657 @accum(),
658 @is_pinned(),
659 @pinned_drop($($pinned_drop)?),
660 );
661 };
662 (find_pinned_fields:
663 @struct_attrs($($struct_attrs:tt)*),
664 @vis($vis:vis),
665 @name($name:ident),
666 @impl_generics($($impl_generics:tt)*),
667 @ty_generics($($ty_generics:tt)*),
668 @decl_generics($($decl_generics:tt)*),
669 @where($($whr:tt)*),
670 // We reached the field declaration.
671 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
672 @pinned($($pinned:tt)*),
673 @not_pinned($($not_pinned:tt)*),
674 @fields($($fields:tt)*),
675 @accum($($accum:tt)*),
676 // This field is not pinned.
677 @is_pinned(),
678 @pinned_drop($($pinned_drop:ident)?),
679 ) => {
680 $crate::__pin_data!(find_pinned_fields:
681 @struct_attrs($($struct_attrs)*),
682 @vis($vis),
683 @name($name),
684 @impl_generics($($impl_generics)*),
685 @ty_generics($($ty_generics)*),
686 @decl_generics($($decl_generics)*),
687 @where($($whr)*),
688 @fields_munch($($rest)*),
689 @pinned($($pinned)*),
690 @not_pinned($($not_pinned)* $($accum)* $field: $type,),
691 @fields($($fields)* $($accum)* $field: $type,),
692 @accum(),
693 @is_pinned(),
694 @pinned_drop($($pinned_drop)?),
695 );
696 };
697 (find_pinned_fields:
698 @struct_attrs($($struct_attrs:tt)*),
699 @vis($vis:vis),
700 @name($name:ident),
701 @impl_generics($($impl_generics:tt)*),
702 @ty_generics($($ty_generics:tt)*),
703 @decl_generics($($decl_generics:tt)*),
704 @where($($whr:tt)*),
705 // We found the `#[pin]` attr.
706 @fields_munch(#[pin] $($rest:tt)*),
707 @pinned($($pinned:tt)*),
708 @not_pinned($($not_pinned:tt)*),
709 @fields($($fields:tt)*),
710 @accum($($accum:tt)*),
711 @is_pinned($($is_pinned:ident)?),
712 @pinned_drop($($pinned_drop:ident)?),
713 ) => {
714 $crate::__pin_data!(find_pinned_fields:
715 @struct_attrs($($struct_attrs)*),
716 @vis($vis),
717 @name($name),
718 @impl_generics($($impl_generics)*),
719 @ty_generics($($ty_generics)*),
720 @decl_generics($($decl_generics)*),
721 @where($($whr)*),
722 @fields_munch($($rest)*),
723 // We do not include `#[pin]` in the list of attributes, since it is not actually an
724 // attribute that is defined somewhere.
725 @pinned($($pinned)*),
726 @not_pinned($($not_pinned)*),
727 @fields($($fields)*),
728 @accum($($accum)*),
729 // Set this to `yes`.
730 @is_pinned(yes),
731 @pinned_drop($($pinned_drop)?),
732 );
733 };
734 (find_pinned_fields:
735 @struct_attrs($($struct_attrs:tt)*),
736 @vis($vis:vis),
737 @name($name:ident),
738 @impl_generics($($impl_generics:tt)*),
739 @ty_generics($($ty_generics:tt)*),
740 @decl_generics($($decl_generics:tt)*),
741 @where($($whr:tt)*),
742 // We reached the field declaration with visibility, for simplicity we only munch the
743 // visibility and put it into `$accum`.
744 @fields_munch($fvis:vis $field:ident $($rest:tt)*),
745 @pinned($($pinned:tt)*),
746 @not_pinned($($not_pinned:tt)*),
747 @fields($($fields:tt)*),
748 @accum($($accum:tt)*),
749 @is_pinned($($is_pinned:ident)?),
750 @pinned_drop($($pinned_drop:ident)?),
751 ) => {
752 $crate::__pin_data!(find_pinned_fields:
753 @struct_attrs($($struct_attrs)*),
754 @vis($vis),
755 @name($name),
756 @impl_generics($($impl_generics)*),
757 @ty_generics($($ty_generics)*),
758 @decl_generics($($decl_generics)*),
759 @where($($whr)*),
760 @fields_munch($field $($rest)*),
761 @pinned($($pinned)*),
762 @not_pinned($($not_pinned)*),
763 @fields($($fields)*),
764 @accum($($accum)* $fvis),
765 @is_pinned($($is_pinned)?),
766 @pinned_drop($($pinned_drop)?),
767 );
768 };
769 (find_pinned_fields:
770 @struct_attrs($($struct_attrs:tt)*),
771 @vis($vis:vis),
772 @name($name:ident),
773 @impl_generics($($impl_generics:tt)*),
774 @ty_generics($($ty_generics:tt)*),
775 @decl_generics($($decl_generics:tt)*),
776 @where($($whr:tt)*),
777 // Some other attribute, just put it into `$accum`.
778 @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
779 @pinned($($pinned:tt)*),
780 @not_pinned($($not_pinned:tt)*),
781 @fields($($fields:tt)*),
782 @accum($($accum:tt)*),
783 @is_pinned($($is_pinned:ident)?),
784 @pinned_drop($($pinned_drop:ident)?),
785 ) => {
786 $crate::__pin_data!(find_pinned_fields:
787 @struct_attrs($($struct_attrs)*),
788 @vis($vis),
789 @name($name),
790 @impl_generics($($impl_generics)*),
791 @ty_generics($($ty_generics)*),
792 @decl_generics($($decl_generics)*),
793 @where($($whr)*),
794 @fields_munch($($rest)*),
795 @pinned($($pinned)*),
796 @not_pinned($($not_pinned)*),
797 @fields($($fields)*),
798 @accum($($accum)* #[$($attr)*]),
799 @is_pinned($($is_pinned)?),
800 @pinned_drop($($pinned_drop)?),
801 );
802 };
803 (find_pinned_fields:
804 @struct_attrs($($struct_attrs:tt)*),
805 @vis($vis:vis),
806 @name($name:ident),
807 @impl_generics($($impl_generics:tt)*),
808 @ty_generics($($ty_generics:tt)*),
809 @decl_generics($($decl_generics:tt)*),
810 @where($($whr:tt)*),
811 // We reached the end of the fields, plus an optional additional comma, since we added one
812 // before and the user is also allowed to put a trailing comma.
813 @fields_munch($(,)?),
814 @pinned($($pinned:tt)*),
815 @not_pinned($($not_pinned:tt)*),
816 @fields($($fields:tt)*),
817 @accum(),
818 @is_pinned(),
819 @pinned_drop($($pinned_drop:ident)?),
820 ) => {
821 // Declare the struct with all fields in the correct order.
822 $($struct_attrs)*
823 $vis struct $name <$($decl_generics)*>
824 where $($whr)*
825 {
826 $($fields)*
827 }
828
829 // We put the rest into this const item, because it then will not be accessible to anything
830 // outside.
831 const _: () = {
832 // We declare this struct which will host all of the projection function for our type.
833 // it will be invariant over all generic parameters which are inherited from the
834 // struct.
835 $vis struct __ThePinData<$($impl_generics)*>
836 where $($whr)*
837 {
838 __phantom: ::core::marker::PhantomData<
839 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
840 >,
841 }
842
843 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
844 where $($whr)*
845 {
846 fn clone(&self) -> Self { *self }
847 }
848
849 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
850 where $($whr)*
851 {}
852
853 // Make all projection functions.
854 $crate::__pin_data!(make_pin_data:
855 @pin_data(__ThePinData),
856 @impl_generics($($impl_generics)*),
857 @ty_generics($($ty_generics)*),
858 @where($($whr)*),
859 @pinned($($pinned)*),
860 @not_pinned($($not_pinned)*),
861 );
862
863 // SAFETY: We have added the correct projection functions above to `__ThePinData` and
864 // we also use the least restrictive generics possible.
865 unsafe impl<$($impl_generics)*>
866 $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
867 where $($whr)*
868 {
869 type PinData = __ThePinData<$($ty_generics)*>;
870
871 unsafe fn __pin_data() -> Self::PinData {
872 __ThePinData { __phantom: ::core::marker::PhantomData }
873 }
874 }
875
876 // SAFETY: TODO.
877 unsafe impl<$($impl_generics)*>
878 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
879 where $($whr)*
880 {
881 type Datee = $name<$($ty_generics)*>;
882 }
883
884 // This struct will be used for the unpin analysis. Since only structurally pinned
885 // fields are relevant whether the struct should implement `Unpin`.
886 #[allow(dead_code)]
887 struct __Unpin <'__pin, $($impl_generics)*>
888 where $($whr)*
889 {
890 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
891 __phantom: ::core::marker::PhantomData<
892 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
893 >,
894 // Only the pinned fields.
895 $($pinned)*
896 }
897
898 #[doc(hidden)]
899 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
900 where
901 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
902 $($whr)*
903 {}
904
905 // We need to disallow normal `Drop` implementation, the exact behavior depends on
906 // whether `PinnedDrop` was specified as the parameter.
907 $crate::__pin_data!(drop_prevention:
908 @name($name),
909 @impl_generics($($impl_generics)*),
910 @ty_generics($($ty_generics)*),
911 @where($($whr)*),
912 @pinned_drop($($pinned_drop)?),
913 );
914 };
915 };
916 // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
917 (drop_prevention:
918 @name($name:ident),
919 @impl_generics($($impl_generics:tt)*),
920 @ty_generics($($ty_generics:tt)*),
921 @where($($whr:tt)*),
922 @pinned_drop(),
923 ) => {
924 // We prevent this by creating a trait that will be implemented for all types implementing
925 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
926 // if it also implements `Drop`
927 #[allow(dead_code)]
928 trait MustNotImplDrop {}
929 #[expect(drop_bounds)]
930 impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
931 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
932 where $($whr)* {}
933 // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
934 // They might implement `PinnedDrop` correctly for the struct, but forget to give
935 // `PinnedDrop` as the parameter to `#[pin_data]`.
936 #[allow(dead_code)]
937 #[expect(non_camel_case_types)]
938 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
939 impl<T: $crate::init::PinnedDrop>
940 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
941 impl<$($impl_generics)*>
942 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
943 where $($whr)* {}
944 };
945 // When `PinnedDrop` was specified we just implement `Drop` and delegate.
946 (drop_prevention:
947 @name($name:ident),
948 @impl_generics($($impl_generics:tt)*),
949 @ty_generics($($ty_generics:tt)*),
950 @where($($whr:tt)*),
951 @pinned_drop(PinnedDrop),
952 ) => {
953 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
954 where $($whr)*
955 {
956 fn drop(&mut self) {
957 // SAFETY: Since this is a destructor, `self` will not move after this function
958 // terminates, since it is inaccessible.
959 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
960 // SAFETY: Since this is a drop function, we can create this token to call the
961 // pinned destructor of this type.
962 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
963 $crate::init::PinnedDrop::drop(pinned, token);
964 }
965 }
966 };
967 // If some other parameter was specified, we emit a readable error.
968 (drop_prevention:
969 @name($name:ident),
970 @impl_generics($($impl_generics:tt)*),
971 @ty_generics($($ty_generics:tt)*),
972 @where($($whr:tt)*),
973 @pinned_drop($($rest:tt)*),
974 ) => {
975 compile_error!(
976 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
977 stringify!($($rest)*),
978 );
979 };
980 (make_pin_data:
981 @pin_data($pin_data:ident),
982 @impl_generics($($impl_generics:tt)*),
983 @ty_generics($($ty_generics:tt)*),
984 @where($($whr:tt)*),
985 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
986 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
987 ) => {
988 $crate::macros::paste! {
989 // For every field, we create a projection function according to its projection type. If a
990 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
991 // structurally pinned, then it can be initialized via `Init`.
992 //
993 // The functions are `unsafe` to prevent accidentally calling them.
994 #[allow(dead_code, non_snake_case)]
995 #[expect(clippy::missing_safety_doc)]
996 impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
997 where $($whr)*
998 {
999 $(
1000 $(#[$($p_attr)*])*
1001 $pvis unsafe fn $p_field<E>(
1002 self,
1003 slot: *mut $p_type,
1004 init: impl $crate::init::PinInit<$p_type, E>,
1005 ) -> ::core::result::Result<(), E> {
1006 // SAFETY: TODO.
1007 unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
1008 }
1009
1010 $(#[$($p_attr)*])*
1011 $pvis unsafe fn [<__project_ $p_field>]<'__slot>(
1012 self,
1013 slot: &'__slot mut $p_type,
1014 ) -> ::core::pin::Pin<&'__slot mut $p_type> {
1015 // SAFETY: TODO.
1016 unsafe { ::core::pin::Pin::new_unchecked(slot) }
1017 }
1018 )*
1019 $(
1020 $(#[$($attr)*])*
1021 $fvis unsafe fn $field<E>(
1022 self,
1023 slot: *mut $type,
1024 init: impl $crate::init::Init<$type, E>,
1025 ) -> ::core::result::Result<(), E> {
1026 // SAFETY: TODO.
1027 unsafe { $crate::init::Init::__init(init, slot) }
1028 }
1029
1030 $(#[$($attr)*])*
1031 $fvis unsafe fn [<__project_ $field>]<'__slot>(
1032 self,
1033 slot: &'__slot mut $type,
1034 ) -> &'__slot mut $type {
1035 slot
1036 }
1037 )*
1038 }
1039 }
1040 };
1041}
1042
1043/// The internal init macro. Do not call manually!
1044///
1045/// This is called by the `{try_}{pin_}init!` macros with various inputs.
1046///
1047/// This macro has multiple internal call configurations, these are always the very first ident:
1048/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
1049/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
1050/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
1051/// - `make_initializer`: recursively create the struct initializer that guarantees that every
1052/// field has been initialized exactly once.
1053#[doc(hidden)]
1054#[macro_export]
1055macro_rules! __init_internal {
1056 (
1057 @this($($this:ident)?),
1058 @typ($t:path),
1059 @fields($($fields:tt)*),
1060 @error($err:ty),
1061 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1062 // case.
1063 @data($data:ident, $($use_data:ident)?),
1064 // `HasPinData` or `HasInitData`.
1065 @has_data($has_data:ident, $get_data:ident),
1066 // `pin_init_from_closure` or `init_from_closure`.
1067 @construct_closure($construct_closure:ident),
1068 @munch_fields(),
1069 ) => {
1070 $crate::__init_internal!(with_update_parsed:
1071 @this($($this)?),
1072 @typ($t),
1073 @fields($($fields)*),
1074 @error($err),
1075 @data($data, $($use_data)?),
1076 @has_data($has_data, $get_data),
1077 @construct_closure($construct_closure),
1078 @zeroed(), // Nothing means default behavior.
1079 )
1080 };
1081 (
1082 @this($($this:ident)?),
1083 @typ($t:path),
1084 @fields($($fields:tt)*),
1085 @error($err:ty),
1086 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1087 // case.
1088 @data($data:ident, $($use_data:ident)?),
1089 // `HasPinData` or `HasInitData`.
1090 @has_data($has_data:ident, $get_data:ident),
1091 // `pin_init_from_closure` or `init_from_closure`.
1092 @construct_closure($construct_closure:ident),
1093 @munch_fields(..Zeroable::zeroed()),
1094 ) => {
1095 $crate::__init_internal!(with_update_parsed:
1096 @this($($this)?),
1097 @typ($t),
1098 @fields($($fields)*),
1099 @error($err),
1100 @data($data, $($use_data)?),
1101 @has_data($has_data, $get_data),
1102 @construct_closure($construct_closure),
1103 @zeroed(()), // `()` means zero all fields not mentioned.
1104 )
1105 };
1106 (
1107 @this($($this:ident)?),
1108 @typ($t:path),
1109 @fields($($fields:tt)*),
1110 @error($err:ty),
1111 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1112 // case.
1113 @data($data:ident, $($use_data:ident)?),
1114 // `HasPinData` or `HasInitData`.
1115 @has_data($has_data:ident, $get_data:ident),
1116 // `pin_init_from_closure` or `init_from_closure`.
1117 @construct_closure($construct_closure:ident),
1118 @munch_fields($ignore:tt $($rest:tt)*),
1119 ) => {
1120 $crate::__init_internal!(
1121 @this($($this)?),
1122 @typ($t),
1123 @fields($($fields)*),
1124 @error($err),
1125 @data($data, $($use_data)?),
1126 @has_data($has_data, $get_data),
1127 @construct_closure($construct_closure),
1128 @munch_fields($($rest)*),
1129 )
1130 };
1131 (with_update_parsed:
1132 @this($($this:ident)?),
1133 @typ($t:path),
1134 @fields($($fields:tt)*),
1135 @error($err:ty),
1136 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1137 // case.
1138 @data($data:ident, $($use_data:ident)?),
1139 // `HasPinData` or `HasInitData`.
1140 @has_data($has_data:ident, $get_data:ident),
1141 // `pin_init_from_closure` or `init_from_closure`.
1142 @construct_closure($construct_closure:ident),
1143 @zeroed($($init_zeroed:expr)?),
1144 ) => {{
1145 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
1146 // type and shadow it later when we insert the arbitrary user code. That way there will be
1147 // no possibility of returning without `unsafe`.
1148 struct __InitOk;
1149 // Get the data about fields from the supplied type.
1150 //
1151 // SAFETY: TODO.
1152 let data = unsafe {
1153 use $crate::init::__internal::$has_data;
1154 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1155 // information that is associated to already parsed fragments, so a path fragment
1156 // cannot be used in this position. Doing the retokenization results in valid rust
1157 // code.
1158 ::kernel::macros::paste!($t::$get_data())
1159 };
1160 // Ensure that `data` really is of type `$data` and help with type inference:
1161 let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
1162 data,
1163 move |slot| {
1164 {
1165 // Shadow the structure so it cannot be used to return early.
1166 struct __InitOk;
1167 // If `$init_zeroed` is present we should zero the slot now and not emit an
1168 // error when fields are missing (since they will be zeroed). We also have to
1169 // check that the type actually implements `Zeroable`.
1170 $({
1171 fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
1172 // Ensure that the struct is indeed `Zeroable`.
1173 assert_zeroable(slot);
1174 // SAFETY: The type implements `Zeroable` by the check above.
1175 unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
1176 $init_zeroed // This will be `()` if set.
1177 })?
1178 // Create the `this` so it can be referenced by the user inside of the
1179 // expressions creating the individual fields.
1180 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
1181 // Initialize every field.
1182 $crate::__init_internal!(init_slot($($use_data)?):
1183 @data(data),
1184 @slot(slot),
1185 @guards(),
1186 @munch_fields($($fields)*,),
1187 );
1188 // We use unreachable code to ensure that all fields have been mentioned exactly
1189 // once, this struct initializer will still be type-checked and complain with a
1190 // very natural error message if a field is forgotten/mentioned more than once.
1191 #[allow(unreachable_code, clippy::diverging_sub_expression)]
1192 let _ = || {
1193 $crate::__init_internal!(make_initializer:
1194 @slot(slot),
1195 @type_name($t),
1196 @munch_fields($($fields)*,),
1197 @acc(),
1198 );
1199 };
1200 }
1201 Ok(__InitOk)
1202 }
1203 );
1204 let init = move |slot| -> ::core::result::Result<(), $err> {
1205 init(slot).map(|__InitOk| ())
1206 };
1207 // SAFETY: TODO.
1208 let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
1209 init
1210 }};
1211 (init_slot($($use_data:ident)?):
1212 @data($data:ident),
1213 @slot($slot:ident),
1214 @guards($($guards:ident,)*),
1215 @munch_fields($(..Zeroable::zeroed())? $(,)?),
1216 ) => {
1217 // Endpoint of munching, no fields are left. If execution reaches this point, all fields
1218 // have been initialized. Therefore we can now dismiss the guards by forgetting them.
1219 $(::core::mem::forget($guards);)*
1220 };
1221 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
1222 @data($data:ident),
1223 @slot($slot:ident),
1224 @guards($($guards:ident,)*),
1225 // In-place initialization syntax.
1226 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1227 ) => {
1228 let init = $val;
1229 // Call the initializer.
1230 //
1231 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1232 // return when an error/panic occurs.
1233 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
1234 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
1235 // NOTE: this ensures that the initialized field is properly aligned.
1236 // Unaligned fields will cause the compiler to emit E0793. We do not support
1237 // unaligned fields since `Init::__init` requires an aligned pointer; the call to
1238 // `ptr::write` below has the same requirement.
1239 // SAFETY: the field has been initialized.
1240 let _ = unsafe { &mut (*$slot).$field };
1241
1242 // Create the drop guard:
1243 //
1244 // We rely on macro hygiene to make it impossible for users to access this local variable.
1245 // We use `paste!` to create new hygiene for `$field`.
1246 ::kernel::macros::paste! {
1247 // SAFETY:
1248 // - `addr_of_mut!((*$slot).$field)` is valid.
1249 // - `(*$slot).$field` has been initialized above.
1250 // - We only need the ownership to the pointee back when initialization has
1251 // succeeded, where we `forget` the guard.
1252 let mut [< __ $field _guard >] = unsafe {
1253 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1254 };
1255
1256 // NOTE: The reference is derived from the guard so that it only lives as long as
1257 // the guard does and cannot escape the scope.
1258 #[allow(unused_variables, unused_assignments)]
1259 // SAFETY: the project function does the correct field projection.
1260 let $field = unsafe { $data.[< __project_ $field >]([< __ $field _guard >].let_binding()) };
1261
1262 $crate::__init_internal!(init_slot($use_data):
1263 @data($data),
1264 @slot($slot),
1265 @guards([< __ $field _guard >], $($guards,)*),
1266 @munch_fields($($rest)*),
1267 );
1268 }
1269 };
1270 (init_slot(): // No `use_data`, so we use `Init::__init` directly.
1271 @data($data:ident),
1272 @slot($slot:ident),
1273 @guards($($guards:ident,)*),
1274 // In-place initialization syntax.
1275 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1276 ) => {
1277 let init = $val;
1278 // Call the initializer.
1279 //
1280 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1281 // return when an error/panic occurs.
1282 unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
1283
1284 // NOTE: this ensures that the initialized field is properly aligned.
1285 // Unaligned fields will cause the compiler to emit E0793. We do not support
1286 // unaligned fields since `Init::__init` requires an aligned pointer; the call to
1287 // `ptr::write` below has the same requirement.
1288 // SAFETY: the field has been initialized.
1289 let _ = unsafe { &mut (*$slot).$field };
1290
1291 // Create the drop guard:
1292 //
1293 // We rely on macro hygiene to make it impossible for users to access this local variable.
1294 // We use `paste!` to create new hygiene for `$field`.
1295 ::kernel::macros::paste! {
1296 // SAFETY:
1297 // - `addr_of_mut!((*$slot).$field)` is valid.
1298 // - `(*$slot).$field` has been initialized above.
1299 // - We only need the ownership to the pointee back when initialization has
1300 // succeeded, where we `forget` the guard.
1301 let mut [< __ $field _guard >] = unsafe {
1302 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1303 };
1304
1305 #[allow(unused_variables, unused_assignments)]
1306 let $field = [< __ $field _guard >].let_binding();
1307
1308 $crate::__init_internal!(init_slot():
1309 @data($data),
1310 @slot($slot),
1311 @guards([< __ $field _guard >], $($guards,)*),
1312 @munch_fields($($rest)*),
1313 );
1314 }
1315 };
1316 (init_slot(): // No `use_data`, so all fields are not structurally pinned
1317 @data($data:ident),
1318 @slot($slot:ident),
1319 @guards($($guards:ident,)*),
1320 // Init by-value.
1321 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1322 ) => {
1323 {
1324 $(let $field = $val;)?
1325 // Initialize the field.
1326 //
1327 // SAFETY: The memory at `slot` is uninitialized.
1328 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1329 }
1330
1331 // NOTE: this ensures that the initialized field is properly aligned.
1332 // Unaligned fields will cause the compiler to emit E0793. We do not support
1333 // unaligned fields since `Init::__init` requires an aligned pointer; the call to
1334 // `ptr::write` below has the same requirement.
1335 // SAFETY: the field has been initialized.
1336 let _ = unsafe { &mut (*$slot).$field };
1337
1338 // Create the drop guard:
1339 //
1340 // We rely on macro hygiene to make it impossible for users to access this local variable.
1341 // We use `paste!` to create new hygiene for `$field`.
1342 ::kernel::macros::paste! {
1343 // SAFETY:
1344 // - `addr_of_mut!((*$slot).$field)` is valid.
1345 // - `(*$slot).$field` has been initialized above.
1346 // - We only need the ownership to the pointee back when initialization has
1347 // succeeded, where we `forget` the guard.
1348 let mut [< __ $field _guard >] = unsafe {
1349 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1350 };
1351
1352 #[allow(unused_variables, unused_assignments)]
1353 let $field = [< __ $field _guard >].let_binding();
1354
1355 $crate::__init_internal!(init_slot():
1356 @data($data),
1357 @slot($slot),
1358 @guards([< __ $field _guard >], $($guards,)*),
1359 @munch_fields($($rest)*),
1360 );
1361 }
1362 };
1363 (init_slot($use_data:ident):
1364 @data($data:ident),
1365 @slot($slot:ident),
1366 @guards($($guards:ident,)*),
1367 // Init by-value.
1368 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1369 ) => {
1370 {
1371 $(let $field = $val;)?
1372 // Initialize the field.
1373 //
1374 // SAFETY: The memory at `slot` is uninitialized.
1375 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1376 }
1377 // NOTE: this ensures that the initialized field is properly aligned.
1378 // Unaligned fields will cause the compiler to emit E0793. We do not support
1379 // unaligned fields since `Init::__init` requires an aligned pointer; the call to
1380 // `ptr::write` below has the same requirement.
1381 // SAFETY: the field has been initialized.
1382 let _ = unsafe { &mut (*$slot).$field };
1383
1384 // Create the drop guard:
1385 //
1386 // We rely on macro hygiene to make it impossible for users to access this local variable.
1387 // We use `paste!` to create new hygiene for `$field`.
1388 $crate::macros::paste! {
1389 // SAFETY:
1390 // - `addr_of_mut!((*$slot).$field)` is valid.
1391 // - `(*$slot).$field` has been initialized above.
1392 // - We only need the ownership to the pointee back when initialization has
1393 // succeeded, where we `forget` the guard.
1394 let mut [< __ $field _guard >] = unsafe {
1395 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1396 };
1397
1398 // NOTE: The reference is derived from the guard so that it only lives as long as
1399 // the guard does and cannot escape the scope.
1400 #[allow(unused_variables, unused_assignments)]
1401 // SAFETY: the project function does the correct field projection.
1402 let $field = unsafe { $data.[< __project_ $field >]([< __ $field _guard >].let_binding()) };
1403
1404 $crate::__init_internal!(init_slot($use_data):
1405 @data($data),
1406 @slot($slot),
1407 @guards([< __ $field _guard >], $($guards,)*),
1408 @munch_fields($($rest)*),
1409 );
1410 }
1411 };
1412 (make_initializer:
1413 @slot($slot:ident),
1414 @type_name($t:path),
1415 @munch_fields(..Zeroable::zeroed() $(,)?),
1416 @acc($($acc:tt)*),
1417 ) => {
1418 // Endpoint, nothing more to munch, create the initializer. Since the users specified
1419 // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
1420 // not been overwritten are thus zero and initialized. We still check that all fields are
1421 // actually accessible by using the struct update syntax ourselves.
1422 // We are inside of a closure that is never executed and thus we can abuse `slot` to
1423 // get the correct type inference here:
1424 #[allow(unused_assignments)]
1425 unsafe {
1426 let mut zeroed = ::core::mem::zeroed();
1427 // We have to use type inference here to make zeroed have the correct type. This does
1428 // not get executed, so it has no effect.
1429 ::core::ptr::write($slot, zeroed);
1430 zeroed = ::core::mem::zeroed();
1431 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1432 // information that is associated to already parsed fragments, so a path fragment
1433 // cannot be used in this position. Doing the retokenization results in valid rust
1434 // code.
1435 ::kernel::macros::paste!(
1436 ::core::ptr::write($slot, $t {
1437 $($acc)*
1438 ..zeroed
1439 });
1440 );
1441 }
1442 };
1443 (make_initializer:
1444 @slot($slot:ident),
1445 @type_name($t:path),
1446 @munch_fields($(,)?),
1447 @acc($($acc:tt)*),
1448 ) => {
1449 // Endpoint, nothing more to munch, create the initializer.
1450 // Since we are in the closure that is never called, this will never get executed.
1451 // We abuse `slot` to get the correct type inference here:
1452 //
1453 // SAFETY: TODO.
1454 unsafe {
1455 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1456 // information that is associated to already parsed fragments, so a path fragment
1457 // cannot be used in this position. Doing the retokenization results in valid rust
1458 // code.
1459 ::kernel::macros::paste!(
1460 ::core::ptr::write($slot, $t {
1461 $($acc)*
1462 });
1463 );
1464 }
1465 };
1466 (make_initializer:
1467 @slot($slot:ident),
1468 @type_name($t:path),
1469 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1470 @acc($($acc:tt)*),
1471 ) => {
1472 $crate::__init_internal!(make_initializer:
1473 @slot($slot),
1474 @type_name($t),
1475 @munch_fields($($rest)*),
1476 @acc($($acc)* $field: ::core::panic!(),),
1477 );
1478 };
1479 (make_initializer:
1480 @slot($slot:ident),
1481 @type_name($t:path),
1482 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1483 @acc($($acc:tt)*),
1484 ) => {
1485 $crate::__init_internal!(make_initializer:
1486 @slot($slot),
1487 @type_name($t),
1488 @munch_fields($($rest)*),
1489 @acc($($acc)* $field: ::core::panic!(),),
1490 );
1491 };
1492}
1493
1494#[doc(hidden)]
1495#[macro_export]
1496macro_rules! __derive_zeroable {
1497 (parse_input:
1498 @sig(
1499 $(#[$($struct_attr:tt)*])*
1500 $vis:vis struct $name:ident
1501 $(where $($whr:tt)*)?
1502 ),
1503 @impl_generics($($impl_generics:tt)*),
1504 @ty_generics($($ty_generics:tt)*),
1505 @body({
1506 $(
1507 $(#[$($field_attr:tt)*])*
1508 $field:ident : $field_ty:ty
1509 ),* $(,)?
1510 }),
1511 ) => {
1512 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1513 #[automatically_derived]
1514 unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
1515 where
1516 $($($whr)*)?
1517 {}
1518 const _: () = {
1519 fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
1520 fn ensure_zeroable<$($impl_generics)*>()
1521 where $($($whr)*)?
1522 {
1523 $(assert_zeroable::<$field_ty>();)*
1524 }
1525 };
1526 };
1527}