762 lines · rust
1// SPDX-License-Identifier: GPL-2.02 3//! A reference-counted pointer.4//!5//! This module implements a way for users to create reference-counted objects and pointers to6//! them. Such a pointer automatically increments and decrements the count, and drops the7//! underlying object when it reaches zero. It is also safe to use concurrently from multiple8//! threads.9//!10//! It is different from the standard library's [`Arc`] in a few ways:11//! 1. It is backed by the kernel's `refcount_t` type.12//! 2. It does not support weak references, which allows it to be half the size.13//! 3. It saturates the reference count instead of aborting when it goes over a threshold.14//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.15//! 5. The object in [`Arc`] is pinned implicitly.16//!17//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html18 19use crate::{20 alloc::{box_ext::BoxExt, AllocError, Flags},21 bindings,22 init::{self, InPlaceInit, Init, PinInit},23 try_init,24 types::{ForeignOwnable, Opaque},25};26use alloc::boxed::Box;27use core::{28 alloc::Layout,29 fmt,30 marker::{PhantomData, Unsize},31 mem::{ManuallyDrop, MaybeUninit},32 ops::{Deref, DerefMut},33 pin::Pin,34 ptr::NonNull,35};36use macros::pin_data;37 38mod std_vendor;39 40/// A reference-counted pointer to an instance of `T`.41///42/// The reference count is incremented when new instances of [`Arc`] are created, and decremented43/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.44///45/// # Invariants46///47/// The reference count on an instance of [`Arc`] is always non-zero.48/// The object pointed to by [`Arc`] is always pinned.49///50/// # Examples51///52/// ```53/// use kernel::sync::Arc;54///55/// struct Example {56/// a: u32,57/// b: u32,58/// }59///60/// // Create a refcounted instance of `Example`.61/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;62///63/// // Get a new pointer to `obj` and increment the refcount.64/// let cloned = obj.clone();65///66/// // Assert that both `obj` and `cloned` point to the same underlying object.67/// assert!(core::ptr::eq(&*obj, &*cloned));68///69/// // Destroy `obj` and decrement its refcount.70/// drop(obj);71///72/// // Check that the values are still accessible through `cloned`.73/// assert_eq!(cloned.a, 10);74/// assert_eq!(cloned.b, 20);75///76/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.77/// # Ok::<(), Error>(())78/// ```79///80/// Using `Arc<T>` as the type of `self`:81///82/// ```83/// use kernel::sync::Arc;84///85/// struct Example {86/// a: u32,87/// b: u32,88/// }89///90/// impl Example {91/// fn take_over(self: Arc<Self>) {92/// // ...93/// }94///95/// fn use_reference(self: &Arc<Self>) {96/// // ...97/// }98/// }99///100/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;101/// obj.use_reference();102/// obj.take_over();103/// # Ok::<(), Error>(())104/// ```105///106/// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:107///108/// ```109/// use kernel::sync::{Arc, ArcBorrow};110///111/// trait MyTrait {112/// // Trait has a function whose `self` type is `Arc<Self>`.113/// fn example1(self: Arc<Self>) {}114///115/// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.116/// fn example2(self: ArcBorrow<'_, Self>) {}117/// }118///119/// struct Example;120/// impl MyTrait for Example {}121///122/// // `obj` has type `Arc<Example>`.123/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;124///125/// // `coerced` has type `Arc<dyn MyTrait>`.126/// let coerced: Arc<dyn MyTrait> = obj;127/// # Ok::<(), Error>(())128/// ```129pub struct Arc<T: ?Sized> {130 ptr: NonNull<ArcInner<T>>,131 _p: PhantomData<ArcInner<T>>,132}133 134#[pin_data]135#[repr(C)]136struct ArcInner<T: ?Sized> {137 refcount: Opaque<bindings::refcount_t>,138 data: T,139}140 141impl<T: ?Sized> ArcInner<T> {142 /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`].143 ///144 /// # Safety145 ///146 /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must147 /// not yet have been destroyed.148 unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {149 let refcount_layout = Layout::new::<bindings::refcount_t>();150 // SAFETY: The caller guarantees that the pointer is valid.151 let val_layout = Layout::for_value(unsafe { &*ptr });152 // SAFETY: We're computing the layout of a real struct that existed when compiling this153 // binary, so its layout is not so large that it can trigger arithmetic overflow.154 let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };155 156 // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and157 // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.158 //159 // This is documented at:160 // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.161 let ptr = ptr as *const ArcInner<T>;162 163 // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the164 // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is165 // still valid.166 let ptr = unsafe { ptr.byte_sub(val_offset) };167 168 // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null169 // address.170 unsafe { NonNull::new_unchecked(ptr.cast_mut()) }171 }172}173 174// This is to allow [`Arc`] (and variants) to be used as the type of `self`.175impl<T: ?Sized> core::ops::Receiver for Arc<T> {}176 177// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the178// dynamically-sized type (DST) `U`.179impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}180 181// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.182impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}183 184// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because185// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs186// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a187// mutable reference when the reference count reaches zero and `T` is dropped.188unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}189 190// SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`191// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,192// it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an193// `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when194// the reference count reaches zero and `T` is dropped.195unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}196 197impl<T> Arc<T> {198 /// Constructs a new reference counted instance of `T`.199 pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {200 // INVARIANT: The refcount is initialised to a non-zero value.201 let value = ArcInner {202 // SAFETY: There are no safety requirements for this FFI call.203 refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),204 data: contents,205 };206 207 let inner = <Box<_> as BoxExt<_>>::new(value, flags)?;208 209 // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new210 // `Arc` object.211 Ok(unsafe { Self::from_inner(Box::leak(inner).into()) })212 }213}214 215impl<T: ?Sized> Arc<T> {216 /// Constructs a new [`Arc`] from an existing [`ArcInner`].217 ///218 /// # Safety219 ///220 /// The caller must ensure that `inner` points to a valid location and has a non-zero reference221 /// count, one of which will be owned by the new [`Arc`] instance.222 unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {223 // INVARIANT: By the safety requirements, the invariants hold.224 Arc {225 ptr: inner,226 _p: PhantomData,227 }228 }229 230 /// Convert the [`Arc`] into a raw pointer.231 ///232 /// The raw pointer has ownership of the refcount that this Arc object owned.233 pub fn into_raw(self) -> *const T {234 let ptr = self.ptr.as_ptr();235 core::mem::forget(self);236 // SAFETY: The pointer is valid.237 unsafe { core::ptr::addr_of!((*ptr).data) }238 }239 240 /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].241 ///242 /// # Safety243 ///244 /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it245 /// must not be called more than once for each previous call to [`Arc::into_raw`].246 pub unsafe fn from_raw(ptr: *const T) -> Self {247 // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an248 // `Arc` that is still valid.249 let ptr = unsafe { ArcInner::container_of(ptr) };250 251 // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the252 // reference count held then will be owned by the new `Arc` object.253 unsafe { Self::from_inner(ptr) }254 }255 256 /// Returns an [`ArcBorrow`] from the given [`Arc`].257 ///258 /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method259 /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.260 #[inline]261 pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {262 // SAFETY: The constraint that the lifetime of the shared reference must outlive that of263 // the returned `ArcBorrow` ensures that the object remains alive and that no mutable264 // reference can be created.265 unsafe { ArcBorrow::new(self.ptr) }266 }267 268 /// Compare whether two [`Arc`] pointers reference the same underlying object.269 pub fn ptr_eq(this: &Self, other: &Self) -> bool {270 core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())271 }272 273 /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.274 ///275 /// When this destroys the `Arc`, it does so while properly avoiding races. This means that276 /// this method will never call the destructor of the value.277 ///278 /// # Examples279 ///280 /// ```281 /// use kernel::sync::{Arc, UniqueArc};282 ///283 /// let arc = Arc::new(42, GFP_KERNEL)?;284 /// let unique_arc = arc.into_unique_or_drop();285 ///286 /// // The above conversion should succeed since refcount of `arc` is 1.287 /// assert!(unique_arc.is_some());288 ///289 /// assert_eq!(*(unique_arc.unwrap()), 42);290 ///291 /// # Ok::<(), Error>(())292 /// ```293 ///294 /// ```295 /// use kernel::sync::{Arc, UniqueArc};296 ///297 /// let arc = Arc::new(42, GFP_KERNEL)?;298 /// let another = arc.clone();299 ///300 /// let unique_arc = arc.into_unique_or_drop();301 ///302 /// // The above conversion should fail since refcount of `arc` is >1.303 /// assert!(unique_arc.is_none());304 ///305 /// # Ok::<(), Error>(())306 /// ```307 pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> {308 // We will manually manage the refcount in this method, so we disable the destructor.309 let me = ManuallyDrop::new(self);310 // SAFETY: We own a refcount, so the pointer is still valid.311 let refcount = unsafe { me.ptr.as_ref() }.refcount.get();312 313 // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will314 // return without further touching the `Arc`. If the refcount reaches zero, then there are315 // no other arcs, and we can create a `UniqueArc`.316 //317 // SAFETY: We own a refcount, so the pointer is not dangling.318 let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };319 if is_zero {320 // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized321 // accesses to the refcount.322 unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) };323 324 // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We325 // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin326 // their values.327 Some(Pin::from(UniqueArc {328 inner: ManuallyDrop::into_inner(me),329 }))330 } else {331 None332 }333 }334}335 336impl<T: 'static> ForeignOwnable for Arc<T> {337 type Borrowed<'a> = ArcBorrow<'a, T>;338 339 fn into_foreign(self) -> *const core::ffi::c_void {340 ManuallyDrop::new(self).ptr.as_ptr() as _341 }342 343 unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> {344 // SAFETY: By the safety requirement of this function, we know that `ptr` came from345 // a previous call to `Arc::into_foreign`.346 let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap();347 348 // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive349 // for the lifetime of the returned value.350 unsafe { ArcBorrow::new(inner) }351 }352 353 unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self {354 // SAFETY: By the safety requirement of this function, we know that `ptr` came from355 // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and356 // holds a reference count increment that is transferrable to us.357 unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) }358 }359}360 361impl<T: ?Sized> Deref for Arc<T> {362 type Target = T;363 364 fn deref(&self) -> &Self::Target {365 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is366 // safe to dereference it.367 unsafe { &self.ptr.as_ref().data }368 }369}370 371impl<T: ?Sized> AsRef<T> for Arc<T> {372 fn as_ref(&self) -> &T {373 self.deref()374 }375}376 377impl<T: ?Sized> Clone for Arc<T> {378 fn clone(&self) -> Self {379 // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.380 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is381 // safe to increment the refcount.382 unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) };383 384 // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.385 unsafe { Self::from_inner(self.ptr) }386 }387}388 389impl<T: ?Sized> Drop for Arc<T> {390 fn drop(&mut self) {391 // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot392 // touch `refcount` after it's decremented to a non-zero value because another thread/CPU393 // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to394 // freed/invalid memory as long as it is never dereferenced.395 let refcount = unsafe { self.ptr.as_ref() }.refcount.get();396 397 // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and398 // this instance is being dropped, so the broken invariant is not observable.399 // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.400 let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };401 if is_zero {402 // The count reached zero, we must free the memory.403 //404 // SAFETY: The pointer was initialised from the result of `Box::leak`.405 unsafe { drop(Box::from_raw(self.ptr.as_ptr())) };406 }407 }408}409 410impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {411 fn from(item: UniqueArc<T>) -> Self {412 item.inner413 }414}415 416impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {417 fn from(item: Pin<UniqueArc<T>>) -> Self {418 // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.419 unsafe { Pin::into_inner_unchecked(item).inner }420 }421}422 423/// A borrowed reference to an [`Arc`] instance.424///425/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler426/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.427///428/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`429/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)430/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double431/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if432/// needed.433///434/// # Invariants435///436/// There are no mutable references to the underlying [`Arc`], and it remains valid for the437/// lifetime of the [`ArcBorrow`] instance.438///439/// # Example440///441/// ```442/// use kernel::sync::{Arc, ArcBorrow};443///444/// struct Example;445///446/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {447/// e.into()448/// }449///450/// let obj = Arc::new(Example, GFP_KERNEL)?;451/// let cloned = do_something(obj.as_arc_borrow());452///453/// // Assert that both `obj` and `cloned` point to the same underlying object.454/// assert!(core::ptr::eq(&*obj, &*cloned));455/// # Ok::<(), Error>(())456/// ```457///458/// Using `ArcBorrow<T>` as the type of `self`:459///460/// ```461/// use kernel::sync::{Arc, ArcBorrow};462///463/// struct Example {464/// a: u32,465/// b: u32,466/// }467///468/// impl Example {469/// fn use_reference(self: ArcBorrow<'_, Self>) {470/// // ...471/// }472/// }473///474/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;475/// obj.as_arc_borrow().use_reference();476/// # Ok::<(), Error>(())477/// ```478pub struct ArcBorrow<'a, T: ?Sized + 'a> {479 inner: NonNull<ArcInner<T>>,480 _p: PhantomData<&'a ()>,481}482 483// This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`.484impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {}485 486// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into487// `ArcBorrow<U>`.488impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>489 for ArcBorrow<'_, T>490{491}492 493impl<T: ?Sized> Clone for ArcBorrow<'_, T> {494 fn clone(&self) -> Self {495 *self496 }497}498 499impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}500 501impl<T: ?Sized> ArcBorrow<'_, T> {502 /// Creates a new [`ArcBorrow`] instance.503 ///504 /// # Safety505 ///506 /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:507 /// 1. That `inner` remains valid;508 /// 2. That no mutable references to `inner` are created.509 unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {510 // INVARIANT: The safety requirements guarantee the invariants.511 Self {512 inner,513 _p: PhantomData,514 }515 }516 517 /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with518 /// [`Arc::into_raw`].519 ///520 /// # Safety521 ///522 /// * The provided pointer must originate from a call to [`Arc::into_raw`].523 /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must524 /// not hit zero.525 /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a526 /// [`UniqueArc`] reference to this value.527 pub unsafe fn from_raw(ptr: *const T) -> Self {528 // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an529 // `Arc` that is still valid.530 let ptr = unsafe { ArcInner::container_of(ptr) };531 532 // SAFETY: The caller promises that the value remains valid since the reference count must533 // not hit zero, and no mutable reference will be created since that would involve a534 // `UniqueArc`.535 unsafe { Self::new(ptr) }536 }537}538 539impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {540 fn from(b: ArcBorrow<'_, T>) -> Self {541 // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`542 // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the543 // increment.544 ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })545 .deref()546 .clone()547 }548}549 550impl<T: ?Sized> Deref for ArcBorrow<'_, T> {551 type Target = T;552 553 fn deref(&self) -> &Self::Target {554 // SAFETY: By the type invariant, the underlying object is still alive with no mutable555 // references to it, so it is safe to create a shared reference.556 unsafe { &self.inner.as_ref().data }557 }558}559 560/// A refcounted object that is known to have a refcount of 1.561///562/// It is mutable and can be converted to an [`Arc`] so that it can be shared.563///564/// # Invariants565///566/// `inner` always has a reference count of 1.567///568/// # Examples569///570/// In the following example, we make changes to the inner object before turning it into an571/// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`572/// cannot fail.573///574/// ```575/// use kernel::sync::{Arc, UniqueArc};576///577/// struct Example {578/// a: u32,579/// b: u32,580/// }581///582/// fn test() -> Result<Arc<Example>> {583/// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;584/// x.a += 1;585/// x.b += 1;586/// Ok(x.into())587/// }588///589/// # test().unwrap();590/// ```591///592/// In the following example we first allocate memory for a refcounted `Example` but we don't593/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],594/// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens595/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):596///597/// ```598/// use kernel::sync::{Arc, UniqueArc};599///600/// struct Example {601/// a: u32,602/// b: u32,603/// }604///605/// fn test() -> Result<Arc<Example>> {606/// let x = UniqueArc::new_uninit(GFP_KERNEL)?;607/// Ok(x.write(Example { a: 10, b: 20 }).into())608/// }609///610/// # test().unwrap();611/// ```612///613/// In the last example below, the caller gets a pinned instance of `Example` while converting to614/// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during615/// initialisation, for example, when initialising fields that are wrapped in locks.616///617/// ```618/// use kernel::sync::{Arc, UniqueArc};619///620/// struct Example {621/// a: u32,622/// b: u32,623/// }624///625/// fn test() -> Result<Arc<Example>> {626/// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?);627/// // We can modify `pinned` because it is `Unpin`.628/// pinned.as_mut().a += 1;629/// Ok(pinned.into())630/// }631///632/// # test().unwrap();633/// ```634pub struct UniqueArc<T: ?Sized> {635 inner: Arc<T>,636}637 638impl<T> UniqueArc<T> {639 /// Tries to allocate a new [`UniqueArc`] instance.640 pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {641 Ok(Self {642 // INVARIANT: The newly-created object has a refcount of 1.643 inner: Arc::new(value, flags)?,644 })645 }646 647 /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.648 pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {649 // INVARIANT: The refcount is initialised to a non-zero value.650 let inner = Box::try_init::<AllocError>(651 try_init!(ArcInner {652 // SAFETY: There are no safety requirements for this FFI call.653 refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),654 data <- init::uninit::<T, AllocError>(),655 }? AllocError),656 flags,657 )?;658 Ok(UniqueArc {659 // INVARIANT: The newly-created object has a refcount of 1.660 // SAFETY: The pointer from the `Box` is valid.661 inner: unsafe { Arc::from_inner(Box::leak(inner).into()) },662 })663 }664}665 666impl<T> UniqueArc<MaybeUninit<T>> {667 /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.668 pub fn write(mut self, value: T) -> UniqueArc<T> {669 self.deref_mut().write(value);670 // SAFETY: We just wrote the value to be initialized.671 unsafe { self.assume_init() }672 }673 674 /// Unsafely assume that `self` is initialized.675 ///676 /// # Safety677 ///678 /// The caller guarantees that the value behind this pointer has been initialized. It is679 /// *immediate* UB to call this when the value is not initialized.680 pub unsafe fn assume_init(self) -> UniqueArc<T> {681 let inner = ManuallyDrop::new(self).inner.ptr;682 UniqueArc {683 // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be684 // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.685 inner: unsafe { Arc::from_inner(inner.cast()) },686 }687 }688 689 /// Initialize `self` using the given initializer.690 pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {691 // SAFETY: The supplied pointer is valid for initialization.692 match unsafe { init.__init(self.as_mut_ptr()) } {693 // SAFETY: Initialization completed successfully.694 Ok(()) => Ok(unsafe { self.assume_init() }),695 Err(err) => Err(err),696 }697 }698 699 /// Pin-initialize `self` using the given pin-initializer.700 pub fn pin_init_with<E>(701 mut self,702 init: impl PinInit<T, E>,703 ) -> core::result::Result<Pin<UniqueArc<T>>, E> {704 // SAFETY: The supplied pointer is valid for initialization and we will later pin the value705 // to ensure it does not move.706 match unsafe { init.__pinned_init(self.as_mut_ptr()) } {707 // SAFETY: Initialization completed successfully.708 Ok(()) => Ok(unsafe { self.assume_init() }.into()),709 Err(err) => Err(err),710 }711 }712}713 714impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {715 fn from(obj: UniqueArc<T>) -> Self {716 // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`717 // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.718 unsafe { Pin::new_unchecked(obj) }719 }720}721 722impl<T: ?Sized> Deref for UniqueArc<T> {723 type Target = T;724 725 fn deref(&self) -> &Self::Target {726 self.inner.deref()727 }728}729 730impl<T: ?Sized> DerefMut for UniqueArc<T> {731 fn deref_mut(&mut self) -> &mut Self::Target {732 // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so733 // it is safe to dereference it. Additionally, we know there is only one reference when734 // it's inside a `UniqueArc`, so it is safe to get a mutable reference.735 unsafe { &mut self.inner.ptr.as_mut().data }736 }737}738 739impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {740 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {741 fmt::Display::fmt(self.deref(), f)742 }743}744 745impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {746 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {747 fmt::Display::fmt(self.deref(), f)748 }749}750 751impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {752 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {753 fmt::Debug::fmt(self.deref(), f)754 }755}756 757impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {758 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {759 fmt::Debug::fmt(self.deref(), f)760 }761}762