251 lines · rust
1// SPDX-License-Identifier: GPL-2.02 3//! Kernel page allocation and management.4 5use crate::{6 alloc::{AllocError, Flags},7 bindings,8 error::code::*,9 error::Result,10 uaccess::UserSliceReader,11};12use core::ptr::{self, NonNull};13 14/// A bitwise shift for the page size.15pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;16 17/// The number of bytes in a page.18pub const PAGE_SIZE: usize = bindings::PAGE_SIZE;19 20/// A bitmask that gives the page containing a given address.21pub const PAGE_MASK: usize = !(PAGE_SIZE - 1);22 23/// A pointer to a page that owns the page allocation.24///25/// # Invariants26///27/// The pointer is valid, and has ownership over the page.28pub struct Page {29 page: NonNull<bindings::page>,30}31 32// SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across33// threads is safe.34unsafe impl Send for Page {}35 36// SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing37// them concurrently is safe.38unsafe impl Sync for Page {}39 40impl Page {41 /// Allocates a new page.42 ///43 /// # Examples44 ///45 /// Allocate memory for a page.46 ///47 /// ```48 /// use kernel::page::Page;49 ///50 /// # fn dox() -> Result<(), kernel::alloc::AllocError> {51 /// let page = Page::alloc_page(GFP_KERNEL)?;52 /// # Ok(()) }53 /// ```54 ///55 /// Allocate memory for a page and zero its contents.56 ///57 /// ```58 /// use kernel::page::Page;59 ///60 /// # fn dox() -> Result<(), kernel::alloc::AllocError> {61 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?;62 /// # Ok(()) }63 /// ```64 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> {65 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it66 // is always safe to call this method.67 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) };68 let page = NonNull::new(page).ok_or(AllocError)?;69 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly70 // allocated page. We transfer that ownership to the new `Page` object.71 Ok(Self { page })72 }73 74 /// Returns a raw pointer to the page.75 pub fn as_ptr(&self) -> *mut bindings::page {76 self.page.as_ptr()77 }78 79 /// Runs a piece of code with this page mapped to an address.80 ///81 /// The page is unmapped when this call returns.82 ///83 /// # Using the raw pointer84 ///85 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for86 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might87 /// only be mapped on the current thread, and when that is the case, dereferencing it on other88 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't89 /// cause data races, the memory may be uninitialized, and so on.90 ///91 /// If multiple threads map the same page at the same time, then they may reference with92 /// different addresses. However, even if the addresses are different, the underlying memory is93 /// still the same for these purposes (e.g., it's still a data race if they both write to the94 /// same underlying byte at the same time).95 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T {96 // SAFETY: `page` is valid due to the type invariants on `Page`.97 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };98 99 let res = f(mapped_addr.cast());100 101 // This unmaps the page mapped above.102 //103 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner104 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`.105 //106 // In other words, if this call to `kunmap_local` happens when a different page should be107 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the108 // call just above in `with_page_mapped` that made that possible. In this case, it is the109 // unsafe block that wraps that other call that is incorrect.110 unsafe { bindings::kunmap_local(mapped_addr) };111 112 res113 }114 115 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking.116 ///117 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the118 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on119 /// this task, as this method uses a local mapping.120 ///121 /// If `off` and `len` refers to a region outside of this page, then this method returns122 /// [`EINVAL`] and does not call `f`.123 ///124 /// # Using the raw pointer125 ///126 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for127 /// `len` bytes and for the duration in which the closure is called. The pointer might only be128 /// mapped on the current thread, and when that is the case, dereferencing it on other threads129 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause130 /// data races, the memory may be uninitialized, and so on.131 ///132 /// If multiple threads map the same page at the same time, then they may reference with133 /// different addresses. However, even if the addresses are different, the underlying memory is134 /// still the same for these purposes (e.g., it's still a data race if they both write to the135 /// same underlying byte at the same time).136 fn with_pointer_into_page<T>(137 &self,138 off: usize,139 len: usize,140 f: impl FnOnce(*mut u8) -> Result<T>,141 ) -> Result<T> {142 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;143 144 if bounds_ok {145 self.with_page_mapped(move |page_addr| {146 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will147 // result in a pointer that is in bounds or one off the end of the page.148 f(unsafe { page_addr.add(off) })149 })150 } else {151 Err(EINVAL)152 }153 }154 155 /// Maps the page and reads from it into the given buffer.156 ///157 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes158 /// outside of the page, then this call returns [`EINVAL`].159 ///160 /// # Safety161 ///162 /// * Callers must ensure that `dst` is valid for writing `len` bytes.163 /// * Callers must ensure that this call does not race with a write to the same page that164 /// overlaps with this read.165 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result {166 self.with_pointer_into_page(offset, len, move |src| {167 // SAFETY: If `with_pointer_into_page` calls into this closure, then168 // it has performed a bounds check and guarantees that `src` is169 // valid for `len` bytes.170 //171 // There caller guarantees that there is no data race.172 unsafe { ptr::copy_nonoverlapping(src, dst, len) };173 Ok(())174 })175 }176 177 /// Maps the page and writes into it from the given buffer.178 ///179 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes180 /// outside of the page, then this call returns [`EINVAL`].181 ///182 /// # Safety183 ///184 /// * Callers must ensure that `src` is valid for reading `len` bytes.185 /// * Callers must ensure that this call does not race with a read or write to the same page186 /// that overlaps with this write.187 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result {188 self.with_pointer_into_page(offset, len, move |dst| {189 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a190 // bounds check and guarantees that `dst` is valid for `len` bytes.191 //192 // There caller guarantees that there is no data race.193 unsafe { ptr::copy_nonoverlapping(src, dst, len) };194 Ok(())195 })196 }197 198 /// Maps the page and zeroes the given slice.199 ///200 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes201 /// outside of the page, then this call returns [`EINVAL`].202 ///203 /// # Safety204 ///205 /// Callers must ensure that this call does not race with a read or write to the same page that206 /// overlaps with this write.207 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result {208 self.with_pointer_into_page(offset, len, move |dst| {209 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a210 // bounds check and guarantees that `dst` is valid for `len` bytes.211 //212 // There caller guarantees that there is no data race.213 unsafe { ptr::write_bytes(dst, 0u8, len) };214 Ok(())215 })216 }217 218 /// Copies data from userspace into this page.219 ///220 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes221 /// outside of the page, then this call returns [`EINVAL`].222 ///223 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address.224 /// However, they are not allowed on the page you are copying into.225 ///226 /// # Safety227 ///228 /// Callers must ensure that this call does not race with a read or write to the same page that229 /// overlaps with this write.230 pub unsafe fn copy_from_user_slice_raw(231 &self,232 reader: &mut UserSliceReader,233 offset: usize,234 len: usize,235 ) -> Result {236 self.with_pointer_into_page(offset, len, move |dst| {237 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a238 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have239 // exclusive access to the slice since the caller guarantees that there are no races.240 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) })241 })242 }243}244 245impl Drop for Page {246 fn drop(&mut self) {247 // SAFETY: By the type invariants, we have ownership of the page and can free it.248 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };249 }250}251