kernel/page.rs
1// SPDX-License-Identifier: GPL-2.0
2
3//! Kernel page allocation and management.
4
5use crate::{
6 alloc::{AllocError, Flags},
7 bindings,
8 error::code::*,
9 error::Result,
10 uaccess::UserSliceReader,
11};
12use core::{
13 marker::PhantomData,
14 mem::ManuallyDrop,
15 ops::Deref,
16 ptr::{self, NonNull},
17};
18
19/// A bitwise shift for the page size.
20pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;
21
22/// The number of bytes in a page.
23pub const PAGE_SIZE: usize = bindings::PAGE_SIZE;
24
25/// A bitmask that gives the page containing a given address.
26pub const PAGE_MASK: usize = !(PAGE_SIZE - 1);
27
28/// Round up the given number to the next multiple of [`PAGE_SIZE`].
29///
30/// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a
31/// [`usize`].
32pub const fn page_align(addr: usize) -> usize {
33 // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong
34 // cases.
35 (addr + (PAGE_SIZE - 1)) & PAGE_MASK
36}
37
38/// Representation of a non-owning reference to a [`Page`].
39///
40/// This type provides a borrowed version of a [`Page`] that is owned by some other entity, e.g. a
41/// [`Vmalloc`] allocation such as [`VBox`].
42///
43/// # Example
44///
45/// ```
46/// # use kernel::{bindings, prelude::*};
47/// use kernel::page::{BorrowedPage, Page, PAGE_SIZE};
48/// # use core::{mem::MaybeUninit, ptr, ptr::NonNull };
49///
50/// fn borrow_page<'a>(vbox: &'a mut VBox<MaybeUninit<[u8; PAGE_SIZE]>>) -> BorrowedPage<'a> {
51/// let ptr = ptr::from_ref(&**vbox);
52///
53/// // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory.
54/// let page = unsafe { bindings::vmalloc_to_page(ptr.cast()) };
55///
56/// // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid
57/// // pointer to `Vmalloc` memory.
58/// let page = unsafe { NonNull::new_unchecked(page) };
59///
60/// // SAFETY:
61/// // - `self.0` is a valid pointer to a `struct page`.
62/// // - `self.0` is valid for the entire lifetime of `self`.
63/// unsafe { BorrowedPage::from_raw(page) }
64/// }
65///
66/// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?;
67/// let page = borrow_page(&mut vbox);
68///
69/// // SAFETY: There is no concurrent read or write to this page.
70/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? };
71/// # Ok::<(), Error>(())
72/// ```
73///
74/// # Invariants
75///
76/// The borrowed underlying pointer to a `struct page` is valid for the entire lifetime `'a`.
77///
78/// [`VBox`]: kernel::alloc::VBox
79/// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc
80pub struct BorrowedPage<'a>(ManuallyDrop<Page>, PhantomData<&'a Page>);
81
82impl<'a> BorrowedPage<'a> {
83 /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`.
84 ///
85 /// # Safety
86 ///
87 /// - `ptr` must point to a valid `bindings::page`.
88 /// - `ptr` must remain valid for the entire lifetime `'a`.
89 pub unsafe fn from_raw(ptr: NonNull<bindings::page>) -> Self {
90 let page = Page { page: ptr };
91
92 // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime
93 // `'a`.
94 Self(ManuallyDrop::new(page), PhantomData)
95 }
96}
97
98impl<'a> Deref for BorrowedPage<'a> {
99 type Target = Page;
100
101 fn deref(&self) -> &Self::Target {
102 &self.0
103 }
104}
105
106/// Trait to be implemented by types which provide an [`Iterator`] implementation of
107/// [`BorrowedPage`] items, such as [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter).
108pub trait AsPageIter {
109 /// The [`Iterator`] type, e.g. [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter).
110 type Iter<'a>: Iterator<Item = BorrowedPage<'a>>
111 where
112 Self: 'a;
113
114 /// Returns an [`Iterator`] of [`BorrowedPage`] items over all pages owned by `self`.
115 fn page_iter(&mut self) -> Self::Iter<'_>;
116}
117
118/// A pointer to a page that owns the page allocation.
119///
120/// # Invariants
121///
122/// The pointer is valid, and has ownership over the page.
123pub struct Page {
124 page: NonNull<bindings::page>,
125}
126
127// SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across
128// threads is safe.
129unsafe impl Send for Page {}
130
131// SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing
132// them concurrently is safe.
133unsafe impl Sync for Page {}
134
135impl Page {
136 /// Allocates a new page.
137 ///
138 /// # Examples
139 ///
140 /// Allocate memory for a page.
141 ///
142 /// ```
143 /// use kernel::page::Page;
144 ///
145 /// let page = Page::alloc_page(GFP_KERNEL)?;
146 /// # Ok::<(), kernel::alloc::AllocError>(())
147 /// ```
148 ///
149 /// Allocate memory for a page and zero its contents.
150 ///
151 /// ```
152 /// use kernel::page::Page;
153 ///
154 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?;
155 /// # Ok::<(), kernel::alloc::AllocError>(())
156 /// ```
157 #[inline]
158 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> {
159 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it
160 // is always safe to call this method.
161 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) };
162 let page = NonNull::new(page).ok_or(AllocError)?;
163 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly
164 // allocated page. We transfer that ownership to the new `Page` object.
165 Ok(Self { page })
166 }
167
168 /// Returns a raw pointer to the page.
169 pub fn as_ptr(&self) -> *mut bindings::page {
170 self.page.as_ptr()
171 }
172
173 /// Runs a piece of code with this page mapped to an address.
174 ///
175 /// The page is unmapped when this call returns.
176 ///
177 /// # Using the raw pointer
178 ///
179 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
180 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might
181 /// only be mapped on the current thread, and when that is the case, dereferencing it on other
182 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't
183 /// cause data races, the memory may be uninitialized, and so on.
184 ///
185 /// If multiple threads map the same page at the same time, then they may reference with
186 /// different addresses. However, even if the addresses are different, the underlying memory is
187 /// still the same for these purposes (e.g., it's still a data race if they both write to the
188 /// same underlying byte at the same time).
189 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T {
190 // SAFETY: `page` is valid due to the type invariants on `Page`.
191 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };
192
193 let res = f(mapped_addr.cast());
194
195 // This unmaps the page mapped above.
196 //
197 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner
198 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`.
199 //
200 // In other words, if this call to `kunmap_local` happens when a different page should be
201 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the
202 // call just above in `with_page_mapped` that made that possible. In this case, it is the
203 // unsafe block that wraps that other call that is incorrect.
204 unsafe { bindings::kunmap_local(mapped_addr) };
205
206 res
207 }
208
209 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking.
210 ///
211 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the
212 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on
213 /// this task, as this method uses a local mapping.
214 ///
215 /// If `off` and `len` refers to a region outside of this page, then this method returns
216 /// [`EINVAL`] and does not call `f`.
217 ///
218 /// # Using the raw pointer
219 ///
220 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
221 /// `len` bytes and for the duration in which the closure is called. The pointer might only be
222 /// mapped on the current thread, and when that is the case, dereferencing it on other threads
223 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause
224 /// data races, the memory may be uninitialized, and so on.
225 ///
226 /// If multiple threads map the same page at the same time, then they may reference with
227 /// different addresses. However, even if the addresses are different, the underlying memory is
228 /// still the same for these purposes (e.g., it's still a data race if they both write to the
229 /// same underlying byte at the same time).
230 fn with_pointer_into_page<T>(
231 &self,
232 off: usize,
233 len: usize,
234 f: impl FnOnce(*mut u8) -> Result<T>,
235 ) -> Result<T> {
236 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;
237
238 if bounds_ok {
239 self.with_page_mapped(move |page_addr| {
240 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will
241 // result in a pointer that is in bounds or one off the end of the page.
242 f(unsafe { page_addr.add(off) })
243 })
244 } else {
245 Err(EINVAL)
246 }
247 }
248
249 /// Maps the page and reads from it into the given buffer.
250 ///
251 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
252 /// outside of the page, then this call returns [`EINVAL`].
253 ///
254 /// # Safety
255 ///
256 /// * Callers must ensure that `dst` is valid for writing `len` bytes.
257 /// * Callers must ensure that this call does not race with a write to the same page that
258 /// overlaps with this read.
259 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result {
260 self.with_pointer_into_page(offset, len, move |src| {
261 // SAFETY: If `with_pointer_into_page` calls into this closure, then
262 // it has performed a bounds check and guarantees that `src` is
263 // valid for `len` bytes.
264 //
265 // There caller guarantees that there is no data race.
266 unsafe { ptr::copy_nonoverlapping(src, dst, len) };
267 Ok(())
268 })
269 }
270
271 /// Maps the page and writes into it from the given buffer.
272 ///
273 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
274 /// outside of the page, then this call returns [`EINVAL`].
275 ///
276 /// # Safety
277 ///
278 /// * Callers must ensure that `src` is valid for reading `len` bytes.
279 /// * Callers must ensure that this call does not race with a read or write to the same page
280 /// that overlaps with this write.
281 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result {
282 self.with_pointer_into_page(offset, len, move |dst| {
283 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
284 // bounds check and guarantees that `dst` is valid for `len` bytes.
285 //
286 // There caller guarantees that there is no data race.
287 unsafe { ptr::copy_nonoverlapping(src, dst, len) };
288 Ok(())
289 })
290 }
291
292 /// Maps the page and zeroes the given slice.
293 ///
294 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
295 /// outside of the page, then this call returns [`EINVAL`].
296 ///
297 /// # Safety
298 ///
299 /// Callers must ensure that this call does not race with a read or write to the same page that
300 /// overlaps with this write.
301 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result {
302 self.with_pointer_into_page(offset, len, move |dst| {
303 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
304 // bounds check and guarantees that `dst` is valid for `len` bytes.
305 //
306 // There caller guarantees that there is no data race.
307 unsafe { ptr::write_bytes(dst, 0u8, len) };
308 Ok(())
309 })
310 }
311
312 /// Copies data from userspace into this page.
313 ///
314 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
315 /// outside of the page, then this call returns [`EINVAL`].
316 ///
317 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address.
318 /// However, they are not allowed on the page you are copying into.
319 ///
320 /// # Safety
321 ///
322 /// Callers must ensure that this call does not race with a read or write to the same page that
323 /// overlaps with this write.
324 pub unsafe fn copy_from_user_slice_raw(
325 &self,
326 reader: &mut UserSliceReader,
327 offset: usize,
328 len: usize,
329 ) -> Result {
330 self.with_pointer_into_page(offset, len, move |dst| {
331 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
332 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have
333 // exclusive access to the slice since the caller guarantees that there are no races.
334 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) })
335 })
336 }
337}
338
339impl Drop for Page {
340 #[inline]
341 fn drop(&mut self) {
342 // SAFETY: By the type invariants, we have ownership of the page and can free it.
343 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };
344 }
345}