kernel/iommu/
pgtable.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! IOMMU page table management.
4//!
5//! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h)
6
7use core::{
8    marker::PhantomData,
9    ptr::NonNull, //
10};
11
12use crate::{
13    alloc,
14    bindings,
15    device::{
16        Bound,
17        Device, //
18    },
19    error::to_result,
20    io::PhysAddr,
21    prelude::*, //
22};
23
24use bindings::io_pgtable_fmt;
25
26/// Protection flags used with IOMMU mappings.
27pub mod prot {
28    /// Read access.
29    pub const READ: u32 = bindings::IOMMU_READ;
30    /// Write access.
31    pub const WRITE: u32 = bindings::IOMMU_WRITE;
32    /// Request cache coherency.
33    pub const CACHE: u32 = bindings::IOMMU_CACHE;
34    /// Request no-execute permission.
35    pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC;
36    /// MMIO peripheral mapping.
37    pub const MMIO: u32 = bindings::IOMMU_MMIO;
38    /// Privileged mapping.
39    pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV;
40}
41
42/// Represents a requested `io_pgtable` configuration.
43pub struct Config {
44    /// Quirk bitmask (type-specific).
45    pub quirks: usize,
46    /// Valid page sizes, as a bitmask of powers of two.
47    pub pgsize_bitmap: usize,
48    /// Input address space size in bits.
49    pub ias: u32,
50    /// Output address space size in bits.
51    pub oas: u32,
52    /// IOMMU uses coherent accesses for page table walks.
53    pub coherent_walk: bool,
54}
55
56/// An io page table using a specific format.
57///
58/// # Invariants
59///
60/// The pointer references a valid io page table.
61pub struct IoPageTable<'a, F: IoPageTableFmt> {
62    ptr: NonNull<bindings::io_pgtable_ops>,
63    _dev: PhantomData<&'a Device<Bound>>,
64    _marker: PhantomData<F>,
65}
66
67// SAFETY: `struct io_pgtable_ops` is not restricted to a single thread.
68unsafe impl<F: IoPageTableFmt> Send for IoPageTable<'_, F> {}
69// SAFETY: `struct io_pgtable_ops` may be accessed concurrently.
70unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<'_, F> {}
71
72/// The format used by this page table.
73pub trait IoPageTableFmt: 'static {
74    /// The value representing this format.
75    const FORMAT: io_pgtable_fmt;
76}
77
78impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> {
79    /// Create a new `IoPageTable`.
80    #[inline]
81    pub fn new(dev: &'a Device<Bound>, config: Config) -> Result<IoPageTable<'a, F>> {
82        let mut raw_cfg = bindings::io_pgtable_cfg {
83            quirks: config.quirks,
84            pgsize_bitmap: config.pgsize_bitmap,
85            ias: config.ias,
86            oas: config.oas,
87            coherent_walk: config.coherent_walk,
88            tlb: &raw const NOOP_FLUSH_OPS,
89            iommu_dev: dev.as_raw(),
90            ..Zeroable::zeroed()
91        };
92
93        // SAFETY:
94        // * The raw_cfg pointer is valid for the duration of this call.
95        // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer
96        //   as cookie.
97        // * The caller ensures that the io pgtable does not outlive the device.
98        let ops = unsafe {
99            bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut())
100        };
101
102        // INVARIANT: We successfully created a valid page table.
103        Ok(IoPageTable {
104            ptr: NonNull::new(ops).ok_or(ENOMEM)?,
105            _dev: PhantomData,
106            _marker: PhantomData,
107        })
108    }
109
110    /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`.
111    #[inline]
112    pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops {
113        self.ptr.as_ptr()
114    }
115
116    /// Obtain a raw pointer to the underlying `struct io_pgtable`.
117    #[inline]
118    pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable {
119        // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable.
120        unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) }
121    }
122
123    /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`.
124    #[inline]
125    pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg {
126        // SAFETY: The `raw_pgtable()` method returns a valid pointer.
127        unsafe { &raw mut (*self.raw_pgtable()).cfg }
128    }
129
130    /// Map a physically contiguous range of pages of the same size.
131    ///
132    /// Even if successful, this operation may not map the entire range. In that case, only a
133    /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In
134    /// this case, the caller will usually call `map_pages` again for the remaining range.
135    ///
136    /// The returned [`Result`] indicates whether an error was encountered while mapping pages.
137    /// Note that this may return a non-zero length even if an error was encountered. The caller
138    /// will usually [unmap the relevant pages](Self::unmap_pages) on error.
139    ///
140    /// The caller must flush the TLB before using the pgtable to access the newly created mapping.
141    ///
142    /// # Safety
143    ///
144    /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
145    ///   this `map_pages` operation executes.
146    /// * This page table must not contain any mapping that overlaps with the mapping created by
147    ///   this call.
148    /// * If this page table is live, then the caller must ensure that it's okay to access the
149    ///   physical address being mapped for the duration in which it is mapped.
150    #[inline]
151    pub unsafe fn map_pages(
152        &self,
153        iova: usize,
154        paddr: PhysAddr,
155        pgsize: usize,
156        pgcount: usize,
157        prot: u32,
158        flags: alloc::Flags,
159    ) -> (usize, Result) {
160        let mut mapped: usize = 0;
161
162        // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null.
163        let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() };
164
165        // SAFETY: The safety requirements of this method are sufficient to call `map_pages`.
166        let ret = to_result(unsafe {
167            (map_pages)(
168                self.raw_ops(),
169                iova,
170                paddr,
171                pgsize,
172                pgcount,
173                prot as i32,
174                flags.as_raw(),
175                &mut mapped,
176            )
177        });
178
179        (mapped, ret)
180    }
181
182    /// Unmap a range of virtually contiguous pages of the same size.
183    ///
184    /// This may not unmap the entire range, and returns the length of the unmapped prefix in
185    /// bytes.
186    ///
187    /// # Safety
188    ///
189    /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
190    ///   this `unmap_pages` operation executes.
191    /// * This page table must contain one or more consecutive mappings starting at `iova` whose
192    ///   total size is `pgcount * pgsize`.
193    #[inline]
194    #[must_use]
195    pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize {
196        // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null.
197        let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() };
198
199        // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`.
200        unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) }
201    }
202}
203
204// For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all
205// required invalidations using a range. There is no need for it get ARM style invalidation
206// instructions from the page table code.
207//
208// Support for flushing the TLB with ARM style invalidation instructions may be added in the
209// future.
210static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops {
211    tlb_flush_all: Some(rust_tlb_flush_all_noop),
212    tlb_flush_walk: Some(rust_tlb_flush_walk_noop),
213    tlb_add_page: None,
214};
215
216#[no_mangle]
217extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {}
218
219#[no_mangle]
220extern "C" fn rust_tlb_flush_walk_noop(
221    _iova: usize,
222    _size: usize,
223    _granule: usize,
224    _cookie: *mut core::ffi::c_void,
225) {
226}
227
228impl<F: IoPageTableFmt> Drop for IoPageTable<'_, F> {
229    fn drop(&mut self) {
230        // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this
231        // destructor runs.
232        unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) };
233    }
234}
235
236/// The `ARM_64_LPAE_S1` page table format.
237pub enum ARM64LPAES1 {}
238
239impl IoPageTableFmt for ARM64LPAES1 {
240    const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt;
241}
242
243impl IoPageTable<'_, ARM64LPAES1> {
244    /// Access the `ttbr` field of the configuration.
245    ///
246    /// This is the physical address of the page table, which may be passed to the device that
247    /// needs to use it.
248    ///
249    /// # Safety
250    ///
251    /// The caller must ensure that the device stops using the page table before dropping it.
252    #[inline]
253    pub unsafe fn ttbr(&self) -> u64 {
254        // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
255        unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr }
256    }
257
258    /// Access the `mair` field of the configuration.
259    #[inline]
260    pub fn mair(&self) -> u64 {
261        // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
262        unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair }
263    }
264}
kernel/iommu/pgtable.rs

kernel/iommu/
pgtable.rs
