kernel/drm/gpuvm/
mod.rs

1// SPDX-License-Identifier: GPL-2.0 OR MIT
2
3#![cfg(CONFIG_RUST_DRM_GPUVM)]
4
5//! DRM GPUVM in immediate mode
6//!
7//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated
8//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM
9//! and the GPU's virtual address space has the same state at all times.
10//!
11//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h)
12
13use kernel::{
14    alloc::{
15        AllocError,
16        Flags as AllocFlags, //
17    },
18    bindings,
19    drm,
20    drm::gem::IntoGEMObject,
21    error::to_result,
22    prelude::*,
23    sync::aref::{
24        ARef,
25        AlwaysRefCounted, //
26    },
27    types::Opaque, //
28};
29
30use core::{
31    cell::UnsafeCell,
32    marker::PhantomData,
33    mem::{
34        ManuallyDrop,
35        MaybeUninit, //
36    },
37    ops::{
38        Deref,
39        DerefMut,
40        Range, //
41    },
42    ptr::{
43        self,
44        NonNull, //
45    }, //
46};
47
48mod sm_ops;
49pub use self::sm_ops::*;
50
51mod vm_bo;
52pub use self::vm_bo::*;
53
54mod va;
55pub use self::va::*;
56
57/// A DRM GPU VA manager.
58///
59/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed
60/// via the special unique handle [`UniqueRefGpuVm`].
61///
62/// # Invariants
63///
64/// * Stored in an allocation managed by the refcount in `self.vm`.
65/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type.
66/// * Does not contain any sparse [`GpuVa<T>`] instances.
67#[pin_data]
68pub struct GpuVm<T: DriverGpuVm> {
69    #[pin]
70    vm: Opaque<bindings::drm_gpuvm>,
71    /// Accessed only through the [`UniqueRefGpuVm`] reference.
72    data: UnsafeCell<T>,
73}
74
75// SAFETY: It is safe to send a `GpuVm<T>` to another thread: all data reachable through it
76// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds.
77unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {}
78// SAFETY: It is safe to share a `&GpuVm<T>` between threads: `&self` methods only alias data
79// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the
80// reference and ultimately drop `T`, which the same bounds make `Send`.
81unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {}
82
83// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
84unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> {
85    fn inc_ref(&self) {
86        // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
87        unsafe { bindings::drm_gpuvm_get(self.vm.get()) };
88    }
89
90    unsafe fn dec_ref(obj: NonNull<Self>) {
91        // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
92        unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) };
93    }
94}
95
96impl<T: DriverGpuVm> PartialEq for GpuVm<T> {
97    #[inline]
98    fn eq(&self, other: &Self) -> bool {
99        core::ptr::eq(self.as_raw(), other.as_raw())
100    }
101}
102impl<T: DriverGpuVm> Eq for GpuVm<T> {}
103
104impl<T: DriverGpuVm> GpuVm<T> {
105    const fn vtable() -> &'static bindings::drm_gpuvm_ops {
106        &bindings::drm_gpuvm_ops {
107            vm_free: Some(Self::vm_free),
108            op_alloc: None,
109            op_free: None,
110            vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN,
111            vm_bo_free: GpuVmBo::<T>::FREE_FN,
112            vm_bo_validate: None,
113            sm_step_map: Some(Self::sm_step_map),
114            sm_step_unmap: Some(Self::sm_step_unmap),
115            sm_step_remap: Some(Self::sm_step_remap),
116        }
117    }
118
119    /// Creates a GPUVM instance.
120    #[expect(clippy::new_ret_no_self)]
121    pub fn new<E, Ctx: drm::DeviceContext>(
122        name: &'static CStr,
123        dev: &drm::Device<T::Driver, Ctx>,
124        r_obj: &T::Object,
125        range: Range<u64>,
126        reserve_range: Range<u64>,
127        data: T,
128    ) -> Result<UniqueRefGpuVm<T>, E>
129    where
130        E: From<AllocError>,
131        E: From<core::convert::Infallible>,
132    {
133        let obj = KBox::try_pin_init::<E>(
134            try_pin_init!(Self {
135                data: UnsafeCell::new(data),
136                vm <- Opaque::ffi_init(|vm| {
137                    // SAFETY: These arguments are valid. `vm` is valid until refcount drops to
138                    // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag
139                    // below.
140                    unsafe {
141                        bindings::drm_gpuvm_init(
142                            vm,
143                            name.as_char_ptr(),
144                            bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE
145                                | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED,
146                            dev.as_raw(),
147                            r_obj.as_raw(),
148                            range.start,
149                            range.end - range.start,
150                            reserve_range.start,
151                            reserve_range.end - reserve_range.start,
152                            const { Self::vtable() },
153                        )
154                    }
155                }),
156            }? E),
157            GFP_KERNEL | __GFP_ZERO,
158        )?;
159        // SAFETY: This transfers the initial refcount to the ARef.
160        let aref = unsafe {
161            ARef::from_raw(NonNull::new_unchecked(KBox::into_raw(
162                Pin::into_inner_unchecked(obj),
163            )))
164        };
165        // INVARIANT: This reference is unique.
166        Ok(UniqueRefGpuVm(aref))
167    }
168
169    /// Access this [`GpuVm`] from a raw pointer.
170    ///
171    /// # Safety
172    ///
173    /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains
174    /// valid for at least `'a`.
175    #[inline]
176    pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self {
177        // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the
178        // pointer is valid for 'a.
179        unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) }
180    }
181
182    /// Returns a raw pointer to the embedded `struct drm_gpuvm`.
183    #[inline]
184    pub fn as_raw(&self) -> *mut bindings::drm_gpuvm {
185        self.vm.get()
186    }
187
188    /// The start of the VA space.
189    #[inline]
190    pub fn va_start(&self) -> u64 {
191        // SAFETY: The `mm_start` field is immutable.
192        unsafe { (*self.as_raw()).mm_start }
193    }
194
195    /// The length of the GPU's virtual address space.
196    #[inline]
197    pub fn va_length(&self) -> u64 {
198        // SAFETY: The `mm_range` field is immutable.
199        unsafe { (*self.as_raw()).mm_range }
200    }
201
202    /// Returns the range of the GPU virtual address space.
203    #[inline]
204    pub fn va_range(&self) -> Range<u64> {
205        let start = self.va_start();
206        // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail.
207        let end = start + self.va_length();
208        Range { start, end }
209    }
210
211    /// Get or create the [`GpuVmBo`] for this gem object.
212    #[inline]
213    pub fn obtain(
214        &self,
215        obj: &T::Object,
216        data: impl PinInit<T::VmBoData>,
217    ) -> Result<ARef<GpuVmBo<T>>, AllocError> {
218        Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain())
219    }
220
221    /// Clean up buffer objects that are no longer used.
222    #[inline]
223    pub fn deferred_cleanup(&self) {
224        // SAFETY: This GPUVM uses immediate mode.
225        unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) }
226    }
227
228    /// Check if this GEM object is an external object for this GPUVM.
229    #[inline]
230    pub fn is_extobj(&self, obj: &T::Object) -> bool {
231        // SAFETY: We may call this with any GPUVM and GEM object.
232        unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) }
233    }
234
235    /// Free this GPUVM.
236    ///
237    /// # Safety
238    ///
239    /// Called when refcount hits zero.
240    unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) {
241        // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`.
242        let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() };
243        // SAFETY: By type invariants we can free it when refcount hits zero.
244        drop(unsafe { KBox::from_raw(me) })
245    }
246
247    #[inline]
248    fn raw_resv(&self) -> *mut bindings::dma_resv {
249        // SAFETY: `r_obj` is immutable and valid for duration of GPUVM.
250        unsafe { (*(*self.as_raw()).r_obj).resv }
251    }
252}
253
254/// The manager for a GPUVM.
255pub trait DriverGpuVm: Sized + Send + Sync {
256    /// Parent `Driver` for this object.
257    type Driver: drm::Driver;
258
259    /// The kind of GEM object stored in this GPUVM.
260    type Object: drm::driver::AllocImpl<Driver = Self::Driver> + Send + Sync;
261
262    /// Data stored with each [`struct drm_gpuva`](struct@GpuVa).
263    ///
264    /// Only `Send` is required: the data has a single owner at all times, moving
265    /// between threads by value (handed back as a [`GpuVaRemoved`]) but never
266    /// accessed by two threads concurrently.
267    type VaData: Send;
268
269    /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo).
270    type VmBoData: Send + Sync;
271
272    /// The private data passed to callbacks.
273    type SmContext<'ctx>
274    where
275        Self: 'ctx;
276
277    /// Indicates that a new mapping should be created.
278    fn sm_step_map<'op, 'ctx>(
279        &mut self,
280        op: OpMap<'op, Self>,
281        context: &mut Self::SmContext<'ctx>,
282    ) -> Result<OpMapped<'op, Self>, Error>;
283
284    /// Indicates that an existing mapping should be removed.
285    fn sm_step_unmap<'op, 'ctx>(
286        &mut self,
287        op: OpUnmap<'op, Self>,
288        context: &mut Self::SmContext<'ctx>,
289    ) -> Result<OpUnmapped<'op, Self>, Error>;
290
291    /// Indicates that an existing mapping should be split up.
292    fn sm_step_remap<'op, 'ctx>(
293        &mut self,
294        op: OpRemap<'op, Self>,
295        context: &mut Self::SmContext<'ctx>,
296    ) -> Result<OpRemapped<'op, Self>, Error>;
297}
298
299/// The core of the DRM GPU VA manager.
300///
301/// This object is a unique reference to the VM that can access the interval tree and the Rust
302/// `data` field.
303///
304/// # Invariants
305///
306/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference.
307// `Send`/`Sync` derive from `ARef<GpuVm<T>>`; the trait bounds make them correct for the unique
308// handle's `&mut T` access.
309pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>);
310
311impl<T: DriverGpuVm> UniqueRefGpuVm<T> {
312    /// Access the data owned by this `UniqueRefGpuVm` immutably.
313    #[inline]
314    pub fn data_ref(&self) -> &T {
315        // SAFETY: By the type invariants we may access `data`.
316        unsafe { &*self.0.data.get() }
317    }
318
319    /// Access the data owned by this `UniqueRefGpuVm` mutably.
320    #[inline]
321    pub fn data(&mut self) -> &mut T {
322        // SAFETY: By the type invariants we may access `data`.
323        unsafe { &mut *self.0.data.get() }
324    }
325}
326
327impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> {
328    type Target = GpuVm<T>;
329
330    #[inline]
331    fn deref(&self) -> &GpuVm<T> {
332        &self.0
333    }
334}
kernel/drm/gpuvm/mod.rs

kernel/drm/gpuvm/
mod.rs
