Skip to main content

tenferro_gpu/cuda/
raw.rs

1//! The owner-scoped raw session an extension crate runs its vendor calls in.
2//!
3//! The session exposes exactly what a vendor library needs — the runtime's stream, device
4//! addresses, and scratch — and nothing that would re-enter the runtime's admission: transfers run
5//! on the runtime before the scope opens, because their admission is the lock the scope holds.
6//!
7//! A caller's tensor is safe to borrow for as long as the vendor work runs: dropping a buffer
8//! releases it with `cuMemFreeAsync` on the same stream, so the free is ordered after any call the
9//! session enqueued. `retain_tensor` therefore only proves the tensor belongs to this runtime.
10
11use tenferro_tensor::{StorageBuffer, TensorScalar, TypedTensor};
12
13use super::engine::CudaEngine;
14use super::storage::CudaStorage;
15
16/// A handle to the runtime's own stream, valid for the session's life.
17#[derive(Debug, Clone, Copy)]
18#[doc(hidden)]
19pub struct StreamRef {
20    handle: u64,
21}
22
23impl StreamRef {
24    /// The raw `CUstream`, as the value a vendor binding takes.
25    #[must_use]
26    #[doc(hidden)]
27    pub fn raw_handle(self) -> u64 {
28        self.handle
29    }
30}
31
32/// A device address the session handed out, valid until the scope's work completes.
33#[derive(Debug, Clone, Copy)]
34#[doc(hidden)]
35pub struct DeviceBytes<'s> {
36    pointer: u64,
37    bytes: usize,
38    _session: std::marker::PhantomData<&'s ()>,
39}
40
41impl DeviceBytes<'_> {
42    /// Run `f` with the device address.
43    #[doc(hidden)]
44    pub fn with_ptr<R>(&self, f: impl FnOnce(*mut std::ffi::c_void) -> R) -> R {
45        f(self.pointer as usize as *mut std::ffi::c_void)
46    }
47
48    /// Number of bytes at the address.
49    #[must_use]
50    #[doc(hidden)]
51    pub fn byte_len(&self) -> usize {
52        self.bytes
53    }
54}
55
56/// A read-only device view of a caller's tensor.
57#[derive(Debug, Clone, Copy)]
58#[doc(hidden)]
59pub struct TensorRef<'a, T> {
60    pointer: u64,
61    bytes: usize,
62    _tensor: std::marker::PhantomData<&'a T>,
63}
64
65impl<T> TensorRef<'_, T> {
66    /// The tensor's device address, as a vendor binding takes it.
67    #[doc(hidden)]
68    pub fn raw_ptr(&self) -> *mut std::ffi::c_void {
69        self.pointer as usize as *mut std::ffi::c_void
70    }
71
72    /// Run `f` with the tensor's device address.
73    #[doc(hidden)]
74    pub fn with_ptr<R>(&self, f: impl FnOnce(*mut std::ffi::c_void) -> R) -> R {
75        f(self.pointer as usize as *mut std::ffi::c_void)
76    }
77
78    /// Number of bytes the tensor spans.
79    #[must_use]
80    #[doc(hidden)]
81    pub fn byte_len(&self) -> usize {
82        self.bytes
83    }
84}
85
86/// A writable device view of a caller's tensor.
87#[derive(Debug, Clone, Copy)]
88#[doc(hidden)]
89pub struct TensorMut<'a, T> {
90    pointer: u64,
91    bytes: usize,
92    _tensor: std::marker::PhantomData<&'a mut T>,
93}
94
95impl<T> TensorMut<'_, T> {
96    /// The tensor's device address, as a vendor binding takes it.
97    #[doc(hidden)]
98    pub fn raw_ptr(&self) -> *mut std::ffi::c_void {
99        self.pointer as usize as *mut std::ffi::c_void
100    }
101
102    /// Run `f` with the tensor's device address.
103    #[doc(hidden)]
104    pub fn with_ptr<R>(&self, f: impl FnOnce(*mut std::ffi::c_void) -> R) -> R {
105        f(self.pointer as usize as *mut std::ffi::c_void)
106    }
107
108    /// Number of bytes the tensor spans.
109    #[must_use]
110    #[doc(hidden)]
111    pub fn byte_len(&self) -> usize {
112        self.bytes
113    }
114}
115
116/// One raw scope, handed to an extension's closure.
117#[doc(hidden)]
118pub struct Session<'s> {
119    runtime: super::CudaRuntime,
120    scope: t4a_cuda_runtime::RawScope<'s>,
121    /// The engine's vendor resources: a handle is created once per engine and reused by every
122    /// later session. The cell is per session and the session is thread-bound, so a shared borrow
123    /// during a vendor call costs nothing and keeps `resource` callable while the session is read.
124    caches: &'s std::cell::RefCell<tenferro_runtime::ExtensionCacheStore>,
125    /// How many resources this engine created rather than reused, for the regression test that
126    /// pins their lifetime to the engine.
127    resource_inits: &'s std::cell::Cell<u64>,
128}
129
130impl<'s> Session<'s> {
131    /// The runtime's stream, for a vendor library to bind its plan to.
132    #[must_use]
133    #[doc(hidden)]
134    pub fn stream(&self) -> StreamRef {
135        StreamRef {
136            handle: self.scope.stream(),
137        }
138    }
139
140    /// Zero-filled scratch in this runtime's domain.
141    ///
142    /// # Errors
143    ///
144    /// [`crate::Error::RuntimeStateSource`] when the allocation or its zero fill fails.
145    #[doc(hidden)]
146    pub fn alloc_bytes(&self, nbytes: usize, op: &'static str) -> crate::Result<DeviceBytes<'s>> {
147        let buffer = self
148            .scope
149            .alloc_zeroed(nbytes, 256)
150            .map_err(|error| crate::Error::runtime_state_source(op, error))?;
151        let pointer = self
152            .scope
153            .register(&buffer)
154            .map_err(|error| crate::Error::runtime_state_source(op, error))?;
155        // The allocation lives as long as this handle: it moves into the scope's keepalives so the
156        // device address stays valid until the scope's completion, whatever the caller does.
157        self.scope
158            .retain(Box::new(buffer), nbytes)
159            .map_err(|error| crate::Error::runtime_state_source(op, error))?;
160        Ok(DeviceBytes {
161            pointer,
162            bytes: nbytes,
163            _session: std::marker::PhantomData,
164        })
165    }
166
167    /// Copy `bytes` between two device addresses on this session's stream.
168    ///
169    /// # Errors
170    ///
171    /// [`crate::Error::RuntimeState`] when the driver refuses the copy.
172    #[doc(hidden)]
173    /// # Safety
174    ///
175    /// `destination` and `source` must be device addresses this session's engine owns, `bytes`
176    /// must not exceed either allocation, and the ranges must not overlap unless
177    /// `cuMemcpyDtoDAsync` allows it. The copy is enqueued on this session's stream, so the
178    /// allocations must outlive the point where that work retires.
179    pub unsafe fn copy_bytes(
180        &self,
181        destination: *mut std::ffi::c_void,
182        source: *const std::ffi::c_void,
183        bytes: usize,
184        op: &'static str,
185    ) -> crate::Result<()> {
186        let status = unsafe {
187            cudarc::driver::sys::cuMemcpyDtoDAsync_v2(
188                destination as usize as cudarc::driver::sys::CUdeviceptr,
189                source as usize as cudarc::driver::sys::CUdeviceptr,
190                bytes,
191                self.scope.stream() as usize as *mut cudarc::driver::sys::CUstream_st,
192            )
193        };
194        if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
195            return Err(crate::Error::runtime_state(
196                op,
197                format!("cuMemcpyDtoDAsync failed with {status:?}"),
198            ));
199        }
200        Ok(())
201    }
202
203    /// Copy host `bytes` into fresh scratch of this runtime, on this session's stream.
204    ///
205    /// The host source is pageable and the copy is asynchronous, so the bytes are moved into the
206    /// scope's keepalives: the driver may still be reading them when the caller returns.
207    ///
208    /// # Errors
209    ///
210    /// [`crate::Error::RuntimeState`] when the allocation or the copy is refused.
211    #[doc(hidden)]
212    pub fn upload_bytes(&self, bytes: &[u8], op: &'static str) -> crate::Result<DeviceBytes<'s>> {
213        let device = self.alloc_bytes(bytes.len(), op)?;
214        let owned = bytes.to_vec();
215        let source = owned.as_ptr() as usize;
216        device.with_ptr(|destination| {
217            let status = unsafe {
218                cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
219                    destination as usize as cudarc::driver::sys::CUdeviceptr,
220                    source as *const std::ffi::c_void,
221                    owned.len(),
222                    self.scope.stream() as usize as *mut cudarc::driver::sys::CUstream_st,
223                )
224            };
225            if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
226                return Err(crate::Error::runtime_state(
227                    op,
228                    format!("cuMemcpyHtoDAsync failed with {status:?}"),
229                ));
230            }
231            Ok(())
232        })?;
233        // The driver may still be reading the host source, so it outlives this call through the
234        // scope's completion.
235        self.scope
236            .retain(Box::new(owned), 0)
237            .map_err(|error| crate::Error::runtime_state_source(op, error))?;
238        Ok(device)
239    }
240
241    /// Copy a device tensor into a host tensor of the same shape.
242    ///
243    /// # Errors
244    ///
245    /// [`crate::Error::RuntimeState`] when the tensor is not this runtime's allocation, and the
246    /// runtime's own failure when the copy or its barrier fails.
247    #[doc(hidden)]
248    pub fn download_tensor<T: TensorScalar>(
249        &self,
250        tensor: &TypedTensor<T>,
251        op: &'static str,
252    ) -> crate::Result<TypedTensor<T>> {
253        let (pointer, bytes) = self.address(tensor, op)?;
254        // One pinned staging buffer per runtime, reused by every later download; a download that
255        // cannot take the cache (a nested one) keeps its own buffer instead of blocking on it.
256        //
257        // Above the retained bound nothing is kept: the download stages through a temporary
258        // buffer that its `Drop` releases when this call returns, so a single large readback
259        // cannot pin that much memory for the runtime's lifetime (tenferro-rs #2440). The bound
260        // is read while holding the same lock a limit change takes, so a concurrent
261        // `set_pinned_staging_limit_bytes` cannot race a decision made from the old value.
262        let accounting = &self.runtime.inner.pinned_staging_accounting;
263        let mut cached = self.runtime.inner.pinned_staging.try_lock().ok();
264        if cached
265            .as_ref()
266            .is_some_and(|_| bytes.max(1) > accounting.limit())
267        {
268            cached = None;
269        }
270        let mut fallback = PinnedStaging::empty();
271        let destination = match cached.as_mut() {
272            Some(slot) => {
273                let entry = slot.get_or_insert_with(PinnedStaging::empty);
274                let pointer = entry.ensure(bytes, op, accounting)?;
275                // Only the cached buffer is retained; a temporary one never enters the count.
276                accounting.record_retained(entry.bytes());
277                pointer
278            }
279            None => fallback.ensure(bytes, op, accounting)?,
280        };
281        let status = unsafe {
282            cudarc::driver::sys::cuMemcpyDtoHAsync_v2(
283                destination,
284                pointer as cudarc::driver::sys::CUdeviceptr,
285                bytes,
286                self.scope.stream() as usize as *mut cudarc::driver::sys::CUstream_st,
287            )
288        };
289        if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
290            return Err(crate::Error::runtime_state(
291                op,
292                format!("cuMemcpyDtoHAsync failed with {status:?}"),
293            ));
294        }
295        self.synchronize()?;
296        let source = unsafe {
297            std::slice::from_raw_parts(
298                destination as *const T,
299                bytes / std::mem::size_of::<T>().max(1),
300            )
301        };
302        let mut values: Vec<T> = Vec::new();
303        values
304            .try_reserve_exact(source.len())
305            .map_err(|error| crate::Error::backend_source(op, error))?;
306        values.extend_from_slice(source);
307        TypedTensor::<T>::from_vec_col_major(tensor.shape().to_vec(), values)
308            .map_err(|error| crate::Error::runtime_state(op, error.to_string()))
309    }
310
311    /// Zero-filled device storage of `shape` for `T`, as a tensor of this runtime.
312    ///
313    /// # Errors
314    ///
315    /// [`crate::Error::RuntimeState`] when the allocation fails or the shape does not describe a
316    /// column-major tensor of `T`.
317    #[doc(hidden)]
318    pub fn alloc_output<T: TensorScalar>(&self, shape: &[usize]) -> crate::Result<TypedTensor<T>> {
319        const OP: &str = "cuda raw alloc_output";
320        let count = shape.iter().try_fold(1usize, |product, &extent| {
321            product
322                .checked_mul(extent)
323                .ok_or_else(|| crate::Error::runtime_state(OP, "the element count overflows usize"))
324        })?;
325        let bytes = count
326            .checked_mul(std::mem::size_of::<T>().max(1))
327            .ok_or_else(|| crate::Error::runtime_state(OP, "the byte length overflows usize"))?;
328        let buffer = self
329            .scope
330            .alloc_zeroed(bytes, 256)
331            .map_err(|error| crate::Error::runtime_state_source(OP, error))?;
332        // The returned tensor owns the allocation, so its own drop frees it — in this runtime's
333        // stream order, after any vendor call the session enqueued.
334        let storage = super::storage::fresh_storage(&self.runtime, buffer);
335        TypedTensor::<T>::from_buffer_col_major(
336            shape.to_vec(),
337            tenferro_tensor::StorageBuffer::Backend(Box::new(storage)),
338            super::transfer::placement(&self.runtime),
339        )
340        .map_err(|error| crate::Error::runtime_state(OP, error.to_string()))
341    }
342
343    /// A read-only device view of `tensor`.
344    ///
345    /// # Errors
346    ///
347    /// [`crate::Error::RuntimeState`] when the tensor is not this runtime's allocation.
348    #[doc(hidden)]
349    pub fn tensor<T: TensorScalar>(
350        &self,
351        tensor: &TypedTensor<T>,
352    ) -> crate::Result<TensorRef<'_, T>> {
353        let (pointer, bytes) = self.address(tensor, "cuda raw tensor")?;
354        Ok(TensorRef {
355            pointer,
356            bytes,
357            _tensor: std::marker::PhantomData,
358        })
359    }
360
361    /// A writable device view of `tensor`.
362    ///
363    /// # Errors
364    ///
365    /// [`crate::Error::RuntimeState`] when the tensor is not this runtime's allocation.
366    #[doc(hidden)]
367    pub fn tensor_mut<T: TensorScalar>(
368        &self,
369        tensor: &TypedTensor<T>,
370    ) -> crate::Result<TensorMut<'_, T>> {
371        let (pointer, bytes) = self.address(tensor, "cuda raw tensor mut")?;
372        Ok(TensorMut {
373            pointer,
374            bytes,
375            _tensor: std::marker::PhantomData,
376        })
377    }
378
379    /// Prove `tensor` is one of this runtime's allocations, for a vendor call that reads it.
380    ///
381    /// # Errors
382    ///
383    /// [`crate::Error::RuntimeState`] when the tensor is not this runtime's allocation.
384    #[doc(hidden)]
385    /// # Safety
386    ///
387    /// The returned address stays valid only while `tensor`'s allocation is resident on this
388    /// engine; the caller must not outlive it.
389    pub unsafe fn retain_tensor<T: TensorScalar>(
390        &self,
391        tensor: &TypedTensor<T>,
392        op: &'static str,
393    ) -> crate::Result<DeviceBytes<'static>> {
394        let (pointer, bytes) = self.address(tensor, op)?;
395        Ok(DeviceBytes {
396            pointer,
397            bytes,
398            _session: std::marker::PhantomData,
399        })
400    }
401
402    /// This session's vendor resource of type `T`, created by `init` on first use.
403    ///
404    /// One slot per type: the cache key carries the type name, so two resources of different types
405    /// in one engine cannot collide, and the same handle is reused by every later session.
406    ///
407    /// # Examples
408    ///
409    /// ```no_run
410    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
411    /// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaEngine};
412    ///
413    /// # #[cfg(feature = "cuda")]
414    /// if gpu_available() {
415    ///     let mut engine = CudaEngine::open(CudaDeviceId::from_ordinal(0))?;
416    ///     // The guard borrows the session, so it stays inside the closure.
417    ///     let total = engine.with_raw("resource", |session| {
418    ///         let counter = session.resource(|| Ok::<u64, tenferro_tensor::Error>(7))?;
419    ///         Ok(*counter + 1)
420    ///     })?;
421    ///     assert_eq!(total, 8);
422    /// }
423    /// # Ok(())
424    /// # }
425    /// ```
426    ///
427    /// # Panics
428    ///
429    /// Only if storing the initialised resource succeeded and it is then missing from the cache,
430    /// which is an internal invariant of the store above.
431    ///
432    /// # Errors
433    ///
434    /// The initialiser's failure, and the engine's when the cache is poisoned.
435    pub fn resource<T: Send + Sync + 'static>(
436        &self,
437        init: impl FnOnce() -> crate::Result<T>,
438    ) -> crate::Result<CudaResourceGuard<'_, T>> {
439        let key = tenferro_runtime::ExtensionCacheKey::new(
440            "tenferro.cuda.session",
441            std::any::type_name::<T>(),
442            0,
443        );
444        let mut store = self.caches.borrow_mut();
445        if store.get::<T>(&key).is_none() {
446            let value = init()?;
447            self.resource_inits.set(self.resource_inits.get() + 1);
448            store.put(key, value, 0);
449        }
450        let entry = std::cell::RefMut::map(store, |store| {
451            store
452                .get_mut::<T>(&key)
453                .expect("the resource was stored above")
454        });
455        Ok(CudaResourceGuard { entry })
456    }
457
458    /// Wait for the work enqueued on this session's stream.
459    ///
460    /// # Errors
461    ///
462    /// [`crate::Error::RuntimeStateSource`] when the stream cannot be synchronised.
463    #[doc(hidden)]
464    pub fn synchronize(&self) -> crate::Result<()> {
465        let stream = self.scope.stream() as usize as *mut cudarc::driver::sys::CUstream_st;
466        // SAFETY: the scope hands out this runtime's own stream; a barrier on it neither reads nor
467        // writes an operand.
468        let status = unsafe { cudarc::driver::sys::cuStreamSynchronize(stream) };
469        if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
470            return Err(crate::Error::runtime_state(
471                "cuda raw synchronize",
472                format!("cuStreamSynchronize failed with {status:?}"),
473            ));
474        }
475        Ok(())
476    }
477
478    fn address<T: TensorScalar>(
479        &self,
480        tensor: &TypedTensor<T>,
481        op: &'static str,
482    ) -> crate::Result<(u64, usize)> {
483        let storage = match tensor.buffer() {
484            StorageBuffer::Backend(buffer) => buffer
485                .as_ref()
486                .as_any()
487                .downcast_ref::<CudaStorage>()
488                .ok_or_else(|| {
489                    crate::Error::runtime_state(op, "expected a CUDA allocation of this backend")
490                })?,
491            StorageBuffer::Host(_) => {
492                return Err(crate::Error::runtime_state(
493                    op,
494                    "expected a device tensor, not host storage",
495                ))
496            }
497        };
498        if storage.domain() != self.runtime.allocation_domain_id() {
499            return Err(crate::Error::runtime_state(
500                op,
501                "the tensor belongs to another runtime",
502            ));
503        }
504        let pointer = self
505            .scope
506            .register(storage.allocation())
507            .map_err(|error| crate::Error::runtime_state_source(op, error))?;
508        Ok((pointer, storage.bytes()))
509    }
510}
511
512/// Marks a [`NativeSessionRef`](tenferro_tensor::NativeSessionRef) as this engine's, so an
513/// extension can reach the concrete engine through `&mut dyn Engine`.
514#[derive(Debug, Clone, Copy)]
515pub(crate) struct CudaEngineMarker;
516
517/// Visit this engine through the tensor crate's engine trait, as an extension does.
518///
519/// Returns `None` when `engine` is not this backend's engine.
520#[doc(hidden)]
521pub fn with_cuda_engine<B, R>(engine: &mut B, f: impl FnOnce(&mut CudaEngine) -> R) -> Option<R>
522where
523    B: tenferro_tensor::Engine + ?Sized,
524{
525    let pointer = engine
526        .native_engine()?
527        .into_marked_ptr::<CudaEngineMarker>()?;
528    // SAFETY: only this crate's `native_engine` creates a token carrying this marker, and it points
529    // that token at a live `CudaEngine`; the `&mut B` keeps the exclusive borrow for the visit.
530    Some(f(unsafe { pointer.cast::<CudaEngine>().as_mut() }))
531}
532
533impl CudaEngine {
534    /// Run `f` inside this engine's raw scope.
535    ///
536    /// Every vendor call the closure makes must be enqueued on [`Session::stream`]; a buffer the
537    /// caller drops afterwards is freed in that stream's order, so the call still sees it.
538    ///
539    /// # Errors
540    ///
541    /// The closure's own failure, and the runtime's when the scope cannot be opened.
542    #[doc(hidden)]
543    pub fn with_raw<R>(
544        &mut self,
545        _op: &'static str,
546        f: impl FnOnce(&mut Session<'_>) -> crate::Result<R>,
547    ) -> crate::Result<R> {
548        // The runtime handle is cloned first so the engine's caches can be borrowed mutably for the
549        // same call.
550        let runtime = self.runtime().clone();
551        // The engine's own resources, not the store a run lends away: a vendor handle has to
552        // outlive the run that created it.
553        let caches = &self.vendor_resources;
554        let resource_inits = &self.vendor_resource_inits;
555        // SAFETY: the closure only reads the addresses the session registers and enqueues its work
556        // on this scope's stream.
557        unsafe {
558            runtime
559                .with_raw_locked(Vec::new(), |scope| {
560                    let mut session = Session {
561                        runtime: runtime.clone(),
562                        resource_inits,
563                        scope,
564                        caches,
565                    };
566                    f(&mut session)
567                })
568                .and_then(|result| result)
569        }
570    }
571}
572
573/// A pinned host buffer used as the staging area of downloads.
574///
575/// The runtime keeps one of these and reuses it, growing on demand: pinning and unpinning a fresh
576/// buffer per download cost more than the copy it staged (a 4-byte `cuMemHostAlloc` measured
577/// 1.9 ms beside a 19 us device-to-host copy in the daily GPU linalg profile).
578pub(crate) struct PinnedStaging {
579    pointer: *mut std::ffi::c_void,
580    bytes: usize,
581}
582
583/// What a runtime's download staging retains, and the bound it retains it under.
584///
585/// The buffer is page-locked host memory, so an unbounded one holds memory the allocator cannot
586/// reclaim: a single large download would keep that much pinned for the runtime's lifetime. The
587/// bound decides what is kept; a download above it stages through a temporary buffer that is
588/// released when the call returns.
589pub(crate) struct StagingAccounting {
590    /// Bytes the cached buffer currently holds.
591    retained: std::sync::atomic::AtomicUsize,
592    /// Pinned allocations this runtime made for staging, growth included.
593    allocations: std::sync::atomic::AtomicU64,
594    /// Largest download whose staging is retained.
595    limit: std::sync::atomic::AtomicUsize,
596}
597
598impl StagingAccounting {
599    pub(crate) fn new(limit: usize) -> Self {
600        Self {
601            retained: std::sync::atomic::AtomicUsize::new(0),
602            allocations: std::sync::atomic::AtomicU64::new(0),
603            limit: std::sync::atomic::AtomicUsize::new(limit),
604        }
605    }
606
607    pub(crate) fn retained(&self) -> usize {
608        self.retained.load(std::sync::atomic::Ordering::Relaxed)
609    }
610
611    pub(crate) fn allocations(&self) -> u64 {
612        self.allocations.load(std::sync::atomic::Ordering::Relaxed)
613    }
614
615    pub(crate) fn limit(&self) -> usize {
616        self.limit.load(std::sync::atomic::Ordering::Relaxed)
617    }
618
619    pub(crate) fn set_limit(&self, limit: usize) {
620        self.limit
621            .store(limit, std::sync::atomic::Ordering::Relaxed);
622    }
623
624    /// Publish what the cached buffer holds. The runtime resets it when it releases the buffer.
625    pub(crate) fn record_retained(&self, retained: usize) {
626        self.retained
627            .store(retained, std::sync::atomic::Ordering::Relaxed);
628    }
629}
630
631// SAFETY: the buffer is a plain page-locked host allocation with no interior aliasing. It lives
632// behind the runtime state's mutex, so one thread owns it at a time; the mutex needs `Send` only.
633unsafe impl Send for PinnedStaging {}
634
635impl PinnedStaging {
636    /// The empty staging entry the runtime fills on its first download.
637    pub(crate) fn empty() -> Self {
638        Self {
639            pointer: std::ptr::null_mut(),
640            bytes: 0,
641        }
642    }
643
644    /// Bytes this entry retains, zero for an empty one.
645    pub(crate) fn bytes(&self) -> usize {
646        self.bytes
647    }
648
649    /// The raw pinned allocation, which the runtime frees inside its own scope.
650    pub(crate) fn pointer(&self) -> *mut std::ffi::c_void {
651        self.pointer
652    }
653
654    /// A writable staging area of at least `bytes` bytes, growing once if needed.
655    ///
656    /// Growth allocates the replacement before releasing the old buffer, so a failed allocation
657    /// leaves both the buffer and the accounting exactly as they were; every path publishes the
658    /// size actually retained.
659    fn ensure(
660        &mut self,
661        bytes: usize,
662        op: &'static str,
663        accounting: &StagingAccounting,
664    ) -> crate::Result<*mut std::ffi::c_void> {
665        let bytes = bytes.max(1);
666        if self.pointer.is_null() || self.bytes < bytes {
667            let mut pointer: *mut std::ffi::c_void = std::ptr::null_mut();
668            let status = unsafe {
669                cudarc::driver::sys::cuMemHostAlloc(
670                    &mut pointer,
671                    bytes,
672                    cudarc::driver::sys::CU_MEMHOSTALLOC_PORTABLE,
673                )
674            };
675            if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
676                return Err(crate::Error::runtime_state(
677                    op,
678                    format!("cuMemHostAlloc failed with {status:?}"),
679                ));
680            }
681            // SAFETY: `pointer` is null or this entry's own live pinned allocation, released once
682            // here after its replacement exists.
683            unsafe {
684                if !self.pointer.is_null() {
685                    let _ = cudarc::driver::sys::cuMemFreeHost(self.pointer);
686                }
687            }
688            self.pointer = pointer;
689            self.bytes = bytes;
690            accounting
691                .allocations
692                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
693        }
694        Ok(self.pointer)
695    }
696}
697
698impl Drop for PinnedStaging {
699    fn drop(&mut self) {
700        // SAFETY: the pointer came from `cuMemHostAlloc` and is freed once here.
701        unsafe {
702            let _ = cudarc::driver::sys::cuMemFreeHost(self.pointer);
703        }
704        self.bytes = 0;
705    }
706}
707
708/// A vendor resource lent from the engine's extension caches for the session's life.
709///
710/// # Examples
711///
712/// ```no_run
713/// use tenferro_gpu::cuda::raw::CudaResourceGuard;
714///
715/// fn handle_lifetime(guard: &CudaResourceGuard<'_, u64>) -> u64 {
716///     // The guard dereferences to the cached resource and releases it when dropped.
717///     **guard + 1
718/// }
719///
720/// # fn main() {}
721/// ```
722pub struct CudaResourceGuard<'s, T> {
723    entry: std::cell::RefMut<'s, T>,
724}
725
726impl<T> std::ops::Deref for CudaResourceGuard<'_, T> {
727    type Target = T;
728
729    fn deref(&self) -> &T {
730        &self.entry
731    }
732}
733
734impl<T> std::ops::DerefMut for CudaResourceGuard<'_, T> {
735    fn deref_mut(&mut self) -> &mut T {
736        &mut self.entry
737    }
738}
739
740impl<T: std::fmt::Debug> std::fmt::Debug for CudaResourceGuard<'_, T> {
741    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
742        self.entry.fmt(formatter)
743    }
744}