tenferro_gpu/cuda/raw.rs
1//! The owner-scoped raw session an extension crate runs its vendor calls in.
2//!
3//! The session exposes exactly what a vendor library needs — the runtime's stream, device
4//! addresses, and scratch — and nothing that would re-enter the runtime's admission: transfers run
5//! on the runtime before the scope opens, because their admission is the lock the scope holds.
6//!
7//! A caller's tensor is safe to borrow for as long as the vendor work runs: dropping a buffer
8//! releases it with `cuMemFreeAsync` on the same stream, so the free is ordered after any call the
9//! session enqueued. `retain_tensor` therefore only proves the tensor belongs to this runtime.
10
11use tenferro_tensor::{StorageBuffer, TensorScalar, TypedTensor};
12
13use super::engine::CudaEngine;
14use super::storage::CudaStorage;
15
16/// A handle to the runtime's own stream, valid for the session's life.
17#[derive(Debug, Clone, Copy)]
18#[doc(hidden)]
19pub struct StreamRef {
20 handle: u64,
21}
22
23impl StreamRef {
24 /// The raw `CUstream`, as the value a vendor binding takes.
25 #[must_use]
26 #[doc(hidden)]
27 pub fn raw_handle(self) -> u64 {
28 self.handle
29 }
30}
31
32/// A device address the session handed out, valid until the scope's work completes.
33#[derive(Debug, Clone, Copy)]
34#[doc(hidden)]
35pub struct DeviceBytes<'s> {
36 pointer: u64,
37 bytes: usize,
38 _session: std::marker::PhantomData<&'s ()>,
39}
40
41impl DeviceBytes<'_> {
42 /// Run `f` with the device address.
43 #[doc(hidden)]
44 pub fn with_ptr<R>(&self, f: impl FnOnce(*mut std::ffi::c_void) -> R) -> R {
45 f(self.pointer as usize as *mut std::ffi::c_void)
46 }
47
48 /// Number of bytes at the address.
49 #[must_use]
50 #[doc(hidden)]
51 pub fn byte_len(&self) -> usize {
52 self.bytes
53 }
54}
55
56/// A read-only device view of a caller's tensor.
57#[derive(Debug, Clone, Copy)]
58#[doc(hidden)]
59pub struct TensorRef<'a, T> {
60 pointer: u64,
61 bytes: usize,
62 _tensor: std::marker::PhantomData<&'a T>,
63}
64
65impl<T> TensorRef<'_, T> {
66 /// The tensor's device address, as a vendor binding takes it.
67 #[doc(hidden)]
68 pub fn raw_ptr(&self) -> *mut std::ffi::c_void {
69 self.pointer as usize as *mut std::ffi::c_void
70 }
71
72 /// Run `f` with the tensor's device address.
73 #[doc(hidden)]
74 pub fn with_ptr<R>(&self, f: impl FnOnce(*mut std::ffi::c_void) -> R) -> R {
75 f(self.pointer as usize as *mut std::ffi::c_void)
76 }
77
78 /// Number of bytes the tensor spans.
79 #[must_use]
80 #[doc(hidden)]
81 pub fn byte_len(&self) -> usize {
82 self.bytes
83 }
84}
85
86/// A writable device view of a caller's tensor.
87#[derive(Debug, Clone, Copy)]
88#[doc(hidden)]
89pub struct TensorMut<'a, T> {
90 pointer: u64,
91 bytes: usize,
92 _tensor: std::marker::PhantomData<&'a mut T>,
93}
94
95impl<T> TensorMut<'_, T> {
96 /// The tensor's device address, as a vendor binding takes it.
97 #[doc(hidden)]
98 pub fn raw_ptr(&self) -> *mut std::ffi::c_void {
99 self.pointer as usize as *mut std::ffi::c_void
100 }
101
102 /// Run `f` with the tensor's device address.
103 #[doc(hidden)]
104 pub fn with_ptr<R>(&self, f: impl FnOnce(*mut std::ffi::c_void) -> R) -> R {
105 f(self.pointer as usize as *mut std::ffi::c_void)
106 }
107
108 /// Number of bytes the tensor spans.
109 #[must_use]
110 #[doc(hidden)]
111 pub fn byte_len(&self) -> usize {
112 self.bytes
113 }
114}
115
116/// One raw scope, handed to an extension's closure.
117#[doc(hidden)]
118pub struct Session<'s> {
119 runtime: super::CudaRuntime,
120 scope: t4a_cuda_runtime::RawScope<'s>,
121 /// The engine's vendor resources: a handle is created once per engine and reused by every
122 /// later session. The cell is per session and the session is thread-bound, so a shared borrow
123 /// during a vendor call costs nothing and keeps `resource` callable while the session is read.
124 caches: &'s std::cell::RefCell<tenferro_runtime::ExtensionCacheStore>,
125 /// How many resources this engine created rather than reused, for the regression test that
126 /// pins their lifetime to the engine.
127 resource_inits: &'s std::cell::Cell<u64>,
128}
129
130impl<'s> Session<'s> {
131 /// The runtime's stream, for a vendor library to bind its plan to.
132 #[must_use]
133 #[doc(hidden)]
134 pub fn stream(&self) -> StreamRef {
135 StreamRef {
136 handle: self.scope.stream(),
137 }
138 }
139
140 /// Zero-filled scratch in this runtime's domain.
141 ///
142 /// # Errors
143 ///
144 /// [`crate::Error::RuntimeStateSource`] when the allocation or its zero fill fails.
145 #[doc(hidden)]
146 pub fn alloc_bytes(&self, nbytes: usize, op: &'static str) -> crate::Result<DeviceBytes<'s>> {
147 let buffer = self
148 .scope
149 .alloc_zeroed(nbytes, 256)
150 .map_err(|error| crate::Error::runtime_state_source(op, error))?;
151 let pointer = self
152 .scope
153 .register(&buffer)
154 .map_err(|error| crate::Error::runtime_state_source(op, error))?;
155 // The allocation lives as long as this handle: it moves into the scope's keepalives so the
156 // device address stays valid until the scope's completion, whatever the caller does.
157 self.scope
158 .retain(Box::new(buffer), nbytes)
159 .map_err(|error| crate::Error::runtime_state_source(op, error))?;
160 Ok(DeviceBytes {
161 pointer,
162 bytes: nbytes,
163 _session: std::marker::PhantomData,
164 })
165 }
166
167 /// Copy `bytes` between two device addresses on this session's stream.
168 ///
169 /// # Errors
170 ///
171 /// [`crate::Error::RuntimeState`] when the driver refuses the copy.
172 #[doc(hidden)]
173 /// # Safety
174 ///
175 /// `destination` and `source` must be device addresses this session's engine owns, `bytes`
176 /// must not exceed either allocation, and the ranges must not overlap unless
177 /// `cuMemcpyDtoDAsync` allows it. The copy is enqueued on this session's stream, so the
178 /// allocations must outlive the point where that work retires.
179 pub unsafe fn copy_bytes(
180 &self,
181 destination: *mut std::ffi::c_void,
182 source: *const std::ffi::c_void,
183 bytes: usize,
184 op: &'static str,
185 ) -> crate::Result<()> {
186 let status = unsafe {
187 cudarc::driver::sys::cuMemcpyDtoDAsync_v2(
188 destination as usize as cudarc::driver::sys::CUdeviceptr,
189 source as usize as cudarc::driver::sys::CUdeviceptr,
190 bytes,
191 self.scope.stream() as usize as *mut cudarc::driver::sys::CUstream_st,
192 )
193 };
194 if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
195 return Err(crate::Error::runtime_state(
196 op,
197 format!("cuMemcpyDtoDAsync failed with {status:?}"),
198 ));
199 }
200 Ok(())
201 }
202
203 /// Copy host `bytes` into fresh scratch of this runtime, on this session's stream.
204 ///
205 /// The host source is pageable and the copy is asynchronous, so the bytes are moved into the
206 /// scope's keepalives: the driver may still be reading them when the caller returns.
207 ///
208 /// # Errors
209 ///
210 /// [`crate::Error::RuntimeState`] when the allocation or the copy is refused.
211 #[doc(hidden)]
212 pub fn upload_bytes(&self, bytes: &[u8], op: &'static str) -> crate::Result<DeviceBytes<'s>> {
213 let device = self.alloc_bytes(bytes.len(), op)?;
214 let owned = bytes.to_vec();
215 let source = owned.as_ptr() as usize;
216 device.with_ptr(|destination| {
217 let status = unsafe {
218 cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
219 destination as usize as cudarc::driver::sys::CUdeviceptr,
220 source as *const std::ffi::c_void,
221 owned.len(),
222 self.scope.stream() as usize as *mut cudarc::driver::sys::CUstream_st,
223 )
224 };
225 if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
226 return Err(crate::Error::runtime_state(
227 op,
228 format!("cuMemcpyHtoDAsync failed with {status:?}"),
229 ));
230 }
231 Ok(())
232 })?;
233 // The driver may still be reading the host source, so it outlives this call through the
234 // scope's completion.
235 self.scope
236 .retain(Box::new(owned), 0)
237 .map_err(|error| crate::Error::runtime_state_source(op, error))?;
238 Ok(device)
239 }
240
241 /// Copy a device tensor into a host tensor of the same shape.
242 ///
243 /// # Errors
244 ///
245 /// [`crate::Error::RuntimeState`] when the tensor is not this runtime's allocation, and the
246 /// runtime's own failure when the copy or its barrier fails.
247 #[doc(hidden)]
248 pub fn download_tensor<T: TensorScalar>(
249 &self,
250 tensor: &TypedTensor<T>,
251 op: &'static str,
252 ) -> crate::Result<TypedTensor<T>> {
253 let (pointer, bytes) = self.address(tensor, op)?;
254 // One pinned staging buffer per runtime, reused by every later download; a download that
255 // cannot take the cache (a nested one) keeps its own buffer instead of blocking on it.
256 //
257 // Above the retained bound nothing is kept: the download stages through a temporary
258 // buffer that its `Drop` releases when this call returns, so a single large readback
259 // cannot pin that much memory for the runtime's lifetime (tenferro-rs #2440). The bound
260 // is read while holding the same lock a limit change takes, so a concurrent
261 // `set_pinned_staging_limit_bytes` cannot race a decision made from the old value.
262 let accounting = &self.runtime.inner.pinned_staging_accounting;
263 let mut cached = self.runtime.inner.pinned_staging.try_lock().ok();
264 if cached
265 .as_ref()
266 .is_some_and(|_| bytes.max(1) > accounting.limit())
267 {
268 cached = None;
269 }
270 let mut fallback = PinnedStaging::empty();
271 let destination = match cached.as_mut() {
272 Some(slot) => {
273 let entry = slot.get_or_insert_with(PinnedStaging::empty);
274 let pointer = entry.ensure(bytes, op, accounting)?;
275 // Only the cached buffer is retained; a temporary one never enters the count.
276 accounting.record_retained(entry.bytes());
277 pointer
278 }
279 None => fallback.ensure(bytes, op, accounting)?,
280 };
281 let status = unsafe {
282 cudarc::driver::sys::cuMemcpyDtoHAsync_v2(
283 destination,
284 pointer as cudarc::driver::sys::CUdeviceptr,
285 bytes,
286 self.scope.stream() as usize as *mut cudarc::driver::sys::CUstream_st,
287 )
288 };
289 if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
290 return Err(crate::Error::runtime_state(
291 op,
292 format!("cuMemcpyDtoHAsync failed with {status:?}"),
293 ));
294 }
295 self.synchronize()?;
296 let source = unsafe {
297 std::slice::from_raw_parts(
298 destination as *const T,
299 bytes / std::mem::size_of::<T>().max(1),
300 )
301 };
302 let mut values: Vec<T> = Vec::new();
303 values
304 .try_reserve_exact(source.len())
305 .map_err(|error| crate::Error::backend_source(op, error))?;
306 values.extend_from_slice(source);
307 TypedTensor::<T>::from_vec_col_major(tensor.shape().to_vec(), values)
308 .map_err(|error| crate::Error::runtime_state(op, error.to_string()))
309 }
310
311 /// Zero-filled device storage of `shape` for `T`, as a tensor of this runtime.
312 ///
313 /// # Errors
314 ///
315 /// [`crate::Error::RuntimeState`] when the allocation fails or the shape does not describe a
316 /// column-major tensor of `T`.
317 #[doc(hidden)]
318 pub fn alloc_output<T: TensorScalar>(&self, shape: &[usize]) -> crate::Result<TypedTensor<T>> {
319 const OP: &str = "cuda raw alloc_output";
320 let count = shape.iter().try_fold(1usize, |product, &extent| {
321 product
322 .checked_mul(extent)
323 .ok_or_else(|| crate::Error::runtime_state(OP, "the element count overflows usize"))
324 })?;
325 let bytes = count
326 .checked_mul(std::mem::size_of::<T>().max(1))
327 .ok_or_else(|| crate::Error::runtime_state(OP, "the byte length overflows usize"))?;
328 let buffer = self
329 .scope
330 .alloc_zeroed(bytes, 256)
331 .map_err(|error| crate::Error::runtime_state_source(OP, error))?;
332 // The returned tensor owns the allocation, so its own drop frees it — in this runtime's
333 // stream order, after any vendor call the session enqueued.
334 let storage = super::storage::fresh_storage(&self.runtime, buffer);
335 TypedTensor::<T>::from_buffer_col_major(
336 shape.to_vec(),
337 tenferro_tensor::StorageBuffer::Backend(Box::new(storage)),
338 super::transfer::placement(&self.runtime),
339 )
340 .map_err(|error| crate::Error::runtime_state(OP, error.to_string()))
341 }
342
343 /// A read-only device view of `tensor`.
344 ///
345 /// # Errors
346 ///
347 /// [`crate::Error::RuntimeState`] when the tensor is not this runtime's allocation.
348 #[doc(hidden)]
349 pub fn tensor<T: TensorScalar>(
350 &self,
351 tensor: &TypedTensor<T>,
352 ) -> crate::Result<TensorRef<'_, T>> {
353 let (pointer, bytes) = self.address(tensor, "cuda raw tensor")?;
354 Ok(TensorRef {
355 pointer,
356 bytes,
357 _tensor: std::marker::PhantomData,
358 })
359 }
360
361 /// A writable device view of `tensor`.
362 ///
363 /// # Errors
364 ///
365 /// [`crate::Error::RuntimeState`] when the tensor is not this runtime's allocation.
366 #[doc(hidden)]
367 pub fn tensor_mut<T: TensorScalar>(
368 &self,
369 tensor: &TypedTensor<T>,
370 ) -> crate::Result<TensorMut<'_, T>> {
371 let (pointer, bytes) = self.address(tensor, "cuda raw tensor mut")?;
372 Ok(TensorMut {
373 pointer,
374 bytes,
375 _tensor: std::marker::PhantomData,
376 })
377 }
378
379 /// Prove `tensor` is one of this runtime's allocations, for a vendor call that reads it.
380 ///
381 /// # Errors
382 ///
383 /// [`crate::Error::RuntimeState`] when the tensor is not this runtime's allocation.
384 #[doc(hidden)]
385 /// # Safety
386 ///
387 /// The returned address stays valid only while `tensor`'s allocation is resident on this
388 /// engine; the caller must not outlive it.
389 pub unsafe fn retain_tensor<T: TensorScalar>(
390 &self,
391 tensor: &TypedTensor<T>,
392 op: &'static str,
393 ) -> crate::Result<DeviceBytes<'static>> {
394 let (pointer, bytes) = self.address(tensor, op)?;
395 Ok(DeviceBytes {
396 pointer,
397 bytes,
398 _session: std::marker::PhantomData,
399 })
400 }
401
402 /// This session's vendor resource of type `T`, created by `init` on first use.
403 ///
404 /// One slot per type: the cache key carries the type name, so two resources of different types
405 /// in one engine cannot collide, and the same handle is reused by every later session.
406 ///
407 /// # Examples
408 ///
409 /// ```no_run
410 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
411 /// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaEngine};
412 ///
413 /// # #[cfg(feature = "cuda")]
414 /// if gpu_available() {
415 /// let mut engine = CudaEngine::open(CudaDeviceId::from_ordinal(0))?;
416 /// // The guard borrows the session, so it stays inside the closure.
417 /// let total = engine.with_raw("resource", |session| {
418 /// let counter = session.resource(|| Ok::<u64, tenferro_tensor::Error>(7))?;
419 /// Ok(*counter + 1)
420 /// })?;
421 /// assert_eq!(total, 8);
422 /// }
423 /// # Ok(())
424 /// # }
425 /// ```
426 ///
427 /// # Panics
428 ///
429 /// Only if storing the initialised resource succeeded and it is then missing from the cache,
430 /// which is an internal invariant of the store above.
431 ///
432 /// # Errors
433 ///
434 /// The initialiser's failure, and the engine's when the cache is poisoned.
435 pub fn resource<T: Send + Sync + 'static>(
436 &self,
437 init: impl FnOnce() -> crate::Result<T>,
438 ) -> crate::Result<CudaResourceGuard<'_, T>> {
439 let key = tenferro_runtime::ExtensionCacheKey::new(
440 "tenferro.cuda.session",
441 std::any::type_name::<T>(),
442 0,
443 );
444 let mut store = self.caches.borrow_mut();
445 if store.get::<T>(&key).is_none() {
446 let value = init()?;
447 self.resource_inits.set(self.resource_inits.get() + 1);
448 store.put(key, value, 0);
449 }
450 let entry = std::cell::RefMut::map(store, |store| {
451 store
452 .get_mut::<T>(&key)
453 .expect("the resource was stored above")
454 });
455 Ok(CudaResourceGuard { entry })
456 }
457
458 /// Wait for the work enqueued on this session's stream.
459 ///
460 /// # Errors
461 ///
462 /// [`crate::Error::RuntimeStateSource`] when the stream cannot be synchronised.
463 #[doc(hidden)]
464 pub fn synchronize(&self) -> crate::Result<()> {
465 let stream = self.scope.stream() as usize as *mut cudarc::driver::sys::CUstream_st;
466 // SAFETY: the scope hands out this runtime's own stream; a barrier on it neither reads nor
467 // writes an operand.
468 let status = unsafe { cudarc::driver::sys::cuStreamSynchronize(stream) };
469 if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
470 return Err(crate::Error::runtime_state(
471 "cuda raw synchronize",
472 format!("cuStreamSynchronize failed with {status:?}"),
473 ));
474 }
475 Ok(())
476 }
477
478 fn address<T: TensorScalar>(
479 &self,
480 tensor: &TypedTensor<T>,
481 op: &'static str,
482 ) -> crate::Result<(u64, usize)> {
483 let storage = match tensor.buffer() {
484 StorageBuffer::Backend(buffer) => buffer
485 .as_ref()
486 .as_any()
487 .downcast_ref::<CudaStorage>()
488 .ok_or_else(|| {
489 crate::Error::runtime_state(op, "expected a CUDA allocation of this backend")
490 })?,
491 StorageBuffer::Host(_) => {
492 return Err(crate::Error::runtime_state(
493 op,
494 "expected a device tensor, not host storage",
495 ))
496 }
497 };
498 if storage.domain() != self.runtime.allocation_domain_id() {
499 return Err(crate::Error::runtime_state(
500 op,
501 "the tensor belongs to another runtime",
502 ));
503 }
504 let pointer = self
505 .scope
506 .register(storage.allocation())
507 .map_err(|error| crate::Error::runtime_state_source(op, error))?;
508 Ok((pointer, storage.bytes()))
509 }
510}
511
512/// Marks a [`NativeSessionRef`](tenferro_tensor::NativeSessionRef) as this engine's, so an
513/// extension can reach the concrete engine through `&mut dyn Engine`.
514#[derive(Debug, Clone, Copy)]
515pub(crate) struct CudaEngineMarker;
516
517/// Visit this engine through the tensor crate's engine trait, as an extension does.
518///
519/// Returns `None` when `engine` is not this backend's engine.
520#[doc(hidden)]
521pub fn with_cuda_engine<B, R>(engine: &mut B, f: impl FnOnce(&mut CudaEngine) -> R) -> Option<R>
522where
523 B: tenferro_tensor::Engine + ?Sized,
524{
525 let pointer = engine
526 .native_engine()?
527 .into_marked_ptr::<CudaEngineMarker>()?;
528 // SAFETY: only this crate's `native_engine` creates a token carrying this marker, and it points
529 // that token at a live `CudaEngine`; the `&mut B` keeps the exclusive borrow for the visit.
530 Some(f(unsafe { pointer.cast::<CudaEngine>().as_mut() }))
531}
532
533impl CudaEngine {
534 /// Run `f` inside this engine's raw scope.
535 ///
536 /// Every vendor call the closure makes must be enqueued on [`Session::stream`]; a buffer the
537 /// caller drops afterwards is freed in that stream's order, so the call still sees it.
538 ///
539 /// # Errors
540 ///
541 /// The closure's own failure, and the runtime's when the scope cannot be opened.
542 #[doc(hidden)]
543 pub fn with_raw<R>(
544 &mut self,
545 _op: &'static str,
546 f: impl FnOnce(&mut Session<'_>) -> crate::Result<R>,
547 ) -> crate::Result<R> {
548 // The runtime handle is cloned first so the engine's caches can be borrowed mutably for the
549 // same call.
550 let runtime = self.runtime().clone();
551 // The engine's own resources, not the store a run lends away: a vendor handle has to
552 // outlive the run that created it.
553 let caches = &self.vendor_resources;
554 let resource_inits = &self.vendor_resource_inits;
555 // SAFETY: the closure only reads the addresses the session registers and enqueues its work
556 // on this scope's stream.
557 unsafe {
558 runtime
559 .with_raw_locked(Vec::new(), |scope| {
560 let mut session = Session {
561 runtime: runtime.clone(),
562 resource_inits,
563 scope,
564 caches,
565 };
566 f(&mut session)
567 })
568 .and_then(|result| result)
569 }
570 }
571}
572
573/// A pinned host buffer used as the staging area of downloads.
574///
575/// The runtime keeps one of these and reuses it, growing on demand: pinning and unpinning a fresh
576/// buffer per download cost more than the copy it staged (a 4-byte `cuMemHostAlloc` measured
577/// 1.9 ms beside a 19 us device-to-host copy in the daily GPU linalg profile).
578pub(crate) struct PinnedStaging {
579 pointer: *mut std::ffi::c_void,
580 bytes: usize,
581}
582
583/// What a runtime's download staging retains, and the bound it retains it under.
584///
585/// The buffer is page-locked host memory, so an unbounded one holds memory the allocator cannot
586/// reclaim: a single large download would keep that much pinned for the runtime's lifetime. The
587/// bound decides what is kept; a download above it stages through a temporary buffer that is
588/// released when the call returns.
589pub(crate) struct StagingAccounting {
590 /// Bytes the cached buffer currently holds.
591 retained: std::sync::atomic::AtomicUsize,
592 /// Pinned allocations this runtime made for staging, growth included.
593 allocations: std::sync::atomic::AtomicU64,
594 /// Largest download whose staging is retained.
595 limit: std::sync::atomic::AtomicUsize,
596}
597
598impl StagingAccounting {
599 pub(crate) fn new(limit: usize) -> Self {
600 Self {
601 retained: std::sync::atomic::AtomicUsize::new(0),
602 allocations: std::sync::atomic::AtomicU64::new(0),
603 limit: std::sync::atomic::AtomicUsize::new(limit),
604 }
605 }
606
607 pub(crate) fn retained(&self) -> usize {
608 self.retained.load(std::sync::atomic::Ordering::Relaxed)
609 }
610
611 pub(crate) fn allocations(&self) -> u64 {
612 self.allocations.load(std::sync::atomic::Ordering::Relaxed)
613 }
614
615 pub(crate) fn limit(&self) -> usize {
616 self.limit.load(std::sync::atomic::Ordering::Relaxed)
617 }
618
619 pub(crate) fn set_limit(&self, limit: usize) {
620 self.limit
621 .store(limit, std::sync::atomic::Ordering::Relaxed);
622 }
623
624 /// Publish what the cached buffer holds. The runtime resets it when it releases the buffer.
625 pub(crate) fn record_retained(&self, retained: usize) {
626 self.retained
627 .store(retained, std::sync::atomic::Ordering::Relaxed);
628 }
629}
630
631// SAFETY: the buffer is a plain page-locked host allocation with no interior aliasing. It lives
632// behind the runtime state's mutex, so one thread owns it at a time; the mutex needs `Send` only.
633unsafe impl Send for PinnedStaging {}
634
635impl PinnedStaging {
636 /// The empty staging entry the runtime fills on its first download.
637 pub(crate) fn empty() -> Self {
638 Self {
639 pointer: std::ptr::null_mut(),
640 bytes: 0,
641 }
642 }
643
644 /// Bytes this entry retains, zero for an empty one.
645 pub(crate) fn bytes(&self) -> usize {
646 self.bytes
647 }
648
649 /// The raw pinned allocation, which the runtime frees inside its own scope.
650 pub(crate) fn pointer(&self) -> *mut std::ffi::c_void {
651 self.pointer
652 }
653
654 /// A writable staging area of at least `bytes` bytes, growing once if needed.
655 ///
656 /// Growth allocates the replacement before releasing the old buffer, so a failed allocation
657 /// leaves both the buffer and the accounting exactly as they were; every path publishes the
658 /// size actually retained.
659 fn ensure(
660 &mut self,
661 bytes: usize,
662 op: &'static str,
663 accounting: &StagingAccounting,
664 ) -> crate::Result<*mut std::ffi::c_void> {
665 let bytes = bytes.max(1);
666 if self.pointer.is_null() || self.bytes < bytes {
667 let mut pointer: *mut std::ffi::c_void = std::ptr::null_mut();
668 let status = unsafe {
669 cudarc::driver::sys::cuMemHostAlloc(
670 &mut pointer,
671 bytes,
672 cudarc::driver::sys::CU_MEMHOSTALLOC_PORTABLE,
673 )
674 };
675 if status != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
676 return Err(crate::Error::runtime_state(
677 op,
678 format!("cuMemHostAlloc failed with {status:?}"),
679 ));
680 }
681 // SAFETY: `pointer` is null or this entry's own live pinned allocation, released once
682 // here after its replacement exists.
683 unsafe {
684 if !self.pointer.is_null() {
685 let _ = cudarc::driver::sys::cuMemFreeHost(self.pointer);
686 }
687 }
688 self.pointer = pointer;
689 self.bytes = bytes;
690 accounting
691 .allocations
692 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
693 }
694 Ok(self.pointer)
695 }
696}
697
698impl Drop for PinnedStaging {
699 fn drop(&mut self) {
700 // SAFETY: the pointer came from `cuMemHostAlloc` and is freed once here.
701 unsafe {
702 let _ = cudarc::driver::sys::cuMemFreeHost(self.pointer);
703 }
704 self.bytes = 0;
705 }
706}
707
708/// A vendor resource lent from the engine's extension caches for the session's life.
709///
710/// # Examples
711///
712/// ```no_run
713/// use tenferro_gpu::cuda::raw::CudaResourceGuard;
714///
715/// fn handle_lifetime(guard: &CudaResourceGuard<'_, u64>) -> u64 {
716/// // The guard dereferences to the cached resource and releases it when dropped.
717/// **guard + 1
718/// }
719///
720/// # fn main() {}
721/// ```
722pub struct CudaResourceGuard<'s, T> {
723 entry: std::cell::RefMut<'s, T>,
724}
725
726impl<T> std::ops::Deref for CudaResourceGuard<'_, T> {
727 type Target = T;
728
729 fn deref(&self) -> &T {
730 &self.entry
731 }
732}
733
734impl<T> std::ops::DerefMut for CudaResourceGuard<'_, T> {
735 fn deref_mut(&mut self) -> &mut T {
736 &mut self.entry
737 }
738}
739
740impl<T: std::fmt::Debug> std::fmt::Debug for CudaResourceGuard<'_, T> {
741 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
742 self.entry.fmt(formatter)
743 }
744}