tenferro_gpu/cuda/runtime.rs
1//! The runtime handle: one `t4a-cuda-runtime` runtime, one device, one allocation
2//! domain, one submission lock for explicit host transfers.
3
4use std::sync::{Arc, Mutex};
5
6use tenferro_tensor::AllocationDomainId;
7
8use crate::cuda::{CudaDeviceError, CudaDeviceId, CudaDeviceInfo};
9
10/// Identity of one opened runtime; equal only for clones of the same handle.
11#[derive(Debug, Clone)]
12#[doc(hidden)]
13pub struct CudaRuntimeIdentity {
14 marker: Arc<u8>,
15}
16
17impl CudaRuntimeIdentity {
18 fn fresh() -> Self {
19 Self {
20 marker: Arc::new(0),
21 }
22 }
23}
24
25impl PartialEq for CudaRuntimeIdentity {
26 fn eq(&self, other: &Self) -> bool {
27 Arc::ptr_eq(&self.marker, &other.marker)
28 }
29}
30
31impl Eq for CudaRuntimeIdentity {}
32
33impl std::hash::Hash for CudaRuntimeIdentity {
34 fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
35 // The marker allocation is retained by every clone, so its address is
36 // move- and clone-invariant while an identity is witnessed.
37 state.write_usize(Arc::as_ptr(&self.marker) as usize);
38 }
39}
40
41std::thread_local! {
42 /// Addresses of the runtimes whose context this thread has already activated.
43 ///
44 /// The raw scope takes the runtime's submission lock, so a second activation on the same
45 /// thread would block forever; nesting is legal because the context is already current.
46 static ACTIVATED_CONTEXTS: std::cell::RefCell<Vec<usize>> =
47 const { std::cell::RefCell::new(Vec::new()) };
48}
49
50/// One activation level of a runtime's context on this thread.
51pub(crate) struct ActivatedContext {
52 key: usize,
53 /// Whether this call found the context already active on this thread.
54 nested: bool,
55 /// `Some` when this call pushed a level; dropping it pops the level again, including on a
56 /// panic, so a leaked level cannot make a later activation silently skip the context.
57 pushed: bool,
58}
59
60impl ActivatedContext {
61 /// Enter `runtime`'s context, reporting whether this thread was already inside it.
62 fn enter(runtime: &CudaRuntime) -> ActivatedContext {
63 let key = Arc::as_ptr(&runtime.inner) as usize;
64 let nested = ACTIVATED_CONTEXTS.with(|active| {
65 let mut active = active.borrow_mut();
66 if active.contains(&key) {
67 true
68 } else {
69 active.push(key);
70 false
71 }
72 });
73 ActivatedContext {
74 key,
75 nested,
76 pushed: !nested,
77 }
78 }
79}
80
81impl Drop for ActivatedContext {
82 fn drop(&mut self) {
83 if self.pushed {
84 ACTIVATED_CONTEXTS.with(|active| {
85 active.borrow_mut().retain(|entry| *entry != self.key);
86 });
87 }
88 }
89}
90
91/// Default bound on the retained download staging: 8 MiB.
92///
93/// Pinning costs about 1.9 ms, so the buffer is kept — but it is page-locked memory the allocator
94/// cannot reclaim. 8 MiB covers the readbacks this path exists for (solver `info` words, norms,
95/// ranks, a 1024x1024 f64 matrix) with room to spare; a larger download stages through a temporary
96/// buffer and leaves nothing resident. Raise it with
97/// [`CudaRuntime::set_pinned_staging_limit_bytes`] when large downloads repeat.
98pub const DEFAULT_PINNED_STAGING_LIMIT_BYTES: usize = 8 * 1024 * 1024;
99
100/// What a runtime's download staging retains, and the bound it retains it under.
101///
102/// # Examples
103///
104/// ```
105/// # fn main() -> Result<(), tenferro_gpu::cuda::CudaEngineError> {
106/// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaRuntime};
107///
108/// # if gpu_available() {
109/// let runtime = CudaRuntime::new(CudaDeviceId::from_ordinal(0))?;
110/// let stats = runtime.pinned_staging_stats();
111/// if stats.retained_bytes > stats.limit_bytes {
112/// runtime.clear_pinned_staging();
113/// }
114/// println!("{stats:?}");
115/// # }
116/// # Ok(())
117/// # }
118/// ```
119#[derive(Clone, Copy, Debug, PartialEq, Eq)]
120pub struct PinnedStagingStats {
121 /// Bytes the cached staging buffer holds right now.
122 pub retained_bytes: usize,
123 /// Pinned host allocations this runtime made for staging, cumulative: the first download, each
124 /// later growth, and each temporary buffer used above the bound or re-pinned after a clear.
125 pub allocations: u64,
126 /// Largest download whose staging is retained.
127 pub limit_bytes: usize,
128}
129
130/// State shared by every handle to one runtime.
131pub(crate) struct CudaRuntimeState {
132 /// Pinned host staging reused by explicit downloads. Allocating and freeing one per download
133 /// cost far more than the copy (a 4-byte `cuMemHostAlloc` measured 1.9 ms beside a 19 us
134 /// device-to-host copy), so one buffer lives here and grows on demand.
135 pub(crate) pinned_staging: Mutex<Option<crate::cuda::raw::PinnedStaging>>,
136 /// What that buffer retains, the bound it retains it under, and how many pinned allocations
137 /// this runtime made for staging.
138 pub(crate) pinned_staging_accounting: crate::cuda::raw::StagingAccounting,
139
140 /// The execution owner. Explicit transfers take this lock for one admission;
141 /// numerical submission borrows it mutably through the engine.
142 pub(crate) runtime: Mutex<t4a_cuda_runtime::Runtime>,
143 pub(crate) device_id: CudaDeviceId,
144 pub(crate) device_info: CudaDeviceInfo,
145 pub(crate) allocation_domain: AllocationDomainId,
146 pub(crate) identity: CudaRuntimeIdentity,
147 /// The allocation pool's retention budget this runtime opened with, in bytes.
148 pub(crate) pool_release_threshold: usize,
149 /// Handles dropped while their work was still queued. The runtime's retirement
150 /// already owns the staging; this queue owns only the caller's packages (a pinned
151 /// source, a tensor handle) until their completion is observed.
152 pub(crate) detached: Mutex<Vec<Detached>>,
153 /// How many scheduled runs are currently enqueueing nodes on this runtime.
154 ///
155 /// While it is non-zero a run owns the completion proof: every node it enqueues records a
156 /// stream-ordered checkpoint, and the run's retirement is the barrier, so an operation's own
157 /// launch does not have to wait. Zero means a standalone call, which proves its own launch.
158 pub(crate) deferred_completions: std::sync::atomic::AtomicUsize,
159 /// Completion checkpoints recorded on this runtime's stream, per node of a scheduled run or
160 /// by a transfer. A coalesced run records none: submission order is its order (#2396).
161 pub(crate) completion_checkpoints: std::sync::atomic::AtomicU64,
162 /// Stream barriers this runtime ran, whoever asked: an operation proving its own launch, a
163 /// download, or a caller's explicit synchronize.
164 pub(crate) stream_waits: std::sync::atomic::AtomicU64,
165}
166
167/// One dropped handle retained until its completion is observed.
168pub(crate) struct Detached {
169 /// The caller's package: the queued copy still reads or writes it, so it stays owned
170 /// here until the completion below proves the work done.
171 #[allow(dead_code)]
172 pub(crate) keepalive: Box<dyn std::any::Any + Send>,
173 pub(crate) completion: Option<t4a_cuda_runtime::Completion>,
174}
175
176/// The runtime configuration this build opens with.
177///
178/// The artifact directory, its bound and the allocation pool's retention budget are read from
179/// the environment on every open: `TENFERRO_CUDA_ARTIFACT_CACHE`,
180/// `TENFERRO_CUDA_ARTIFACT_CACHE_MIB` and `TENFERRO_CUDA_POOL_RELEASE_MIB`.
181///
182/// `TENFERRO_CUDA_ARTIFACT_CACHE` names the directory that persists compiled kernel images, so a
183/// later process reads them back instead of paying NVRTC again: a whole module is about 8 s per
184/// process on an A100, and a warm read is a few milliseconds. The value wins over the default;
185/// `none`, `off` and `memory` select the in-memory cache only; unset selects
186/// `$XDG_CACHE_HOME/tenferro/t4a-cuda-artifacts`, else `$HOME/.cache/tenferro/t4a-cuda-artifacts`
187/// (`%LOCALAPPDATA%\tenferro\t4a-cuda-artifacts` on Windows), and falls back to the in-memory
188/// cache when no absolute per-user root is set or the directory cannot be created. A newly created
189/// directory is mode 0700 on Unix.
190///
191/// The directory is bounded, never unlimited: at most [`artifact_cache_max_bytes`] bytes of
192/// images, 256 MiB unless `TENFERRO_CUDA_ARTIFACT_CACHE_MIB` says otherwise. Publishing past the
193/// bound removes the least recently used images first, best effort: a removal that fails is
194/// dropped, so the directory can stay above its bound. [`CudaRuntime::artifact_dir_evictions`]
195/// counts what was actually removed, and [`CudaRuntime::artifact_dir_stats`] and
196/// [`CudaRuntime::clear_artifact_dir`] read and empty it.
197///
198/// `TENFERRO_CUDA_POOL_RELEASE_MIB` overrides the allocation pool's retention budget, the idle
199/// device memory the pool keeps mapped instead of handing back to the driver. See
200/// [`pool_release_threshold_bytes`].
201fn default_config(total_memory_bytes: u64) -> t4a_cuda_runtime::Config {
202 let artifact_cache_dir = artifact_cache_dir(
203 std::env::var_os("TENFERRO_CUDA_ARTIFACT_CACHE").as_deref(),
204 |name| std::env::var_os(name),
205 )
206 .and_then(usable_artifact_cache_dir);
207 t4a_cuda_runtime::Config {
208 artifact_cache_dir,
209 pool_release_threshold: pool_release_threshold_bytes(total_memory_bytes),
210 ..t4a_cuda_runtime::Config::default()
211 }
212}
213
214/// The least idle device memory the pool keeps mapped, whatever the device size.
215const MIN_POOL_RELEASE_BYTES: usize = 256 << 20;
216/// The most idle device memory the pool keeps mapped, whatever the device size.
217const MAX_POOL_RELEASE_BYTES: usize = 4 << 30;
218/// The share of the device's memory the pool may keep mapped.
219const POOL_RELEASE_SHARE: u64 = 8;
220
221/// Idle device memory the allocation pool keeps mapped instead of returning to the driver.
222///
223/// The driver's own default is a fixed 256 MiB, which is smaller than the working set of a single
224/// large operation: an eager 3072^3 f64 matmul holds about 430 MiB of 72 MiB buffers, so every
225/// call's output allocation missed the pool and re-mapped its pages — 864 us of
226/// `cuMemAllocFromPoolAsync` per call, 87 of 87 calls on the A100 host, reproducing in plain CUDA
227/// once six buffers are live (#2381). A budget scaled to the device keeps those pages.
228///
229/// `TENFERRO_CUDA_POOL_RELEASE_MIB` overrides the result, in MiB. The budget is a ceiling on
230/// retained *idle* memory, not a reservation: nothing is allocated before it is needed.
231fn pool_release_threshold_bytes(total_memory_bytes: u64) -> usize {
232 let override_mib = std::env::var("TENFERRO_CUDA_POOL_RELEASE_MIB")
233 .ok()
234 .and_then(|value| value.trim().parse::<u64>().ok());
235 pool_release_threshold_from(override_mib, total_memory_bytes)
236}
237
238/// [`pool_release_threshold_bytes`] with the environment's override supplied explicitly.
239fn pool_release_threshold_from(override_mib: Option<u64>, total_memory_bytes: u64) -> usize {
240 if let Some(mib) = override_mib {
241 return usize::try_from(mib.saturating_mul(1 << 20))
242 .unwrap_or(usize::MAX)
243 .clamp(MIN_POOL_RELEASE_BYTES, MAX_POOL_RELEASE_BYTES);
244 }
245 let share = usize::try_from(total_memory_bytes / POOL_RELEASE_SHARE).unwrap_or(usize::MAX);
246 share.clamp(MIN_POOL_RELEASE_BYTES, MAX_POOL_RELEASE_BYTES)
247}
248
249/// The bound on the persistent artifact directory, in bytes.
250///
251/// `TENFERRO_CUDA_ARTIFACT_CACHE_MIB` (`value`) sets it in MiB. `0` is a bound like any other, not
252/// a read-only mode: the runtime applies a bound through
253/// [`t4a_cuda_runtime::Runtime::set_artifact_dir_max_bytes`], which trims the directory as it
254/// records the bound. That trim is best effort — a removal that fails is dropped — so `0` stops
255/// new images being published and does not guarantee the directory is empty; an image a failed
256/// trim left behind is still served. A process that wants no persistent directory asks for that
257/// with `TENFERRO_CUDA_ARTIFACT_CACHE=none`. Unset, or a value that is not a whole number of MiB,
258/// keeps [`t4a_cuda_runtime::DEFAULT_ARTIFACT_DIR_BYTES`] (256 MiB, the in-memory artifact
259/// budget): there is no spelling of "unlimited".
260fn artifact_cache_max_bytes(value: Option<&std::ffi::OsStr>) -> u64 {
261 value
262 .and_then(std::ffi::OsStr::to_str)
263 .and_then(|value| value.trim().parse::<u64>().ok())
264 .map_or(t4a_cuda_runtime::DEFAULT_ARTIFACT_DIR_BYTES, |mib| {
265 mib.saturating_mul(1 << 20)
266 })
267}
268
269/// The one operation opening a runtime performs on its artifact cache.
270///
271/// The cache lives behind a device, so the open applies the bound through this trait and a test
272/// drives a fake instead: a compile of the decision alone would not show that it is applied.
273trait ArtifactDirBound {
274 fn set_artifact_dir_max_bytes(&mut self, max_bytes: u64);
275}
276
277impl ArtifactDirBound for t4a_cuda_runtime::Runtime {
278 fn set_artifact_dir_max_bytes(&mut self, max_bytes: u64) {
279 // The dependency's own method, not this trait's: no recursion.
280 t4a_cuda_runtime::Runtime::set_artifact_dir_max_bytes(self, max_bytes);
281 }
282}
283
284/// Apply `TENFERRO_CUDA_ARTIFACT_CACHE_MIB` to `cache`, the open's only cache mutation.
285fn apply_artifact_cache_bound(
286 cache: &mut dyn ArtifactDirBound,
287 read: impl Fn(&str) -> Option<std::ffi::OsString>,
288) {
289 cache.set_artifact_dir_max_bytes(artifact_cache_max_bytes(
290 read("TENFERRO_CUDA_ARTIFACT_CACHE_MIB").as_deref(),
291 ));
292}
293
294/// `tenferro`'s directory under the user's cache root, which holds compiled kernel images.
295const ARTIFACT_CACHE_PATH: &str = "tenferro/t4a-cuda-artifacts";
296
297/// The persistent artifact directory the environment selects, or `None` for memory only.
298///
299/// `read` supplies the environment so the fallback chain is testable without one.
300fn artifact_cache_dir(
301 value: Option<&std::ffi::OsStr>,
302 read: impl Fn(&str) -> Option<std::ffi::OsString>,
303) -> Option<std::path::PathBuf> {
304 if let Some(value) = value {
305 return match value.to_str() {
306 Some(value) => {
307 let value = value.trim();
308 if value.is_empty() || matches!(value, "none" | "off" | "memory") {
309 None
310 } else {
311 Some(std::path::PathBuf::from(value))
312 }
313 }
314 // Not UTF-8, so it cannot be one of the tokens: it names a directory as it is.
315 None => Some(std::path::PathBuf::from(value)),
316 };
317 }
318 cache_root(read).map(|root| root.join(ARTIFACT_CACHE_PATH))
319}
320
321/// This user's cache root.
322///
323/// `$XDG_CACHE_HOME` when it is absolute, else `$HOME/.cache`; a relative `XDG_CACHE_HOME` is
324/// ignored, as the specification requires, so the cache location never depends on the working
325/// directory. Windows has no XDG and uses `%LOCALAPPDATA%`, which is already a per-user root.
326fn cache_root(read: impl Fn(&str) -> Option<std::ffi::OsString>) -> Option<std::path::PathBuf> {
327 let root = |name: &str| {
328 read(name)
329 .filter(|value| !value.is_empty())
330 .map(std::path::PathBuf::from)
331 .filter(|dir| dir.is_absolute())
332 };
333 if cfg!(windows) {
334 return root("LOCALAPPDATA");
335 }
336 root("XDG_CACHE_HOME").or_else(|| root("HOME").map(|home| home.join(".cache")))
337}
338
339/// The same directory, once this process has created it or found it present.
340///
341/// A home it cannot create the directory under degrades to the in-memory cache with one warning,
342/// never to a failed operation, so nothing here returns an error into the launch path.
343fn usable_artifact_cache_dir(dir: std::path::PathBuf) -> Option<std::path::PathBuf> {
344 match create_private_dir(&dir) {
345 Ok(()) => Some(dir),
346 Err(error) => {
347 ARTIFACT_CACHE_WARNING.call_once(|| {
348 write_best_effort_warning(
349 &mut std::io::stderr(),
350 format_args!(
351 "tenferro-gpu: cannot create the artifact cache directory {}: {error}; \
352 compiled kernels stay in memory for this process",
353 dir.display()
354 ),
355 );
356 });
357 None
358 }
359 }
360}
361
362/// Write one warning line to `sink`, dropping it if the write fails.
363///
364/// `eprintln!` panics when stderr is closed, and a diagnostic on a best-effort path must never
365/// turn into a panic in the library.
366pub(crate) fn write_best_effort_warning(
367 sink: &mut impl std::io::Write,
368 message: std::fmt::Arguments<'_>,
369) {
370 let _ = writeln!(sink, "{message}");
371}
372
373/// One warning per process, however many runtimes open against an unusable directory.
374static ARTIFACT_CACHE_WARNING: std::sync::Once = std::sync::Once::new();
375
376/// Create `dir` and any missing parent. A directory this creates is mode 0700 on Unix; an existing
377/// one keeps its own mode, so a shared read-only cache stays readable and its writes stay dropped.
378fn create_private_dir(dir: &std::path::Path) -> std::io::Result<()> {
379 let mut builder = std::fs::DirBuilder::new();
380 builder.recursive(true);
381 #[cfg(unix)]
382 {
383 use std::os::unix::fs::DirBuilderExt;
384 builder.mode(0o700);
385 }
386 builder.create(dir)
387}
388
389/// A cheap, cloneable handle to one opened CUDA runtime.
390///
391/// The handle carries no submission authority of its own: explicit host transfers
392/// take the runtime's submission lock for the duration of one admission, and
393/// numerical submission belongs to the [`CudaEngine`](super::CudaEngine) that owns
394/// the same state.
395#[derive(Clone)]
396#[doc(hidden)]
397pub struct CudaRuntime {
398 pub(crate) inner: Arc<CudaRuntimeState>,
399}
400
401impl CudaRuntime {
402 /// Open the CUDA runtime on `device_id` with the default configuration.
403 ///
404 /// # Errors
405 ///
406 /// [`CudaDeviceError`] when the device is unknown or the driver cannot open it.
407 #[doc(hidden)]
408 pub fn new(device_id: CudaDeviceId) -> Result<CudaRuntime, CudaDeviceError> {
409 let total_memory = crate::cuda::cuda_devices()?
410 .iter()
411 .find(|info| info.id() == device_id)
412 .map(crate::cuda::CudaDeviceInfo::total_memory_bytes)
413 .unwrap_or(0);
414 let runtime = Self::with_config(device_id, default_config(total_memory))?;
415 apply_artifact_cache_bound(&mut *runtime.submit(), |name| std::env::var_os(name));
416 Ok(runtime)
417 }
418
419 /// Open the CUDA runtime on `device_id` with an explicit configuration.
420 ///
421 /// The configuration bounds the runtime's staging chunk and pending budget, which is
422 /// what a caller uses to force chunked transfers.
423 ///
424 /// # Errors
425 ///
426 /// [`CudaDeviceError`] when the device is unknown, the configuration is invalid, or
427 /// the driver cannot open the device.
428 #[doc(hidden)]
429 pub fn with_config(
430 device_id: CudaDeviceId,
431 config: t4a_cuda_runtime::Config,
432 ) -> Result<CudaRuntime, CudaDeviceError> {
433 let devices = crate::cuda::cuda_devices()?;
434 let info = devices
435 .iter()
436 .find(|info| info.id() == device_id)
437 .cloned()
438 .ok_or_else(|| CudaDeviceError::Unavailable {
439 requested: device_id,
440 discovered: devices.into_boxed_slice(),
441 })?;
442 let pool_release_threshold = config.pool_release_threshold;
443 let runtime = t4a_cuda_runtime::Runtime::open(device_id.ordinal() as usize, config)
444 .map_err(|error| CudaDeviceError::Initialization {
445 device: device_id,
446 operation: "t4a_cuda_runtime::Runtime::open",
447 source: Box::new(error),
448 })?;
449 Ok(CudaRuntime {
450 inner: Arc::new(CudaRuntimeState {
451 runtime: Mutex::new(runtime),
452 device_id,
453 device_info: info,
454 allocation_domain: AllocationDomainId::fresh(),
455 pool_release_threshold,
456 identity: CudaRuntimeIdentity::fresh(),
457 detached: Mutex::new(Vec::new()),
458 deferred_completions: std::sync::atomic::AtomicUsize::new(0),
459 stream_waits: std::sync::atomic::AtomicU64::new(0),
460 completion_checkpoints: std::sync::atomic::AtomicU64::new(0),
461 pinned_staging: Mutex::new(None),
462 pinned_staging_accounting: crate::cuda::raw::StagingAccounting::new(
463 DEFAULT_PINNED_STAGING_LIMIT_BYTES,
464 ),
465 }),
466 })
467 }
468
469 /// The device this runtime owns.
470 #[doc(hidden)]
471 pub fn device_id(&self) -> CudaDeviceId {
472 self.inner.device_id
473 }
474
475 /// The device metadata this runtime was opened with.
476 #[doc(hidden)]
477 pub fn device_info(&self) -> &CudaDeviceInfo {
478 &self.inner.device_info
479 }
480
481 /// The ordinal of the owning device.
482 #[doc(hidden)]
483 pub fn device_ordinal(&self) -> usize {
484 self.inner.device_id.ordinal() as usize
485 }
486
487 /// The identity of this runtime; distinct for every opened runtime.
488 #[doc(hidden)]
489 pub(crate) fn identity(&self) -> &CudaRuntimeIdentity {
490 &self.inner.identity
491 }
492
493 /// The shared-allocation domain every buffer of this runtime belongs to.
494 #[doc(hidden)]
495 pub fn allocation_domain_id(&self) -> AllocationDomainId {
496 self.inner.allocation_domain
497 }
498
499 /// NVRTC compilation attempts this runtime's artifact cache performed.
500 pub fn compile_attempts(&self) -> u64 {
501 self.submit().compile_attempts()
502 }
503
504 /// Module loads this runtime performed.
505 pub fn module_loads(&self) -> u64 {
506 self.submit().module_loads()
507 }
508
509 /// Artifact-cache hits this runtime served.
510 pub fn artifact_cache_hits(&self) -> u64 {
511 self.submit().artifact_cache_hits()
512 }
513
514 /// Images and bytes in the persistent artifact directory; empty for an in-memory cache.
515 ///
516 /// # Errors
517 ///
518 /// The [`std::io::Error`] from listing a directory that exists but cannot be read, such as
519 /// `ErrorKind::PermissionDenied`. A directory that does not exist yet is not an error; it
520 /// reports zero entries.
521 pub fn artifact_dir_stats(&self) -> std::io::Result<t4a_cuda_runtime::ArtifactDirStats> {
522 self.submit().artifact_dir_stats()
523 }
524
525 /// The bound on the persistent artifact directory, in bytes.
526 pub fn artifact_dir_max_bytes(&self) -> u64 {
527 self.submit().artifact_dir_max_bytes()
528 }
529
530 /// Remove every compiled image from the persistent artifact directory and return what went.
531 ///
532 /// Images already loaded by this process stay usable; only later processes recompile.
533 ///
534 /// # Errors
535 ///
536 /// The [`std::io::Error`] from listing the directory or removing an entry, such as
537 /// `ErrorKind::PermissionDenied` on a read-only cache. An entry another process removed
538 /// first (`ErrorKind::NotFound`) is not an error.
539 pub fn clear_artifact_dir(&self) -> std::io::Result<t4a_cuda_runtime::ArtifactDirStats> {
540 self.submit().clear_artifact_dir()
541 }
542
543 /// Images this runtime actually removed from the persistent artifact directory to stay within
544 /// its bound, least recently used first.
545 ///
546 /// Trimming is best effort, so a removal that failed is not counted here and the directory can
547 /// stay above its bound; [`CudaRuntime::artifact_dir_stats`] reports what is really there.
548 pub fn artifact_dir_evictions(&self) -> u64 {
549 self.submit().artifact_dir_evictions()
550 }
551
552 /// Run `closure` inside a raw scope on this runtime's stream, under the submission
553 /// lock.
554 ///
555 /// # Safety
556 ///
557 /// As [`crate::Runtime::with_raw`](t4a_cuda_runtime::Runtime::with_raw): the closure
558 /// must use this runtime's stream, keep its enqueues within the registered resources
559 /// and obey each kernel's or vendor library's contract.
560 pub(crate) unsafe fn with_raw_locked<R>(
561 &self,
562 resources: Vec<t4a_cuda_runtime::RawResource<'_>>,
563 closure: impl FnOnce(t4a_cuda_runtime::RawScope<'_>) -> R,
564 ) -> crate::Result<R> {
565 let mut guard = self.submit();
566 // SAFETY: forwarded to the caller's contract.
567 unsafe { guard.with_raw(resources, closure) }
568 .map_err(|error| crate::Error::runtime_state_source("cuda raw scope", error))
569 }
570
571 /// Run `f` with this runtime's CUDA context current, for a vendor call that only sets up state
572 /// (a cuFFT or cuSOLVER plan) and enqueues nothing.
573 ///
574 /// # Errors
575 ///
576 /// The closure's own failure, and the runtime's when the context cannot be activated.
577 #[doc(hidden)]
578 pub fn with_current_context<R>(
579 &self,
580 op: &'static str,
581 f: impl FnOnce() -> R,
582 ) -> crate::Result<R> {
583 // The scope holds the runtime's submission lock, so a nested call would deadlock on it (a
584 // cuFFT plan's rollback runs inside the same context). Nesting is tracked per thread:
585 // inside an activated context the closure simply runs, and the level is popped when this
586 // call leaves, panic included.
587 let activation = ActivatedContext::enter(self);
588 if activation.nested {
589 return Ok(f());
590 }
591 // SAFETY: the scope activates this runtime's context for the closure's duration; it declares
592 // no resources, so nothing is registered against the closure's work.
593 unsafe { self.with_raw_locked(Vec::new(), |_scope| f()) }
594 .map_err(|error| crate::Error::runtime_state_source(op, error))
595 }
596
597 /// Resource counters of this runtime.
598 ///
599 /// The snapshot is the runtime's own: live and pooled device bytes, pending
600 /// host/module bytes, retirement records and the transfer counters. The
601 /// compatibility package adds its detached-handle count beside them
602 /// ([`CudaRuntime::pending_transfer_retirements`]).
603 #[doc(hidden)]
604 pub fn stats(&self) -> t4a_cuda_runtime::Stats {
605 let stats = self.submit().stats();
606 self.retire_detached();
607 stats
608 }
609
610 /// Copy `value` from `self` into a fresh allocation of `destination`.
611 ///
612 /// The copy is ordered on both runtimes' streams: the producer may be dropped and
613 /// the value reused immediately, and a later read on the destination stream observes
614 /// the copied bytes without a host wait.
615 ///
616 /// # Errors
617 ///
618 /// A typed refusal when the two handles are the same runtime, when `value` is not a
619 /// CUDA allocation of this runtime, or when the driver cannot admit the copy.
620 #[doc(hidden)]
621 pub fn copy_to(
622 &self,
623 destination: &CudaRuntime,
624 value: &tenferro_tensor::Tensor,
625 ) -> crate::Result<tenferro_tensor::Tensor> {
626 use tenferro_tensor::{Complex32, Complex64, DType, TensorScalar, TypedTensor};
627
628 if self.identity() == destination.identity() {
629 return Err(crate::Error::runtime_state(
630 "cuda copy",
631 "the source and destination runtimes are the same",
632 ));
633 }
634 let storage = super::pending::cuda_storage(value).ok_or_else(|| {
635 crate::Error::runtime_state("cuda copy", "expected a CUDA allocation")
636 })?;
637 if storage.device_ordinal() != self.device_ordinal() {
638 return Err(crate::Error::runtime_state(
639 "cuda copy",
640 "the value belongs to another runtime",
641 ));
642 }
643
644 // Both runtimes participate in admission. Take their submission locks in a
645 // stable order (by allocation domain) so two opposing copies cannot invert them,
646 // while the copy direction stays source -> destination in both branches.
647 let source_is_lower = self.inner.allocation_domain < destination.inner.allocation_domain;
648 let (buffer, _completion) = if source_is_lower {
649 let mut source_guard = self.submit();
650 let mut destination_guard = destination.submit();
651 let (buffer, completion, _route) = destination_guard
652 .copy_between(&mut source_guard, storage.allocation())
653 .map_err(|error| crate::Error::runtime_state_source("cuda copy", error))?;
654 (buffer, completion)
655 } else {
656 let mut destination_guard = destination.submit();
657 let mut source_guard = self.submit();
658 let (buffer, completion, _route) = destination_guard
659 .copy_between(&mut source_guard, storage.allocation())
660 .map_err(|error| crate::Error::runtime_state_source("cuda copy", error))?;
661 (buffer, completion)
662 };
663 fn rebuild<T: TensorScalar + Send + Sync + 'static>(
664 value: &tenferro_tensor::Tensor,
665 buffer: t4a_cuda_runtime::Buffer,
666 destination: &CudaRuntime,
667 ) -> crate::Result<tenferro_tensor::Tensor> {
668 let typed: &TypedTensor<T> = value.as_typed::<T>().ok_or_else(|| {
669 crate::Error::runtime_state("cuda copy", "expected a preset scalar")
670 })?;
671 let storage = super::storage::fresh_storage(destination, buffer);
672 TypedTensor::<T>::from_buffer_col_major(
673 typed.shape().to_vec(),
674 tenferro_tensor::StorageBuffer::Backend(Box::new(storage)),
675 super::transfer::placement(destination),
676 )
677 .map(tenferro_tensor::Tensor::from_typed)
678 }
679 match value.dtype() {
680 DType::F64 => rebuild::<f64>(value, buffer, destination),
681 DType::F32 => rebuild::<f32>(value, buffer, destination),
682 DType::I32 => rebuild::<i32>(value, buffer, destination),
683 DType::I64 => rebuild::<i64>(value, buffer, destination),
684 DType::C32 => rebuild::<Complex32>(value, buffer, destination),
685 DType::C64 => rebuild::<Complex64>(value, buffer, destination),
686 DType::Bool | DType::External(_) => Err(crate::Error::unsupported(
687 "cuda copy",
688 "the transfer route does not serve this dtype",
689 )),
690 }
691 }
692
693 /// Read a runtime allocation back into a byte vector.
694 ///
695 /// # Errors
696 ///
697 /// [`cuda::CudaDeviceError::Initialization`] when the copy cannot be admitted or
698 /// completed.
699 ///
700 /// [`cuda::CudaDeviceError::Initialization`]: crate::cuda::CudaDeviceError::Initialization
701 pub fn download_bytes(&self, buffer: &t4a_cuda_runtime::Buffer) -> crate::Result<Vec<u8>> {
702 let mut guard = self.submit();
703 let mut pending = guard
704 .download(buffer)
705 .map_err(|error| crate::Error::runtime_state_source("cuda download", error))?;
706 pending
707 .wait()
708 .map_err(|error| crate::Error::runtime_state_source("cuda download", error))
709 }
710
711 /// Admit one raw host-to-device upload and return its allocation.
712 ///
713 /// This is the seam's own upload path, used by callers that already hold bytes (the
714 /// publication test queues work with it); a typed [`upload_tensor`] wraps it with
715 /// the tensor and placement metadata.
716 ///
717 /// [`upload_tensor`]: super::upload_tensor
718 ///
719 /// # Errors
720 ///
721 /// [`CudaDeviceError::Initialization`] when the runtime cannot allocate or enqueue
722 /// the copy.
723 #[doc(hidden)]
724 pub fn upload_bytes(&self, bytes: &[u8]) -> Result<t4a_cuda_runtime::Buffer, CudaDeviceError> {
725 let mut guard = self.submit();
726 guard
727 .upload(bytes)
728 .map_err(|error| CudaDeviceError::Initialization {
729 device: self.inner.device_id,
730 operation: "t4a_cuda_runtime::Runtime::upload",
731 source: Box::new(error),
732 })
733 }
734
735 /// Allocate a pinned host buffer owned by this runtime.
736 ///
737 /// # Errors
738 ///
739 /// [`CudaDeviceError::Initialization`] when the driver cannot allocate pinned
740 /// memory.
741 #[doc(hidden)]
742 pub(crate) fn alloc_pinned(
743 &self,
744 bytes: usize,
745 ) -> Result<t4a_cuda_runtime::PinnedHostBuffer, CudaDeviceError> {
746 let mut guard = self.submit();
747 guard
748 .alloc_pinned(bytes)
749 .map_err(|error| CudaDeviceError::Initialization {
750 device: self.inner.device_id,
751 operation: "t4a_cuda_runtime::Runtime::alloc_pinned",
752 source: Box::new(error),
753 })
754 }
755
756 /// Pinned staging allocations this runtime made, whoever asked.
757 ///
758 /// One for the first download and one per later growth of the staging buffer: a download that
759 /// allocated per call would count every download instead.
760 #[doc(hidden)]
761 #[must_use]
762 pub fn pinned_staging_allocations(&self) -> u64 {
763 self.inner.pinned_staging_accounting.allocations()
764 }
765
766 /// What the download staging currently retains: bytes, pinned allocations, and the bound.
767 ///
768 /// Downloads reuse one page-locked host buffer, because pinning it costs about 1.9 ms beside a
769 /// 19 us copy. The buffer is memory the OS cannot reclaim, so it is bounded (see
770 /// [`CudaRuntime::set_pinned_staging_limit_bytes`]) and can be released early with
771 /// [`CudaRuntime::clear_pinned_staging`].
772 ///
773 /// # Examples
774 ///
775 /// ```
776 /// # fn main() -> Result<(), tenferro_gpu::cuda::CudaEngineError> {
777 /// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaRuntime};
778 ///
779 /// # if gpu_available() {
780 /// let runtime = CudaRuntime::new(CudaDeviceId::from_ordinal(0))?;
781 /// let stats = runtime.pinned_staging_stats();
782 /// println!("{} bytes retained, {} allocations", stats.retained_bytes, stats.allocations);
783 /// # }
784 /// # Ok(())
785 /// # }
786 /// ```
787 #[must_use]
788 pub fn pinned_staging_stats(&self) -> PinnedStagingStats {
789 let accounting = &self.inner.pinned_staging_accounting;
790 PinnedStagingStats {
791 retained_bytes: accounting.retained(),
792 allocations: accounting.allocations(),
793 limit_bytes: accounting.limit(),
794 }
795 }
796
797 /// Bound the download staging, releasing a larger buffer at once.
798 ///
799 /// A download above this size still stages through host memory, but through a buffer released
800 /// when that download returns, so nothing above the bound stays resident. Lowering the bound
801 /// below what is retained releases the cached buffer; `usize::MAX` keeps every download.
802 ///
803 /// # Examples
804 ///
805 /// ```
806 /// # fn main() -> Result<(), tenferro_gpu::cuda::CudaEngineError> {
807 /// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaRuntime};
808 ///
809 /// # if gpu_available() {
810 /// let runtime = CudaRuntime::new(CudaDeviceId::from_ordinal(0))?;
811 /// runtime.set_pinned_staging_limit_bytes(1024 * 1024);
812 /// assert_eq!(runtime.pinned_staging_stats().limit_bytes, 1024 * 1024);
813 /// # }
814 /// # Ok(())
815 /// # }
816 /// ```
817 pub fn set_pinned_staging_limit_bytes(&self, limit: usize) {
818 self.inner.pinned_staging_accounting.set_limit(limit);
819 self.release_staging_above(limit);
820 }
821
822 /// Release the retained download staging now.
823 ///
824 /// The next download pins a buffer again. Returns the bytes released, so a caller can tell the
825 /// call apart from one that found nothing to release.
826 ///
827 /// # Examples
828 ///
829 /// ```
830 /// # fn main() -> Result<(), tenferro_gpu::cuda::CudaEngineError> {
831 /// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaRuntime};
832 ///
833 /// # if gpu_available() {
834 /// let runtime = CudaRuntime::new(CudaDeviceId::from_ordinal(0))?;
835 /// let released = runtime.clear_pinned_staging();
836 /// assert_eq!(runtime.pinned_staging_stats().retained_bytes, 0);
837 /// # let _ = released;
838 /// # }
839 /// # Ok(())
840 /// # }
841 /// ```
842 pub fn clear_pinned_staging(&self) -> usize {
843 self.release_staging_above(0)
844 }
845
846 /// Drop the cached buffer when it exceeds `limit`, and report the bytes actually released.
847 ///
848 /// The free runs inside this runtime's raw scope, which makes the owning context current on
849 /// this thread; without it `cuMemFreeHost` can fail with `CUDA_ERROR_INVALID_CONTEXT` on a
850 /// thread that never activated the context. A failed free keeps the buffer and its accounting,
851 /// because dropping the pointer would leak the allocation while reporting it released.
852 fn release_staging_above(&self, limit: usize) -> usize {
853 let Ok(mut slot) = self.inner.pinned_staging.lock() else {
854 return 0;
855 };
856 let Some(entry) = slot.as_ref() else {
857 return 0;
858 };
859 let released = entry.bytes();
860 if released == 0 || released <= limit {
861 return 0;
862 }
863 // No download is in flight: it would hold this lock, and it synchronizes before releasing
864 // it, so nothing queued still reads this buffer.
865 let pointer = entry.pointer();
866 let freed = unsafe {
867 self.with_raw_locked(Vec::new(), |_raw| {
868 cudarc::driver::sys::cuMemFreeHost(pointer)
869 })
870 };
871 match freed {
872 Ok(cudarc::driver::sys::CUresult::CUDA_SUCCESS) => {
873 *slot = None;
874 self.inner.pinned_staging_accounting.record_retained(0);
875 released
876 }
877 other => {
878 // Keep the buffer: it is still allocated, so the accounting stays truthful and a
879 // later clear can retry.
880 let _ = other;
881 0
882 }
883 }
884 }
885
886 /// Wait for every queued operation and retire the detached handles it completes.
887 ///
888 /// # Errors
889 ///
890 /// [`CudaDeviceError::Initialization`] when the stream reports a failure.
891 #[doc(hidden)]
892 pub fn synchronize(&self) -> Result<(), CudaDeviceError> {
893 self.inner
894 .stream_waits
895 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
896 {
897 // Hold the submission lock through the reclamation: a concurrent clone cannot
898 // enqueue a transfer it then detaches in the gap between the stream wait and
899 // the queue drain, which would free its pinned source while still queued.
900 let mut guard = self.submit();
901 guard
902 .synchronize()
903 .map_err(|error| CudaDeviceError::Initialization {
904 device: self.inner.device_id,
905 operation: "t4a_cuda_runtime::Runtime::synchronize",
906 source: Box::new(error),
907 })?;
908 self.retire_detached_locked();
909 }
910 Ok(())
911 }
912
913 /// Completion checkpoints this runtime recorded on its stream.
914 ///
915 /// A scheduled run records one per node, unless it took the single-domain coalesced path,
916 /// where submission order is the ordering and the run's retirement is the only barrier.
917 /// Exposed so a regression test can pin that (#2396).
918 #[doc(hidden)]
919 pub fn completion_checkpoints(&self) -> u64 {
920 self.inner
921 .completion_checkpoints
922 .load(std::sync::atomic::Ordering::Relaxed)
923 }
924
925 /// Record one completion checkpoint here, for callers that own the submission lock.
926 pub(crate) fn note_completion_checkpoint(&self) {
927 self.inner
928 .completion_checkpoints
929 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
930 }
931
932 /// Stream barriers this runtime ran, including the ones a scheduled run's operations skip.
933 ///
934 /// A prepared run adds exactly the caller's own barrier; an operation that drains its own
935 /// launch adds one per launch. Exposed so the regression test can count real waits.
936 #[doc(hidden)]
937 pub fn stream_waits(&self) -> u64 {
938 self.inner
939 .stream_waits
940 .load(std::sync::atomic::Ordering::Relaxed)
941 }
942
943 /// Whether a scheduled run currently owns the completion of the launches it enqueues.
944 ///
945 /// A run that holds the deferral records a stream-ordered checkpoint after every node and is
946 /// the barrier for its own work (see the CUDA engine's event domain), so an operation's launch
947 /// does not prove its own completion while it is zero. A standalone call sees zero and waits.
948 #[doc(hidden)]
949 pub(crate) fn completion_deferred(&self) -> bool {
950 self.inner
951 .deferred_completions
952 .load(std::sync::atomic::Ordering::Relaxed)
953 > 0
954 }
955
956 /// Take a scheduled run's completion ownership for as long as the guard lives.
957 ///
958 /// Nested guards are counted, so a run enqueueing a node that itself starts a run keeps the
959 /// deferral in force until the outermost guard is dropped.
960 pub(crate) fn defer_completion(&self) -> CompletionDeferral<'_> {
961 self.inner
962 .deferred_completions
963 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
964 CompletionDeferral { runtime: self }
965 }
966
967 /// Idle device memory the allocation pool keeps mapped before returning it to the driver.
968 ///
969 /// Reported next to [`Self::stats`] because it is the bound the pool's retained bytes obey
970 /// (`PERFORMANCE_TIPS.md`, Cache Ownership): raise it with
971 /// `TENFERRO_CUDA_POOL_RELEASE_MIB` when a working set of live buffers exceeds it, which is
972 /// what made every eager 3072^3 output allocation re-map its pages (#2381).
973 #[doc(hidden)]
974 pub fn pool_release_threshold_bytes(&self) -> usize {
975 self.inner.pool_release_threshold
976 }
977
978 /// Return idle pool memory above `keep_bytes` to the driver now.
979 ///
980 /// The pool trims itself at the threshold recorded by
981 /// [`Self::pool_release_threshold_bytes`]; this is the explicit clear that a caller uses to
982 /// give device memory back, for example before handing the device to another library.
983 ///
984 /// # Errors
985 ///
986 /// [`CudaDeviceError::Initialization`] when the driver refuses the trim.
987 #[doc(hidden)]
988 pub fn trim_pool(&self, keep_bytes: usize) -> Result<(), CudaDeviceError> {
989 self.submit()
990 .trim_pool(keep_bytes)
991 .map_err(|error| CudaDeviceError::Initialization {
992 device: self.inner.device_id,
993 operation: "t4a_cuda_runtime::Runtime::trim_pool",
994 source: Box::new(error),
995 })
996 }
997
998 /// Detached pending handles that have not been retired yet, the compatibility
999 /// package's own count (upstream `pending_records` counts retirement epochs, a
1000 /// different quantity).
1001 #[doc(hidden)]
1002 pub fn pending_transfer_retirements(&self) -> u64 {
1003 self.inner
1004 .detached
1005 .lock()
1006 .unwrap_or_else(|poison| poison.into_inner())
1007 .len() as u64
1008 }
1009
1010 /// Retain a dropped handle until its completion is observed.
1011 pub(crate) fn detach(
1012 &self,
1013 keepalive: Box<dyn std::any::Any + Send>,
1014 completion: Option<t4a_cuda_runtime::Completion>,
1015 ) {
1016 self.inner
1017 .detached
1018 .lock()
1019 .unwrap_or_else(|poison| poison.into_inner())
1020 .push(Detached {
1021 keepalive,
1022 completion,
1023 });
1024 }
1025
1026 /// Release the completed detached packages while the caller already holds the
1027 /// submission lock.
1028 fn retire_detached_locked(&self) {
1029 let mut detached = self
1030 .inner
1031 .detached
1032 .lock()
1033 .unwrap_or_else(|poison| poison.into_inner());
1034 detached.retain_mut(|entry| match entry.completion.as_mut() {
1035 Some(completion) => !matches!(completion.poll(), Ok(Some(()))),
1036 None => false,
1037 });
1038 }
1039
1040 /// Release the detached packages whose completion is observed, keeping the rest.
1041 ///
1042 /// Reclamation is explicit: a package is only released once its own completion says
1043 /// the work is done, so a queued transfer never loses its source or destination.
1044 pub(crate) fn retire_detached(&self) {
1045 let mut detached = self
1046 .inner
1047 .detached
1048 .lock()
1049 .unwrap_or_else(|poison| poison.into_inner());
1050 detached.retain_mut(|entry| match entry.completion.as_mut() {
1051 Some(completion) => match completion.poll() {
1052 Ok(Some(())) => false,
1053 Ok(None) => true,
1054 // A failed completion is kept: the failure surfaces through the next
1055 // explicit observation rather than being dropped silently here.
1056 Err(_) => true,
1057 },
1058 None => false,
1059 });
1060 }
1061
1062 /// Synchronise and then reclaim what completed, for teardown paths that cannot
1063 /// report an error.
1064 pub(crate) fn synchronize_best_effort(&self) {
1065 {
1066 let mut guard = self.submit();
1067 let _ = guard.synchronize();
1068 }
1069 self.retire_detached();
1070 }
1071
1072 /// Take the submission lock for one explicit transfer.
1073 pub(crate) fn submit(&self) -> std::sync::MutexGuard<'_, t4a_cuda_runtime::Runtime> {
1074 self.inner
1075 .runtime
1076 .lock()
1077 .unwrap_or_else(|poison| poison.into_inner())
1078 }
1079}
1080
1081impl Drop for CudaRuntimeState {
1082 fn drop(&mut self) {
1083 // Last handle: drain the stream, then release the detached packages. A failure
1084 // leaves their completion unproved, so the queue is leaked instead of freeing
1085 // pinned memory a queued copy may still read or write.
1086 let proved = self
1087 .runtime
1088 .lock()
1089 .unwrap_or_else(|poison| poison.into_inner())
1090 .synchronize()
1091 .is_ok();
1092 if proved {
1093 let mut detached = self
1094 .detached
1095 .lock()
1096 .unwrap_or_else(|poison| poison.into_inner());
1097 detached.clear();
1098 } else {
1099 let leaked = std::mem::take(
1100 &mut *self
1101 .detached
1102 .lock()
1103 .unwrap_or_else(|poison| poison.into_inner()),
1104 );
1105 std::mem::forget(leaked);
1106 }
1107 }
1108}
1109
1110/// One scheduled run's ownership of its nodes' launch completion, for as long as it is enqueued.
1111///
1112/// Dropping the guard hands completion back to the operations, which is what a standalone call
1113/// expects; it is deliberately trivial so no failure path can leave the count raised.
1114pub(crate) struct CompletionDeferral<'a> {
1115 runtime: &'a CudaRuntime,
1116}
1117
1118impl Drop for CompletionDeferral<'_> {
1119 fn drop(&mut self) {
1120 self.runtime
1121 .inner
1122 .deferred_completions
1123 .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
1124 }
1125}
1126
1127impl std::fmt::Debug for CompletionDeferral<'_> {
1128 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1129 f.write_str("CompletionDeferral")
1130 }
1131}
1132
1133impl std::fmt::Debug for CudaRuntime {
1134 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1135 f.debug_struct("CudaRuntime")
1136 .field("device", &self.inner.device_id)
1137 .field("domain", &self.inner.allocation_domain)
1138 .finish()
1139 }
1140}
1141
1142#[cfg(test)]
1143mod tests;