Skip to main content

tenferro_gpu/cuda/
runtime.rs

1//! The runtime handle: one `t4a-cuda-runtime` runtime, one device, one allocation
2//! domain, one submission lock for explicit host transfers.
3
4use std::sync::{Arc, Mutex};
5
6use tenferro_tensor::AllocationDomainId;
7
8use crate::cuda::{CudaDeviceError, CudaDeviceId, CudaDeviceInfo};
9
10/// Identity of one opened runtime; equal only for clones of the same handle.
11#[derive(Debug, Clone)]
12#[doc(hidden)]
13pub struct CudaRuntimeIdentity {
14    marker: Arc<u8>,
15}
16
17impl CudaRuntimeIdentity {
18    fn fresh() -> Self {
19        Self {
20            marker: Arc::new(0),
21        }
22    }
23}
24
25impl PartialEq for CudaRuntimeIdentity {
26    fn eq(&self, other: &Self) -> bool {
27        Arc::ptr_eq(&self.marker, &other.marker)
28    }
29}
30
31impl Eq for CudaRuntimeIdentity {}
32
33impl std::hash::Hash for CudaRuntimeIdentity {
34    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
35        // The marker allocation is retained by every clone, so its address is
36        // move- and clone-invariant while an identity is witnessed.
37        state.write_usize(Arc::as_ptr(&self.marker) as usize);
38    }
39}
40
41std::thread_local! {
42    /// Addresses of the runtimes whose context this thread has already activated.
43    ///
44    /// The raw scope takes the runtime's submission lock, so a second activation on the same
45    /// thread would block forever; nesting is legal because the context is already current.
46    static ACTIVATED_CONTEXTS: std::cell::RefCell<Vec<usize>> =
47        const { std::cell::RefCell::new(Vec::new()) };
48}
49
50/// One activation level of a runtime's context on this thread.
51pub(crate) struct ActivatedContext {
52    key: usize,
53    /// Whether this call found the context already active on this thread.
54    nested: bool,
55    /// `Some` when this call pushed a level; dropping it pops the level again, including on a
56    /// panic, so a leaked level cannot make a later activation silently skip the context.
57    pushed: bool,
58}
59
60impl ActivatedContext {
61    /// Enter `runtime`'s context, reporting whether this thread was already inside it.
62    fn enter(runtime: &CudaRuntime) -> ActivatedContext {
63        let key = Arc::as_ptr(&runtime.inner) as usize;
64        let nested = ACTIVATED_CONTEXTS.with(|active| {
65            let mut active = active.borrow_mut();
66            if active.contains(&key) {
67                true
68            } else {
69                active.push(key);
70                false
71            }
72        });
73        ActivatedContext {
74            key,
75            nested,
76            pushed: !nested,
77        }
78    }
79}
80
81impl Drop for ActivatedContext {
82    fn drop(&mut self) {
83        if self.pushed {
84            ACTIVATED_CONTEXTS.with(|active| {
85                active.borrow_mut().retain(|entry| *entry != self.key);
86            });
87        }
88    }
89}
90
91/// Default bound on the retained download staging: 8 MiB.
92///
93/// Pinning costs about 1.9 ms, so the buffer is kept — but it is page-locked memory the allocator
94/// cannot reclaim. 8 MiB covers the readbacks this path exists for (solver `info` words, norms,
95/// ranks, a 1024x1024 f64 matrix) with room to spare; a larger download stages through a temporary
96/// buffer and leaves nothing resident. Raise it with
97/// [`CudaRuntime::set_pinned_staging_limit_bytes`] when large downloads repeat.
98pub const DEFAULT_PINNED_STAGING_LIMIT_BYTES: usize = 8 * 1024 * 1024;
99
100/// What a runtime's download staging retains, and the bound it retains it under.
101///
102/// # Examples
103///
104/// ```
105/// # fn main() -> Result<(), tenferro_gpu::cuda::CudaEngineError> {
106/// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaRuntime};
107///
108/// # if gpu_available() {
109/// let runtime = CudaRuntime::new(CudaDeviceId::from_ordinal(0))?;
110/// let stats = runtime.pinned_staging_stats();
111/// if stats.retained_bytes > stats.limit_bytes {
112///     runtime.clear_pinned_staging();
113/// }
114/// println!("{stats:?}");
115/// # }
116/// # Ok(())
117/// # }
118/// ```
119#[derive(Clone, Copy, Debug, PartialEq, Eq)]
120pub struct PinnedStagingStats {
121    /// Bytes the cached staging buffer holds right now.
122    pub retained_bytes: usize,
123    /// Pinned host allocations this runtime made for staging, cumulative: the first download, each
124    /// later growth, and each temporary buffer used above the bound or re-pinned after a clear.
125    pub allocations: u64,
126    /// Largest download whose staging is retained.
127    pub limit_bytes: usize,
128}
129
130/// State shared by every handle to one runtime.
131pub(crate) struct CudaRuntimeState {
132    /// Pinned host staging reused by explicit downloads. Allocating and freeing one per download
133    /// cost far more than the copy (a 4-byte `cuMemHostAlloc` measured 1.9 ms beside a 19 us
134    /// device-to-host copy), so one buffer lives here and grows on demand.
135    pub(crate) pinned_staging: Mutex<Option<crate::cuda::raw::PinnedStaging>>,
136    /// What that buffer retains, the bound it retains it under, and how many pinned allocations
137    /// this runtime made for staging.
138    pub(crate) pinned_staging_accounting: crate::cuda::raw::StagingAccounting,
139
140    /// The execution owner. Explicit transfers take this lock for one admission;
141    /// numerical submission borrows it mutably through the engine.
142    pub(crate) runtime: Mutex<t4a_cuda_runtime::Runtime>,
143    pub(crate) device_id: CudaDeviceId,
144    pub(crate) device_info: CudaDeviceInfo,
145    pub(crate) allocation_domain: AllocationDomainId,
146    pub(crate) identity: CudaRuntimeIdentity,
147    /// The allocation pool's retention budget this runtime opened with, in bytes.
148    pub(crate) pool_release_threshold: usize,
149    /// Handles dropped while their work was still queued. The runtime's retirement
150    /// already owns the staging; this queue owns only the caller's packages (a pinned
151    /// source, a tensor handle) until their completion is observed.
152    pub(crate) detached: Mutex<Vec<Detached>>,
153    /// How many scheduled runs are currently enqueueing nodes on this runtime.
154    ///
155    /// While it is non-zero a run owns the completion proof: every node it enqueues records a
156    /// stream-ordered checkpoint, and the run's retirement is the barrier, so an operation's own
157    /// launch does not have to wait. Zero means a standalone call, which proves its own launch.
158    pub(crate) deferred_completions: std::sync::atomic::AtomicUsize,
159    /// Completion checkpoints recorded on this runtime's stream, per node of a scheduled run or
160    /// by a transfer. A coalesced run records none: submission order is its order (#2396).
161    pub(crate) completion_checkpoints: std::sync::atomic::AtomicU64,
162    /// Stream barriers this runtime ran, whoever asked: an operation proving its own launch, a
163    /// download, or a caller's explicit synchronize.
164    pub(crate) stream_waits: std::sync::atomic::AtomicU64,
165}
166
167/// One dropped handle retained until its completion is observed.
168pub(crate) struct Detached {
169    /// The caller's package: the queued copy still reads or writes it, so it stays owned
170    /// here until the completion below proves the work done.
171    #[allow(dead_code)]
172    pub(crate) keepalive: Box<dyn std::any::Any + Send>,
173    pub(crate) completion: Option<t4a_cuda_runtime::Completion>,
174}
175
176/// The runtime configuration this build opens with.
177///
178/// The artifact directory, its bound and the allocation pool's retention budget are read from
179/// the environment on every open: `TENFERRO_CUDA_ARTIFACT_CACHE`,
180/// `TENFERRO_CUDA_ARTIFACT_CACHE_MIB` and `TENFERRO_CUDA_POOL_RELEASE_MIB`.
181///
182/// `TENFERRO_CUDA_ARTIFACT_CACHE` names the directory that persists compiled kernel images, so a
183/// later process reads them back instead of paying NVRTC again: a whole module is about 8 s per
184/// process on an A100, and a warm read is a few milliseconds. The value wins over the default;
185/// `none`, `off` and `memory` select the in-memory cache only; unset selects
186/// `$XDG_CACHE_HOME/tenferro/t4a-cuda-artifacts`, else `$HOME/.cache/tenferro/t4a-cuda-artifacts`
187/// (`%LOCALAPPDATA%\tenferro\t4a-cuda-artifacts` on Windows), and falls back to the in-memory
188/// cache when no absolute per-user root is set or the directory cannot be created. A newly created
189/// directory is mode 0700 on Unix.
190///
191/// The directory is bounded, never unlimited: at most [`artifact_cache_max_bytes`] bytes of
192/// images, 256 MiB unless `TENFERRO_CUDA_ARTIFACT_CACHE_MIB` says otherwise. Publishing past the
193/// bound removes the least recently used images first, best effort: a removal that fails is
194/// dropped, so the directory can stay above its bound. [`CudaRuntime::artifact_dir_evictions`]
195/// counts what was actually removed, and [`CudaRuntime::artifact_dir_stats`] and
196/// [`CudaRuntime::clear_artifact_dir`] read and empty it.
197///
198/// `TENFERRO_CUDA_POOL_RELEASE_MIB` overrides the allocation pool's retention budget, the idle
199/// device memory the pool keeps mapped instead of handing back to the driver. See
200/// [`pool_release_threshold_bytes`].
201fn default_config(total_memory_bytes: u64) -> t4a_cuda_runtime::Config {
202    let artifact_cache_dir = artifact_cache_dir(
203        std::env::var_os("TENFERRO_CUDA_ARTIFACT_CACHE").as_deref(),
204        |name| std::env::var_os(name),
205    )
206    .and_then(usable_artifact_cache_dir);
207    t4a_cuda_runtime::Config {
208        artifact_cache_dir,
209        pool_release_threshold: pool_release_threshold_bytes(total_memory_bytes),
210        ..t4a_cuda_runtime::Config::default()
211    }
212}
213
214/// The least idle device memory the pool keeps mapped, whatever the device size.
215const MIN_POOL_RELEASE_BYTES: usize = 256 << 20;
216/// The most idle device memory the pool keeps mapped, whatever the device size.
217const MAX_POOL_RELEASE_BYTES: usize = 4 << 30;
218/// The share of the device's memory the pool may keep mapped.
219const POOL_RELEASE_SHARE: u64 = 8;
220
221/// Idle device memory the allocation pool keeps mapped instead of returning to the driver.
222///
223/// The driver's own default is a fixed 256 MiB, which is smaller than the working set of a single
224/// large operation: an eager 3072^3 f64 matmul holds about 430 MiB of 72 MiB buffers, so every
225/// call's output allocation missed the pool and re-mapped its pages — 864 us of
226/// `cuMemAllocFromPoolAsync` per call, 87 of 87 calls on the A100 host, reproducing in plain CUDA
227/// once six buffers are live (#2381). A budget scaled to the device keeps those pages.
228///
229/// `TENFERRO_CUDA_POOL_RELEASE_MIB` overrides the result, in MiB. The budget is a ceiling on
230/// retained *idle* memory, not a reservation: nothing is allocated before it is needed.
231fn pool_release_threshold_bytes(total_memory_bytes: u64) -> usize {
232    let override_mib = std::env::var("TENFERRO_CUDA_POOL_RELEASE_MIB")
233        .ok()
234        .and_then(|value| value.trim().parse::<u64>().ok());
235    pool_release_threshold_from(override_mib, total_memory_bytes)
236}
237
238/// [`pool_release_threshold_bytes`] with the environment's override supplied explicitly.
239fn pool_release_threshold_from(override_mib: Option<u64>, total_memory_bytes: u64) -> usize {
240    if let Some(mib) = override_mib {
241        return usize::try_from(mib.saturating_mul(1 << 20))
242            .unwrap_or(usize::MAX)
243            .clamp(MIN_POOL_RELEASE_BYTES, MAX_POOL_RELEASE_BYTES);
244    }
245    let share = usize::try_from(total_memory_bytes / POOL_RELEASE_SHARE).unwrap_or(usize::MAX);
246    share.clamp(MIN_POOL_RELEASE_BYTES, MAX_POOL_RELEASE_BYTES)
247}
248
249/// The bound on the persistent artifact directory, in bytes.
250///
251/// `TENFERRO_CUDA_ARTIFACT_CACHE_MIB` (`value`) sets it in MiB. `0` is a bound like any other, not
252/// a read-only mode: the runtime applies a bound through
253/// [`t4a_cuda_runtime::Runtime::set_artifact_dir_max_bytes`], which trims the directory as it
254/// records the bound. That trim is best effort — a removal that fails is dropped — so `0` stops
255/// new images being published and does not guarantee the directory is empty; an image a failed
256/// trim left behind is still served. A process that wants no persistent directory asks for that
257/// with `TENFERRO_CUDA_ARTIFACT_CACHE=none`. Unset, or a value that is not a whole number of MiB,
258/// keeps [`t4a_cuda_runtime::DEFAULT_ARTIFACT_DIR_BYTES`] (256 MiB, the in-memory artifact
259/// budget): there is no spelling of "unlimited".
260fn artifact_cache_max_bytes(value: Option<&std::ffi::OsStr>) -> u64 {
261    value
262        .and_then(std::ffi::OsStr::to_str)
263        .and_then(|value| value.trim().parse::<u64>().ok())
264        .map_or(t4a_cuda_runtime::DEFAULT_ARTIFACT_DIR_BYTES, |mib| {
265            mib.saturating_mul(1 << 20)
266        })
267}
268
269/// The one operation opening a runtime performs on its artifact cache.
270///
271/// The cache lives behind a device, so the open applies the bound through this trait and a test
272/// drives a fake instead: a compile of the decision alone would not show that it is applied.
273trait ArtifactDirBound {
274    fn set_artifact_dir_max_bytes(&mut self, max_bytes: u64);
275}
276
277impl ArtifactDirBound for t4a_cuda_runtime::Runtime {
278    fn set_artifact_dir_max_bytes(&mut self, max_bytes: u64) {
279        // The dependency's own method, not this trait's: no recursion.
280        t4a_cuda_runtime::Runtime::set_artifact_dir_max_bytes(self, max_bytes);
281    }
282}
283
284/// Apply `TENFERRO_CUDA_ARTIFACT_CACHE_MIB` to `cache`, the open's only cache mutation.
285fn apply_artifact_cache_bound(
286    cache: &mut dyn ArtifactDirBound,
287    read: impl Fn(&str) -> Option<std::ffi::OsString>,
288) {
289    cache.set_artifact_dir_max_bytes(artifact_cache_max_bytes(
290        read("TENFERRO_CUDA_ARTIFACT_CACHE_MIB").as_deref(),
291    ));
292}
293
294/// `tenferro`'s directory under the user's cache root, which holds compiled kernel images.
295const ARTIFACT_CACHE_PATH: &str = "tenferro/t4a-cuda-artifacts";
296
297/// The persistent artifact directory the environment selects, or `None` for memory only.
298///
299/// `read` supplies the environment so the fallback chain is testable without one.
300fn artifact_cache_dir(
301    value: Option<&std::ffi::OsStr>,
302    read: impl Fn(&str) -> Option<std::ffi::OsString>,
303) -> Option<std::path::PathBuf> {
304    if let Some(value) = value {
305        return match value.to_str() {
306            Some(value) => {
307                let value = value.trim();
308                if value.is_empty() || matches!(value, "none" | "off" | "memory") {
309                    None
310                } else {
311                    Some(std::path::PathBuf::from(value))
312                }
313            }
314            // Not UTF-8, so it cannot be one of the tokens: it names a directory as it is.
315            None => Some(std::path::PathBuf::from(value)),
316        };
317    }
318    cache_root(read).map(|root| root.join(ARTIFACT_CACHE_PATH))
319}
320
321/// This user's cache root.
322///
323/// `$XDG_CACHE_HOME` when it is absolute, else `$HOME/.cache`; a relative `XDG_CACHE_HOME` is
324/// ignored, as the specification requires, so the cache location never depends on the working
325/// directory. Windows has no XDG and uses `%LOCALAPPDATA%`, which is already a per-user root.
326fn cache_root(read: impl Fn(&str) -> Option<std::ffi::OsString>) -> Option<std::path::PathBuf> {
327    let root = |name: &str| {
328        read(name)
329            .filter(|value| !value.is_empty())
330            .map(std::path::PathBuf::from)
331            .filter(|dir| dir.is_absolute())
332    };
333    if cfg!(windows) {
334        return root("LOCALAPPDATA");
335    }
336    root("XDG_CACHE_HOME").or_else(|| root("HOME").map(|home| home.join(".cache")))
337}
338
339/// The same directory, once this process has created it or found it present.
340///
341/// A home it cannot create the directory under degrades to the in-memory cache with one warning,
342/// never to a failed operation, so nothing here returns an error into the launch path.
343fn usable_artifact_cache_dir(dir: std::path::PathBuf) -> Option<std::path::PathBuf> {
344    match create_private_dir(&dir) {
345        Ok(()) => Some(dir),
346        Err(error) => {
347            ARTIFACT_CACHE_WARNING.call_once(|| {
348                write_best_effort_warning(
349                    &mut std::io::stderr(),
350                    format_args!(
351                        "tenferro-gpu: cannot create the artifact cache directory {}: {error}; \
352                         compiled kernels stay in memory for this process",
353                        dir.display()
354                    ),
355                );
356            });
357            None
358        }
359    }
360}
361
362/// Write one warning line to `sink`, dropping it if the write fails.
363///
364/// `eprintln!` panics when stderr is closed, and a diagnostic on a best-effort path must never
365/// turn into a panic in the library.
366pub(crate) fn write_best_effort_warning(
367    sink: &mut impl std::io::Write,
368    message: std::fmt::Arguments<'_>,
369) {
370    let _ = writeln!(sink, "{message}");
371}
372
373/// One warning per process, however many runtimes open against an unusable directory.
374static ARTIFACT_CACHE_WARNING: std::sync::Once = std::sync::Once::new();
375
376/// Create `dir` and any missing parent. A directory this creates is mode 0700 on Unix; an existing
377/// one keeps its own mode, so a shared read-only cache stays readable and its writes stay dropped.
378fn create_private_dir(dir: &std::path::Path) -> std::io::Result<()> {
379    let mut builder = std::fs::DirBuilder::new();
380    builder.recursive(true);
381    #[cfg(unix)]
382    {
383        use std::os::unix::fs::DirBuilderExt;
384        builder.mode(0o700);
385    }
386    builder.create(dir)
387}
388
389/// A cheap, cloneable handle to one opened CUDA runtime.
390///
391/// The handle carries no submission authority of its own: explicit host transfers
392/// take the runtime's submission lock for the duration of one admission, and
393/// numerical submission belongs to the [`CudaEngine`](super::CudaEngine) that owns
394/// the same state.
395#[derive(Clone)]
396#[doc(hidden)]
397pub struct CudaRuntime {
398    pub(crate) inner: Arc<CudaRuntimeState>,
399}
400
401impl CudaRuntime {
402    /// Open the CUDA runtime on `device_id` with the default configuration.
403    ///
404    /// # Errors
405    ///
406    /// [`CudaDeviceError`] when the device is unknown or the driver cannot open it.
407    #[doc(hidden)]
408    pub fn new(device_id: CudaDeviceId) -> Result<CudaRuntime, CudaDeviceError> {
409        let total_memory = crate::cuda::cuda_devices()?
410            .iter()
411            .find(|info| info.id() == device_id)
412            .map(crate::cuda::CudaDeviceInfo::total_memory_bytes)
413            .unwrap_or(0);
414        let runtime = Self::with_config(device_id, default_config(total_memory))?;
415        apply_artifact_cache_bound(&mut *runtime.submit(), |name| std::env::var_os(name));
416        Ok(runtime)
417    }
418
419    /// Open the CUDA runtime on `device_id` with an explicit configuration.
420    ///
421    /// The configuration bounds the runtime's staging chunk and pending budget, which is
422    /// what a caller uses to force chunked transfers.
423    ///
424    /// # Errors
425    ///
426    /// [`CudaDeviceError`] when the device is unknown, the configuration is invalid, or
427    /// the driver cannot open the device.
428    #[doc(hidden)]
429    pub fn with_config(
430        device_id: CudaDeviceId,
431        config: t4a_cuda_runtime::Config,
432    ) -> Result<CudaRuntime, CudaDeviceError> {
433        let devices = crate::cuda::cuda_devices()?;
434        let info = devices
435            .iter()
436            .find(|info| info.id() == device_id)
437            .cloned()
438            .ok_or_else(|| CudaDeviceError::Unavailable {
439                requested: device_id,
440                discovered: devices.into_boxed_slice(),
441            })?;
442        let pool_release_threshold = config.pool_release_threshold;
443        let runtime = t4a_cuda_runtime::Runtime::open(device_id.ordinal() as usize, config)
444            .map_err(|error| CudaDeviceError::Initialization {
445                device: device_id,
446                operation: "t4a_cuda_runtime::Runtime::open",
447                source: Box::new(error),
448            })?;
449        Ok(CudaRuntime {
450            inner: Arc::new(CudaRuntimeState {
451                runtime: Mutex::new(runtime),
452                device_id,
453                device_info: info,
454                allocation_domain: AllocationDomainId::fresh(),
455                pool_release_threshold,
456                identity: CudaRuntimeIdentity::fresh(),
457                detached: Mutex::new(Vec::new()),
458                deferred_completions: std::sync::atomic::AtomicUsize::new(0),
459                stream_waits: std::sync::atomic::AtomicU64::new(0),
460                completion_checkpoints: std::sync::atomic::AtomicU64::new(0),
461                pinned_staging: Mutex::new(None),
462                pinned_staging_accounting: crate::cuda::raw::StagingAccounting::new(
463                    DEFAULT_PINNED_STAGING_LIMIT_BYTES,
464                ),
465            }),
466        })
467    }
468
469    /// The device this runtime owns.
470    #[doc(hidden)]
471    pub fn device_id(&self) -> CudaDeviceId {
472        self.inner.device_id
473    }
474
475    /// The device metadata this runtime was opened with.
476    #[doc(hidden)]
477    pub fn device_info(&self) -> &CudaDeviceInfo {
478        &self.inner.device_info
479    }
480
481    /// The ordinal of the owning device.
482    #[doc(hidden)]
483    pub fn device_ordinal(&self) -> usize {
484        self.inner.device_id.ordinal() as usize
485    }
486
487    /// The identity of this runtime; distinct for every opened runtime.
488    #[doc(hidden)]
489    pub(crate) fn identity(&self) -> &CudaRuntimeIdentity {
490        &self.inner.identity
491    }
492
493    /// The shared-allocation domain every buffer of this runtime belongs to.
494    #[doc(hidden)]
495    pub fn allocation_domain_id(&self) -> AllocationDomainId {
496        self.inner.allocation_domain
497    }
498
499    /// NVRTC compilation attempts this runtime's artifact cache performed.
500    pub fn compile_attempts(&self) -> u64 {
501        self.submit().compile_attempts()
502    }
503
504    /// Module loads this runtime performed.
505    pub fn module_loads(&self) -> u64 {
506        self.submit().module_loads()
507    }
508
509    /// Artifact-cache hits this runtime served.
510    pub fn artifact_cache_hits(&self) -> u64 {
511        self.submit().artifact_cache_hits()
512    }
513
514    /// Images and bytes in the persistent artifact directory; empty for an in-memory cache.
515    ///
516    /// # Errors
517    ///
518    /// The [`std::io::Error`] from listing a directory that exists but cannot be read, such as
519    /// `ErrorKind::PermissionDenied`. A directory that does not exist yet is not an error; it
520    /// reports zero entries.
521    pub fn artifact_dir_stats(&self) -> std::io::Result<t4a_cuda_runtime::ArtifactDirStats> {
522        self.submit().artifact_dir_stats()
523    }
524
525    /// The bound on the persistent artifact directory, in bytes.
526    pub fn artifact_dir_max_bytes(&self) -> u64 {
527        self.submit().artifact_dir_max_bytes()
528    }
529
530    /// Remove every compiled image from the persistent artifact directory and return what went.
531    ///
532    /// Images already loaded by this process stay usable; only later processes recompile.
533    ///
534    /// # Errors
535    ///
536    /// The [`std::io::Error`] from listing the directory or removing an entry, such as
537    /// `ErrorKind::PermissionDenied` on a read-only cache. An entry another process removed
538    /// first (`ErrorKind::NotFound`) is not an error.
539    pub fn clear_artifact_dir(&self) -> std::io::Result<t4a_cuda_runtime::ArtifactDirStats> {
540        self.submit().clear_artifact_dir()
541    }
542
543    /// Images this runtime actually removed from the persistent artifact directory to stay within
544    /// its bound, least recently used first.
545    ///
546    /// Trimming is best effort, so a removal that failed is not counted here and the directory can
547    /// stay above its bound; [`CudaRuntime::artifact_dir_stats`] reports what is really there.
548    pub fn artifact_dir_evictions(&self) -> u64 {
549        self.submit().artifact_dir_evictions()
550    }
551
552    /// Run `closure` inside a raw scope on this runtime's stream, under the submission
553    /// lock.
554    ///
555    /// # Safety
556    ///
557    /// As [`crate::Runtime::with_raw`](t4a_cuda_runtime::Runtime::with_raw): the closure
558    /// must use this runtime's stream, keep its enqueues within the registered resources
559    /// and obey each kernel's or vendor library's contract.
560    pub(crate) unsafe fn with_raw_locked<R>(
561        &self,
562        resources: Vec<t4a_cuda_runtime::RawResource<'_>>,
563        closure: impl FnOnce(t4a_cuda_runtime::RawScope<'_>) -> R,
564    ) -> crate::Result<R> {
565        let mut guard = self.submit();
566        // SAFETY: forwarded to the caller's contract.
567        unsafe { guard.with_raw(resources, closure) }
568            .map_err(|error| crate::Error::runtime_state_source("cuda raw scope", error))
569    }
570
571    /// Run `f` with this runtime's CUDA context current, for a vendor call that only sets up state
572    /// (a cuFFT or cuSOLVER plan) and enqueues nothing.
573    ///
574    /// # Errors
575    ///
576    /// The closure's own failure, and the runtime's when the context cannot be activated.
577    #[doc(hidden)]
578    pub fn with_current_context<R>(
579        &self,
580        op: &'static str,
581        f: impl FnOnce() -> R,
582    ) -> crate::Result<R> {
583        // The scope holds the runtime's submission lock, so a nested call would deadlock on it (a
584        // cuFFT plan's rollback runs inside the same context). Nesting is tracked per thread:
585        // inside an activated context the closure simply runs, and the level is popped when this
586        // call leaves, panic included.
587        let activation = ActivatedContext::enter(self);
588        if activation.nested {
589            return Ok(f());
590        }
591        // SAFETY: the scope activates this runtime's context for the closure's duration; it declares
592        // no resources, so nothing is registered against the closure's work.
593        unsafe { self.with_raw_locked(Vec::new(), |_scope| f()) }
594            .map_err(|error| crate::Error::runtime_state_source(op, error))
595    }
596
597    /// Resource counters of this runtime.
598    ///
599    /// The snapshot is the runtime's own: live and pooled device bytes, pending
600    /// host/module bytes, retirement records and the transfer counters. The
601    /// compatibility package adds its detached-handle count beside them
602    /// ([`CudaRuntime::pending_transfer_retirements`]).
603    #[doc(hidden)]
604    pub fn stats(&self) -> t4a_cuda_runtime::Stats {
605        let stats = self.submit().stats();
606        self.retire_detached();
607        stats
608    }
609
610    /// Copy `value` from `self` into a fresh allocation of `destination`.
611    ///
612    /// The copy is ordered on both runtimes' streams: the producer may be dropped and
613    /// the value reused immediately, and a later read on the destination stream observes
614    /// the copied bytes without a host wait.
615    ///
616    /// # Errors
617    ///
618    /// A typed refusal when the two handles are the same runtime, when `value` is not a
619    /// CUDA allocation of this runtime, or when the driver cannot admit the copy.
620    #[doc(hidden)]
621    pub fn copy_to(
622        &self,
623        destination: &CudaRuntime,
624        value: &tenferro_tensor::Tensor,
625    ) -> crate::Result<tenferro_tensor::Tensor> {
626        use tenferro_tensor::{Complex32, Complex64, DType, TensorScalar, TypedTensor};
627
628        if self.identity() == destination.identity() {
629            return Err(crate::Error::runtime_state(
630                "cuda copy",
631                "the source and destination runtimes are the same",
632            ));
633        }
634        let storage = super::pending::cuda_storage(value).ok_or_else(|| {
635            crate::Error::runtime_state("cuda copy", "expected a CUDA allocation")
636        })?;
637        if storage.device_ordinal() != self.device_ordinal() {
638            return Err(crate::Error::runtime_state(
639                "cuda copy",
640                "the value belongs to another runtime",
641            ));
642        }
643
644        // Both runtimes participate in admission. Take their submission locks in a
645        // stable order (by allocation domain) so two opposing copies cannot invert them,
646        // while the copy direction stays source -> destination in both branches.
647        let source_is_lower = self.inner.allocation_domain < destination.inner.allocation_domain;
648        let (buffer, _completion) = if source_is_lower {
649            let mut source_guard = self.submit();
650            let mut destination_guard = destination.submit();
651            let (buffer, completion, _route) = destination_guard
652                .copy_between(&mut source_guard, storage.allocation())
653                .map_err(|error| crate::Error::runtime_state_source("cuda copy", error))?;
654            (buffer, completion)
655        } else {
656            let mut destination_guard = destination.submit();
657            let mut source_guard = self.submit();
658            let (buffer, completion, _route) = destination_guard
659                .copy_between(&mut source_guard, storage.allocation())
660                .map_err(|error| crate::Error::runtime_state_source("cuda copy", error))?;
661            (buffer, completion)
662        };
663        fn rebuild<T: TensorScalar + Send + Sync + 'static>(
664            value: &tenferro_tensor::Tensor,
665            buffer: t4a_cuda_runtime::Buffer,
666            destination: &CudaRuntime,
667        ) -> crate::Result<tenferro_tensor::Tensor> {
668            let typed: &TypedTensor<T> = value.as_typed::<T>().ok_or_else(|| {
669                crate::Error::runtime_state("cuda copy", "expected a preset scalar")
670            })?;
671            let storage = super::storage::fresh_storage(destination, buffer);
672            TypedTensor::<T>::from_buffer_col_major(
673                typed.shape().to_vec(),
674                tenferro_tensor::StorageBuffer::Backend(Box::new(storage)),
675                super::transfer::placement(destination),
676            )
677            .map(tenferro_tensor::Tensor::from_typed)
678        }
679        match value.dtype() {
680            DType::F64 => rebuild::<f64>(value, buffer, destination),
681            DType::F32 => rebuild::<f32>(value, buffer, destination),
682            DType::I32 => rebuild::<i32>(value, buffer, destination),
683            DType::I64 => rebuild::<i64>(value, buffer, destination),
684            DType::C32 => rebuild::<Complex32>(value, buffer, destination),
685            DType::C64 => rebuild::<Complex64>(value, buffer, destination),
686            DType::Bool | DType::External(_) => Err(crate::Error::unsupported(
687                "cuda copy",
688                "the transfer route does not serve this dtype",
689            )),
690        }
691    }
692
693    /// Read a runtime allocation back into a byte vector.
694    ///
695    /// # Errors
696    ///
697    /// [`cuda::CudaDeviceError::Initialization`] when the copy cannot be admitted or
698    /// completed.
699    ///
700    /// [`cuda::CudaDeviceError::Initialization`]: crate::cuda::CudaDeviceError::Initialization
701    pub fn download_bytes(&self, buffer: &t4a_cuda_runtime::Buffer) -> crate::Result<Vec<u8>> {
702        let mut guard = self.submit();
703        let mut pending = guard
704            .download(buffer)
705            .map_err(|error| crate::Error::runtime_state_source("cuda download", error))?;
706        pending
707            .wait()
708            .map_err(|error| crate::Error::runtime_state_source("cuda download", error))
709    }
710
711    /// Admit one raw host-to-device upload and return its allocation.
712    ///
713    /// This is the seam's own upload path, used by callers that already hold bytes (the
714    /// publication test queues work with it); a typed [`upload_tensor`] wraps it with
715    /// the tensor and placement metadata.
716    ///
717    /// [`upload_tensor`]: super::upload_tensor
718    ///
719    /// # Errors
720    ///
721    /// [`CudaDeviceError::Initialization`] when the runtime cannot allocate or enqueue
722    /// the copy.
723    #[doc(hidden)]
724    pub fn upload_bytes(&self, bytes: &[u8]) -> Result<t4a_cuda_runtime::Buffer, CudaDeviceError> {
725        let mut guard = self.submit();
726        guard
727            .upload(bytes)
728            .map_err(|error| CudaDeviceError::Initialization {
729                device: self.inner.device_id,
730                operation: "t4a_cuda_runtime::Runtime::upload",
731                source: Box::new(error),
732            })
733    }
734
735    /// Allocate a pinned host buffer owned by this runtime.
736    ///
737    /// # Errors
738    ///
739    /// [`CudaDeviceError::Initialization`] when the driver cannot allocate pinned
740    /// memory.
741    #[doc(hidden)]
742    pub(crate) fn alloc_pinned(
743        &self,
744        bytes: usize,
745    ) -> Result<t4a_cuda_runtime::PinnedHostBuffer, CudaDeviceError> {
746        let mut guard = self.submit();
747        guard
748            .alloc_pinned(bytes)
749            .map_err(|error| CudaDeviceError::Initialization {
750                device: self.inner.device_id,
751                operation: "t4a_cuda_runtime::Runtime::alloc_pinned",
752                source: Box::new(error),
753            })
754    }
755
756    /// Pinned staging allocations this runtime made, whoever asked.
757    ///
758    /// One for the first download and one per later growth of the staging buffer: a download that
759    /// allocated per call would count every download instead.
760    #[doc(hidden)]
761    #[must_use]
762    pub fn pinned_staging_allocations(&self) -> u64 {
763        self.inner.pinned_staging_accounting.allocations()
764    }
765
766    /// What the download staging currently retains: bytes, pinned allocations, and the bound.
767    ///
768    /// Downloads reuse one page-locked host buffer, because pinning it costs about 1.9 ms beside a
769    /// 19 us copy. The buffer is memory the OS cannot reclaim, so it is bounded (see
770    /// [`CudaRuntime::set_pinned_staging_limit_bytes`]) and can be released early with
771    /// [`CudaRuntime::clear_pinned_staging`].
772    ///
773    /// # Examples
774    ///
775    /// ```
776    /// # fn main() -> Result<(), tenferro_gpu::cuda::CudaEngineError> {
777    /// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaRuntime};
778    ///
779    /// # if gpu_available() {
780    /// let runtime = CudaRuntime::new(CudaDeviceId::from_ordinal(0))?;
781    /// let stats = runtime.pinned_staging_stats();
782    /// println!("{} bytes retained, {} allocations", stats.retained_bytes, stats.allocations);
783    /// # }
784    /// # Ok(())
785    /// # }
786    /// ```
787    #[must_use]
788    pub fn pinned_staging_stats(&self) -> PinnedStagingStats {
789        let accounting = &self.inner.pinned_staging_accounting;
790        PinnedStagingStats {
791            retained_bytes: accounting.retained(),
792            allocations: accounting.allocations(),
793            limit_bytes: accounting.limit(),
794        }
795    }
796
797    /// Bound the download staging, releasing a larger buffer at once.
798    ///
799    /// A download above this size still stages through host memory, but through a buffer released
800    /// when that download returns, so nothing above the bound stays resident. Lowering the bound
801    /// below what is retained releases the cached buffer; `usize::MAX` keeps every download.
802    ///
803    /// # Examples
804    ///
805    /// ```
806    /// # fn main() -> Result<(), tenferro_gpu::cuda::CudaEngineError> {
807    /// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaRuntime};
808    ///
809    /// # if gpu_available() {
810    /// let runtime = CudaRuntime::new(CudaDeviceId::from_ordinal(0))?;
811    /// runtime.set_pinned_staging_limit_bytes(1024 * 1024);
812    /// assert_eq!(runtime.pinned_staging_stats().limit_bytes, 1024 * 1024);
813    /// # }
814    /// # Ok(())
815    /// # }
816    /// ```
817    pub fn set_pinned_staging_limit_bytes(&self, limit: usize) {
818        self.inner.pinned_staging_accounting.set_limit(limit);
819        self.release_staging_above(limit);
820    }
821
822    /// Release the retained download staging now.
823    ///
824    /// The next download pins a buffer again. Returns the bytes released, so a caller can tell the
825    /// call apart from one that found nothing to release.
826    ///
827    /// # Examples
828    ///
829    /// ```
830    /// # fn main() -> Result<(), tenferro_gpu::cuda::CudaEngineError> {
831    /// use tenferro_gpu::cuda::{gpu_available, CudaDeviceId, CudaRuntime};
832    ///
833    /// # if gpu_available() {
834    /// let runtime = CudaRuntime::new(CudaDeviceId::from_ordinal(0))?;
835    /// let released = runtime.clear_pinned_staging();
836    /// assert_eq!(runtime.pinned_staging_stats().retained_bytes, 0);
837    /// # let _ = released;
838    /// # }
839    /// # Ok(())
840    /// # }
841    /// ```
842    pub fn clear_pinned_staging(&self) -> usize {
843        self.release_staging_above(0)
844    }
845
846    /// Drop the cached buffer when it exceeds `limit`, and report the bytes actually released.
847    ///
848    /// The free runs inside this runtime's raw scope, which makes the owning context current on
849    /// this thread; without it `cuMemFreeHost` can fail with `CUDA_ERROR_INVALID_CONTEXT` on a
850    /// thread that never activated the context. A failed free keeps the buffer and its accounting,
851    /// because dropping the pointer would leak the allocation while reporting it released.
852    fn release_staging_above(&self, limit: usize) -> usize {
853        let Ok(mut slot) = self.inner.pinned_staging.lock() else {
854            return 0;
855        };
856        let Some(entry) = slot.as_ref() else {
857            return 0;
858        };
859        let released = entry.bytes();
860        if released == 0 || released <= limit {
861            return 0;
862        }
863        // No download is in flight: it would hold this lock, and it synchronizes before releasing
864        // it, so nothing queued still reads this buffer.
865        let pointer = entry.pointer();
866        let freed = unsafe {
867            self.with_raw_locked(Vec::new(), |_raw| {
868                cudarc::driver::sys::cuMemFreeHost(pointer)
869            })
870        };
871        match freed {
872            Ok(cudarc::driver::sys::CUresult::CUDA_SUCCESS) => {
873                *slot = None;
874                self.inner.pinned_staging_accounting.record_retained(0);
875                released
876            }
877            other => {
878                // Keep the buffer: it is still allocated, so the accounting stays truthful and a
879                // later clear can retry.
880                let _ = other;
881                0
882            }
883        }
884    }
885
886    /// Wait for every queued operation and retire the detached handles it completes.
887    ///
888    /// # Errors
889    ///
890    /// [`CudaDeviceError::Initialization`] when the stream reports a failure.
891    #[doc(hidden)]
892    pub fn synchronize(&self) -> Result<(), CudaDeviceError> {
893        self.inner
894            .stream_waits
895            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
896        {
897            // Hold the submission lock through the reclamation: a concurrent clone cannot
898            // enqueue a transfer it then detaches in the gap between the stream wait and
899            // the queue drain, which would free its pinned source while still queued.
900            let mut guard = self.submit();
901            guard
902                .synchronize()
903                .map_err(|error| CudaDeviceError::Initialization {
904                    device: self.inner.device_id,
905                    operation: "t4a_cuda_runtime::Runtime::synchronize",
906                    source: Box::new(error),
907                })?;
908            self.retire_detached_locked();
909        }
910        Ok(())
911    }
912
913    /// Completion checkpoints this runtime recorded on its stream.
914    ///
915    /// A scheduled run records one per node, unless it took the single-domain coalesced path,
916    /// where submission order is the ordering and the run's retirement is the only barrier.
917    /// Exposed so a regression test can pin that (#2396).
918    #[doc(hidden)]
919    pub fn completion_checkpoints(&self) -> u64 {
920        self.inner
921            .completion_checkpoints
922            .load(std::sync::atomic::Ordering::Relaxed)
923    }
924
925    /// Record one completion checkpoint here, for callers that own the submission lock.
926    pub(crate) fn note_completion_checkpoint(&self) {
927        self.inner
928            .completion_checkpoints
929            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
930    }
931
932    /// Stream barriers this runtime ran, including the ones a scheduled run's operations skip.
933    ///
934    /// A prepared run adds exactly the caller's own barrier; an operation that drains its own
935    /// launch adds one per launch. Exposed so the regression test can count real waits.
936    #[doc(hidden)]
937    pub fn stream_waits(&self) -> u64 {
938        self.inner
939            .stream_waits
940            .load(std::sync::atomic::Ordering::Relaxed)
941    }
942
943    /// Whether a scheduled run currently owns the completion of the launches it enqueues.
944    ///
945    /// A run that holds the deferral records a stream-ordered checkpoint after every node and is
946    /// the barrier for its own work (see the CUDA engine's event domain), so an operation's launch
947    /// does not prove its own completion while it is zero. A standalone call sees zero and waits.
948    #[doc(hidden)]
949    pub(crate) fn completion_deferred(&self) -> bool {
950        self.inner
951            .deferred_completions
952            .load(std::sync::atomic::Ordering::Relaxed)
953            > 0
954    }
955
956    /// Take a scheduled run's completion ownership for as long as the guard lives.
957    ///
958    /// Nested guards are counted, so a run enqueueing a node that itself starts a run keeps the
959    /// deferral in force until the outermost guard is dropped.
960    pub(crate) fn defer_completion(&self) -> CompletionDeferral<'_> {
961        self.inner
962            .deferred_completions
963            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
964        CompletionDeferral { runtime: self }
965    }
966
967    /// Idle device memory the allocation pool keeps mapped before returning it to the driver.
968    ///
969    /// Reported next to [`Self::stats`] because it is the bound the pool's retained bytes obey
970    /// (`PERFORMANCE_TIPS.md`, Cache Ownership): raise it with
971    /// `TENFERRO_CUDA_POOL_RELEASE_MIB` when a working set of live buffers exceeds it, which is
972    /// what made every eager 3072^3 output allocation re-map its pages (#2381).
973    #[doc(hidden)]
974    pub fn pool_release_threshold_bytes(&self) -> usize {
975        self.inner.pool_release_threshold
976    }
977
978    /// Return idle pool memory above `keep_bytes` to the driver now.
979    ///
980    /// The pool trims itself at the threshold recorded by
981    /// [`Self::pool_release_threshold_bytes`]; this is the explicit clear that a caller uses to
982    /// give device memory back, for example before handing the device to another library.
983    ///
984    /// # Errors
985    ///
986    /// [`CudaDeviceError::Initialization`] when the driver refuses the trim.
987    #[doc(hidden)]
988    pub fn trim_pool(&self, keep_bytes: usize) -> Result<(), CudaDeviceError> {
989        self.submit()
990            .trim_pool(keep_bytes)
991            .map_err(|error| CudaDeviceError::Initialization {
992                device: self.inner.device_id,
993                operation: "t4a_cuda_runtime::Runtime::trim_pool",
994                source: Box::new(error),
995            })
996    }
997
998    /// Detached pending handles that have not been retired yet, the compatibility
999    /// package's own count (upstream `pending_records` counts retirement epochs, a
1000    /// different quantity).
1001    #[doc(hidden)]
1002    pub fn pending_transfer_retirements(&self) -> u64 {
1003        self.inner
1004            .detached
1005            .lock()
1006            .unwrap_or_else(|poison| poison.into_inner())
1007            .len() as u64
1008    }
1009
1010    /// Retain a dropped handle until its completion is observed.
1011    pub(crate) fn detach(
1012        &self,
1013        keepalive: Box<dyn std::any::Any + Send>,
1014        completion: Option<t4a_cuda_runtime::Completion>,
1015    ) {
1016        self.inner
1017            .detached
1018            .lock()
1019            .unwrap_or_else(|poison| poison.into_inner())
1020            .push(Detached {
1021                keepalive,
1022                completion,
1023            });
1024    }
1025
1026    /// Release the completed detached packages while the caller already holds the
1027    /// submission lock.
1028    fn retire_detached_locked(&self) {
1029        let mut detached = self
1030            .inner
1031            .detached
1032            .lock()
1033            .unwrap_or_else(|poison| poison.into_inner());
1034        detached.retain_mut(|entry| match entry.completion.as_mut() {
1035            Some(completion) => !matches!(completion.poll(), Ok(Some(()))),
1036            None => false,
1037        });
1038    }
1039
1040    /// Release the detached packages whose completion is observed, keeping the rest.
1041    ///
1042    /// Reclamation is explicit: a package is only released once its own completion says
1043    /// the work is done, so a queued transfer never loses its source or destination.
1044    pub(crate) fn retire_detached(&self) {
1045        let mut detached = self
1046            .inner
1047            .detached
1048            .lock()
1049            .unwrap_or_else(|poison| poison.into_inner());
1050        detached.retain_mut(|entry| match entry.completion.as_mut() {
1051            Some(completion) => match completion.poll() {
1052                Ok(Some(())) => false,
1053                Ok(None) => true,
1054                // A failed completion is kept: the failure surfaces through the next
1055                // explicit observation rather than being dropped silently here.
1056                Err(_) => true,
1057            },
1058            None => false,
1059        });
1060    }
1061
1062    /// Synchronise and then reclaim what completed, for teardown paths that cannot
1063    /// report an error.
1064    pub(crate) fn synchronize_best_effort(&self) {
1065        {
1066            let mut guard = self.submit();
1067            let _ = guard.synchronize();
1068        }
1069        self.retire_detached();
1070    }
1071
1072    /// Take the submission lock for one explicit transfer.
1073    pub(crate) fn submit(&self) -> std::sync::MutexGuard<'_, t4a_cuda_runtime::Runtime> {
1074        self.inner
1075            .runtime
1076            .lock()
1077            .unwrap_or_else(|poison| poison.into_inner())
1078    }
1079}
1080
1081impl Drop for CudaRuntimeState {
1082    fn drop(&mut self) {
1083        // Last handle: drain the stream, then release the detached packages. A failure
1084        // leaves their completion unproved, so the queue is leaked instead of freeing
1085        // pinned memory a queued copy may still read or write.
1086        let proved = self
1087            .runtime
1088            .lock()
1089            .unwrap_or_else(|poison| poison.into_inner())
1090            .synchronize()
1091            .is_ok();
1092        if proved {
1093            let mut detached = self
1094                .detached
1095                .lock()
1096                .unwrap_or_else(|poison| poison.into_inner());
1097            detached.clear();
1098        } else {
1099            let leaked = std::mem::take(
1100                &mut *self
1101                    .detached
1102                    .lock()
1103                    .unwrap_or_else(|poison| poison.into_inner()),
1104            );
1105            std::mem::forget(leaked);
1106        }
1107    }
1108}
1109
1110/// One scheduled run's ownership of its nodes' launch completion, for as long as it is enqueued.
1111///
1112/// Dropping the guard hands completion back to the operations, which is what a standalone call
1113/// expects; it is deliberately trivial so no failure path can leave the count raised.
1114pub(crate) struct CompletionDeferral<'a> {
1115    runtime: &'a CudaRuntime,
1116}
1117
1118impl Drop for CompletionDeferral<'_> {
1119    fn drop(&mut self) {
1120        self.runtime
1121            .inner
1122            .deferred_completions
1123            .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
1124    }
1125}
1126
1127impl std::fmt::Debug for CompletionDeferral<'_> {
1128    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1129        f.write_str("CompletionDeferral")
1130    }
1131}
1132
1133impl std::fmt::Debug for CudaRuntime {
1134    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1135        f.debug_struct("CudaRuntime")
1136            .field("device", &self.inner.device_id)
1137            .field("domain", &self.inner.allocation_domain)
1138            .finish()
1139    }
1140}
1141
1142#[cfg(test)]
1143mod tests;