Skip to main content

tensor4all_tensorbackend/
context.rs

1//! Explicit and optional process-global tenferro CPU execution contexts.
2
3use std::cell::RefCell;
4#[cfg(feature = "global-defaults")]
5use std::collections::HashMap;
6use std::num::NonZeroUsize;
7use std::ops::{Deref, DerefMut};
8use std::sync::atomic::{AtomicU64, Ordering};
9#[cfg(feature = "global-defaults")]
10use std::sync::Weak;
11use std::sync::{Arc, Mutex, OnceLock};
12
13use tenferro::{
14    CompiledGraph, ExecutionEngine, GraphCompiler, Runtime, Tensor, TensorRead, TracedGraph,
15};
16#[cfg(feature = "global-defaults")]
17use tenferro_ad::ContextId;
18use tenferro_ad::{AdContext, EagerRuntime};
19use tenferro_cpu::{BufferPoolStats, CpuEngine, Pool, PoolBuilder};
20
21/// Linux CPU-affinity FFI, kept in one place so every `unsafe` block has one
22/// auditable owner.
23///
24/// The mask is a growable word buffer in the kernel's `sched_*affinity` layout.
25/// No fixed `cpu_set_t` is used: a mask larger than glibc's 1024-CPU set is
26/// retried with a larger buffer instead of failing, and CPU ids are validated
27/// before they index the buffer.
28#[cfg(target_os = "linux")]
29mod affinity {
30    use std::io;
31
32    use super::CpuThreadError;
33
34    const BITS_PER_WORD: usize = libc::c_ulong::BITS as usize;
35    const INITIAL_WORDS: usize = 1;
36    /// 512 KiB of mask on a 64-bit host, far beyond any current CPU count.
37    const MAX_WORDS: usize = 1 << 16;
38
39    /// One CPU set in the layout `sched_getaffinity` and `sched_setaffinity`
40    /// expect. The words are owned by the `Vec`, so the buffer is word-aligned.
41    #[derive(Clone, Debug)]
42    pub(super) struct CpuMask {
43        words: Vec<libc::c_ulong>,
44    }
45
46    impl CpuMask {
47        /// The mask holding exactly `cpu`, with at least `min_words` words.
48        ///
49        /// Callers validate `cpu` against the allowed set first, so the small
50        /// allocation is bounded by the current kernel mask. `min_words` keeps
51        /// the buffer at the kernel's mask size on hosts where the highest
52        /// allowed CPU is in a low word.
53        pub(super) fn single(cpu: usize, min_words: usize) -> Self {
54            let word = cpu / BITS_PER_WORD;
55            let bit = cpu % BITS_PER_WORD;
56            let mut words = vec![0; (word + 1).max(min_words)];
57            words[word] |= 1 << bit;
58            Self { words }
59        }
60
61        /// The number of words in this mask's kernel buffer.
62        pub(super) fn words_len(&self) -> usize {
63            self.words.len()
64        }
65
66        /// The CPU ids in this mask, in ascending order.
67        pub(super) fn cpus(&self) -> Vec<usize> {
68            let mut cpus = Vec::new();
69            for (index, &word) in self.words.iter().enumerate() {
70                let mut bits = word;
71                while bits != 0 {
72                    let bit = bits.trailing_zeros() as usize;
73                    cpus.push(index * BITS_PER_WORD + bit);
74                    bits &= bits - 1;
75                }
76            }
77            cpus
78        }
79
80        /// Whether this mask holds exactly the same CPUs as `other`.
81        pub(super) fn equals(&self, other: &Self) -> bool {
82            let len = self.words.len().max(other.words.len());
83            (0..len).all(|index| self.word(index) == other.word(index))
84        }
85
86        fn word(&self, index: usize) -> libc::c_ulong {
87            self.words.get(index).copied().unwrap_or(0)
88        }
89
90        fn as_bytes(&self) -> usize {
91            std::mem::size_of_val(self.words.as_slice())
92        }
93
94        fn as_ptr(&self) -> *const libc::c_void {
95            self.words.as_ptr().cast()
96        }
97    }
98
99    /// Read the mask of the thread identified by `pid` (`0` is the calling
100    /// thread).
101    fn get_mask(pid: libc::pid_t) -> io::Result<CpuMask> {
102        let mut words = vec![0 as libc::c_ulong; INITIAL_WORDS];
103        loop {
104            let bytes = std::mem::size_of_val(words.as_slice());
105            // SAFETY: `words` is an owned, word-aligned buffer and `bytes` is
106            // its exact byte length; the kernel writes at most that many bytes.
107            let result = unsafe { libc::sched_getaffinity(pid, bytes, words.as_mut_ptr().cast()) };
108            if result == 0 {
109                // Keep the successful kernel buffer size: `sched_setaffinity`
110                // requires at least that many bytes even when the high words
111                // are empty.
112                return Ok(CpuMask { words });
113            }
114            let error = io::Error::last_os_error();
115            if error.raw_os_error() != Some(libc::EINVAL) {
116                return Err(error);
117            }
118            let next = words.len().checked_mul(2).filter(|&len| len <= MAX_WORDS);
119            let Some(next) = next else {
120                return Err(io::Error::other(
121                    "CPU affinity mask is larger than the supported buffer",
122                ));
123            };
124            words.resize(next, 0);
125        }
126    }
127
128    /// Replace the mask of the thread identified by `pid` (`0` is the calling
129    /// thread) with `mask`.
130    fn set_mask(pid: libc::pid_t, mask: &CpuMask) -> io::Result<()> {
131        // SAFETY: `mask` owns a word-aligned buffer and `as_bytes` is its exact
132        // byte length; the kernel reads at most that many bytes.
133        let result = unsafe { libc::sched_setaffinity(pid, mask.as_bytes(), mask.as_ptr().cast()) };
134        if result == 0 {
135            Ok(())
136        } else {
137            Err(io::Error::last_os_error())
138        }
139    }
140
141    /// The main thread's mask at construction, a snapshot taken with
142    /// `sched_getaffinity(getpid())`.
143    ///
144    /// The process id names the thread-group leader, so a pool built lazily
145    /// from a worker of an already-pinned pool does not inherit that worker's
146    /// single CPU.
147    pub(super) fn main_thread_mask() -> Result<CpuMask, CpuThreadError> {
148        get_mask(std::process::id() as libc::pid_t).map_err(|source| CpuThreadError::Affinity {
149            operation: "sched_getaffinity",
150            source,
151        })
152    }
153
154    /// Set the calling thread's mask and verify the kernel kept all of it.
155    fn set_and_verify_current_thread(mask: &CpuMask) -> io::Result<()> {
156        set_mask(0, mask)?;
157        let actual = get_mask(0)?;
158        if actual.equals(mask) {
159            Ok(())
160        } else {
161            Err(io::Error::other(
162                "the effective CPU mask differs from the requested mask",
163            ))
164        }
165    }
166
167    /// Set the calling thread's mask to `mask`.
168    ///
169    /// A threaded cgroup may silently intersect the request with a narrower
170    /// cpuset, so the read-back is what turns a partially applied mask into a
171    /// typed failure before Rayon builds any worker.
172    pub(super) fn set_current_thread_to(mask: &CpuMask) -> Result<(), CpuThreadError> {
173        set_and_verify_current_thread(mask).map_err(|source| CpuThreadError::Affinity {
174            operation: "sched_setaffinity",
175            source,
176        })
177    }
178
179    /// Pin the calling thread to exactly `cpu` and verify the effective mask.
180    pub(super) fn pin_current_thread(cpu: usize) -> io::Result<()> {
181        let min_words = get_mask(0)?.words_len();
182        let expected = CpuMask::single(cpu, min_words);
183        set_and_verify_current_thread(&expected)
184    }
185
186    /// The calling thread's effective CPU ids.
187    pub(super) fn current_thread_cpus() -> Result<Vec<usize>, CpuThreadError> {
188        get_mask(0)
189            .map(|mask| mask.cpus())
190            .map_err(|source| CpuThreadError::Affinity {
191                operation: "sched_getaffinity",
192                source,
193            })
194    }
195}
196
197/// Failure reported when building a [`CpuBackend`] pool or applying its
198/// affinity policy.
199///
200/// A pool build and, on Linux, the affinity snapshot are the fallible steps of a
201/// [`CpuBackend`]: tenferro itself never builds threads, so the caller's thread
202/// policy is the caller's error to
203/// carry.
204///
205/// # Examples
206///
207/// ```
208/// use tensor4all_tensorbackend::CpuThreadError;
209///
210/// let error = CpuThreadError::ZeroThreads;
211/// assert!(error.to_string().contains("at least one"));
212/// ```
213#[derive(Debug, thiserror::Error)]
214pub enum CpuThreadError {
215    /// A [`CpuBackend`] was asked for zero workers.
216    #[error("a CPU backend needs at least one worker thread")]
217    ZeroThreads,
218    /// The worker pool for the requested worker count could not be built.
219    #[error("failed to build the CPU worker pool: {source}")]
220    Pool {
221        /// Original pool diagnostic.
222        #[source]
223        source: Box<tenferro_cpu::threadteam::Error>,
224    },
225    /// [`CpuBackend::pinned`] was given an empty CPU list.
226    #[error("a pinned CPU backend needs at least one CPU")]
227    EmptyCpuList,
228    /// [`CpuBackend::pinned`] was given the same CPU twice.
229    #[error("the pinned CPU list repeats CPU {cpu}")]
230    DuplicateCpu {
231        /// The repeated CPU id.
232        cpu: usize,
233    },
234    /// [`CpuBackend::pinned`] named a CPU outside the main thread's mask.
235    #[error("CPU {cpu} is outside the main thread's allowed CPU set")]
236    CpuNotAllowed {
237        /// The rejected CPU id.
238        cpu: usize,
239    },
240    /// CPU affinity is only implemented on Linux.
241    #[error("CPU affinity is not supported on this platform")]
242    UnsupportedPlatform,
243    /// An affinity system call failed.
244    #[error("{operation} failed: {source}")]
245    Affinity {
246        /// The system call that failed.
247        operation: &'static str,
248        /// Original operating-system diagnostic.
249        #[source]
250        source: std::io::Error,
251    },
252    /// The short-lived thread that builds a pool could not be started.
253    #[error("failed to start the CPU pool construction thread: {source}")]
254    ConstructionThread {
255        /// Original thread-spawn diagnostic.
256        #[source]
257        source: std::io::Error,
258    },
259    /// The CPU pool construction thread panicked before returning the pool.
260    #[error("the CPU pool construction thread panicked")]
261    ConstructionThreadPanic,
262    /// A pinned worker could not be confined to its CPU, or its startup
263    /// handshake failed.
264    #[error("CPU worker {worker} failed its affinity handshake: {source}")]
265    WorkerAffinity {
266        /// Rayon worker index, or zero for a closed startup channel.
267        worker: usize,
268        /// Original operating-system or verification diagnostic.
269        #[source]
270        source: std::io::Error,
271    },
272}
273
274/// Failure reported by [`set_default_cpu_threads`] when the default is fixed.
275///
276/// # Examples
277///
278/// ```
279/// use tensor4all_tensorbackend::AlreadyInitialized;
280///
281/// let error = AlreadyInitialized;
282/// assert_eq!(
283///     error.to_string(),
284///     "the process-wide default CPU thread count is already initialized"
285/// );
286/// ```
287#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
288#[error("the process-wide default CPU thread count is already initialized")]
289pub struct AlreadyInitialized;
290
291static DEFAULT_CPU_THREADS: OnceLock<usize> = OnceLock::new();
292
293/// Fix the worker count [`CpuBackend::new`] uses for the rest of the process.
294///
295/// The value is read once, by the first [`CpuBackend::new`] call that resolves a
296/// count; a backend built later keeps whatever policy it was built with. A
297/// first [`CpuBackend::new`] that cannot build its pool falls back to serial for
298/// that backend only and does not fix the default, so a later call tries again.
299/// Call this before any other thread can construct a default backend.
300///
301/// # Errors
302///
303/// Returns [`AlreadyInitialized`] when the default thread configuration was
304/// already fixed successfully, so a change is unsupported: either an earlier
305/// call or a [`CpuBackend::new`] that resolved a count fixed it. A failed first
306/// construction does not fix it.
307///
308/// # Examples
309///
310/// ```
311/// use std::num::NonZeroUsize;
312/// use tensor4all_tensorbackend::{set_default_cpu_threads, AlreadyInitialized};
313///
314/// // Whichever call sets the default first wins, and every later one is
315/// // rejected typed instead of panicking.
316/// let _ = set_default_cpu_threads(NonZeroUsize::new(3).unwrap());
317/// assert_eq!(
318///     set_default_cpu_threads(NonZeroUsize::MAX),
319///     Err(AlreadyInitialized)
320/// );
321/// ```
322pub fn set_default_cpu_threads(threads: NonZeroUsize) -> Result<(), AlreadyInitialized> {
323    DEFAULT_CPU_THREADS
324        .set(threads.get())
325        .map_err(|_| AlreadyInitialized)
326}
327
328/// How the idle workers of a CPU pool wait for the next parallel section.
329///
330/// A worker that has nothing to do polls for a while before it sleeps. With the
331/// default budget it keeps polling through a serial phase of the caller that is
332/// shorter than that budget, so a pool of `N` workers can use `N` times the wall
333/// time in CPU time. A smaller budget trades that CPU time for a longer wake-up
334/// of the next parallel section. The names follow OpenMP's `OMP_WAIT_POLICY`.
335///
336/// The budget is threadteam's `PoolBuilder::spin_polls`, a count of polls and
337/// not a duration, so how long it lasts depends on the CPU.
338///
339/// # Examples
340///
341/// ```
342/// use tensor4all_tensorbackend::CpuWaitPolicy;
343///
344/// assert_eq!(CpuWaitPolicy::default(), CpuWaitPolicy::Active);
345/// assert_eq!(CpuWaitPolicy::Passive.spin_polls(), Some(0));
346/// assert_eq!(CpuWaitPolicy::SpinPolls(4096).spin_polls(), Some(4096));
347/// ```
348#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
349pub enum CpuWaitPolicy {
350    /// Keep the pool's default spin budget (the behaviour before this type
351    /// existed).
352    #[default]
353    Active,
354    /// Do not spin: a waiting worker yields a few times and then sleeps, for a
355    /// shared machine or parallel test processes.
356    Passive,
357    /// Spin for this many polls before the yields and the sleep.
358    SpinPolls(u32),
359}
360
361impl CpuWaitPolicy {
362    /// The spin budget this policy sets on the pool, or `None` for the pool's
363    /// default.
364    ///
365    /// # Examples
366    ///
367    /// ```
368    /// use tensor4all_tensorbackend::CpuWaitPolicy;
369    ///
370    /// assert_eq!(CpuWaitPolicy::Active.spin_polls(), None);
371    /// assert_eq!(CpuWaitPolicy::SpinPolls(7).spin_polls(), Some(7));
372    /// ```
373    pub fn spin_polls(self) -> Option<u32> {
374        match self {
375            Self::Active => None,
376            Self::Passive => Some(0),
377            Self::SpinPolls(polls) => Some(polls),
378        }
379    }
380}
381
382/// Failure reported by [`set_default_cpu_wait_policy`] when the default is fixed.
383///
384/// # Examples
385///
386/// ```
387/// use tensor4all_tensorbackend::WaitPolicyAlreadyInitialized;
388///
389/// assert_eq!(
390///     WaitPolicyAlreadyInitialized.to_string(),
391///     "the process-wide default CPU wait policy is already initialized"
392/// );
393/// ```
394#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
395#[error("the process-wide default CPU wait policy is already initialized")]
396pub struct WaitPolicyAlreadyInitialized;
397
398static DEFAULT_CPU_WAIT_POLICY: OnceLock<CpuWaitPolicy> = OnceLock::new();
399
400/// Fix the [`CpuWaitPolicy`] of the pools that [`CpuBackend::new`] and
401/// [`CpuBackend::with_threads`] build for the rest of the process.
402///
403/// Only pools built after the call use it; a backend that already exists keeps
404/// the policy it was built with. Call it before any other thread can construct a
405/// backend. Without a call the policy is [`CpuWaitPolicy::Active`].
406///
407/// # Errors
408///
409/// Returns [`WaitPolicyAlreadyInitialized`] when the process-wide wait-policy
410/// configuration was already fixed by an earlier call, so a change is
411/// unsupported and the first value stays. Build a backend with
412/// [`CpuBackend::with_threads_and_wait_policy`] to use another policy for one
413/// backend.
414///
415/// # Examples
416///
417/// ```
418/// use tensor4all_tensorbackend::{
419///     default_cpu_wait_policy, set_default_cpu_wait_policy, CpuWaitPolicy,
420///     WaitPolicyAlreadyInitialized,
421/// };
422///
423/// // Whichever call sets the default first wins; a later one is rejected typed.
424/// let _ = set_default_cpu_wait_policy(CpuWaitPolicy::Passive);
425/// assert_eq!(
426///     set_default_cpu_wait_policy(CpuWaitPolicy::SpinPolls(1)),
427///     Err(WaitPolicyAlreadyInitialized)
428/// );
429/// assert_eq!(default_cpu_wait_policy(), CpuWaitPolicy::Passive);
430/// ```
431pub fn set_default_cpu_wait_policy(
432    policy: CpuWaitPolicy,
433) -> Result<(), WaitPolicyAlreadyInitialized> {
434    DEFAULT_CPU_WAIT_POLICY
435        .set(policy)
436        .map_err(|_| WaitPolicyAlreadyInitialized)
437}
438
439/// The [`CpuWaitPolicy`] new default pools use: the one
440/// [`set_default_cpu_wait_policy`] fixed, else [`CpuWaitPolicy::Active`].
441///
442/// # Examples
443///
444/// ```
445/// use tensor4all_tensorbackend::{default_cpu_wait_policy, CpuWaitPolicy};
446///
447/// // A process that never set it keeps today's behaviour.
448/// assert_eq!(default_cpu_wait_policy(), CpuWaitPolicy::Active);
449/// ```
450pub fn default_cpu_wait_policy() -> CpuWaitPolicy {
451    DEFAULT_CPU_WAIT_POLICY.get().copied().unwrap_or_default()
452}
453
454/// The CPU set each worker of a pool reported when it started, by worker index.
455type WorkerCpus = Vec<Vec<usize>>;
456
457/// The calling thread's CPU set, empty off Linux.
458#[cfg(target_os = "linux")]
459fn current_worker_cpus() -> Vec<usize> {
460    affinity::current_thread_cpus().unwrap_or_default()
461}
462
463/// The calling thread's CPU set, empty off Linux.
464#[cfg(not(target_os = "linux"))]
465fn current_worker_cpus() -> Vec<usize> {
466    Vec::new()
467}
468
469/// Build one worker pool with `threads` (or the environment's count when
470/// `None`), recording the CPU set each worker starts on.
471///
472/// On Linux the build runs on a short-lived construction thread whose mask is
473/// first set to the main thread's mask snapshot, so a pool built from a singly
474/// pinned worker still spans the main thread's CPUs. The workers inherit the
475/// construction thread's mask.
476///
477/// Off Linux the pool is built on the calling thread and no affinity call is
478/// made.
479fn build_pool(
480    threads: Option<usize>,
481    wait: CpuWaitPolicy,
482) -> Result<(Pool, WorkerCpus), CpuThreadError> {
483    #[cfg(target_os = "linux")]
484    {
485        let mask = affinity::main_thread_mask()?;
486        std::thread::Builder::new()
487            .name("tensor4all-cpu-pool".to_owned())
488            .spawn(move || {
489                affinity::set_current_thread_to(&mask)?;
490                build_pool_here(threads, wait)
491            })
492            .map_err(|source| CpuThreadError::ConstructionThread { source })?
493            .join()
494            .map_err(|_| CpuThreadError::ConstructionThreadPanic)?
495    }
496    #[cfg(not(target_os = "linux"))]
497    {
498        build_pool_here(threads, wait)
499    }
500}
501
502/// The worker count Rayon itself resolves from the environment, in Rayon's
503/// order: `RAYON_NUM_THREADS`, the deprecated `RAYON_RS_NUM_CPUS`, then
504/// [`std::thread::available_parallelism`] (which already accounts for the
505/// process affinity mask and cgroup quotas). `RAYON_NUM_THREADS=0` or a
506/// malformed value means "use the platform default", as in Rayon.
507///
508/// The pool builder this type uses does not read the environment, so the
509/// precedence of [`set_default_cpu_threads`] → `RAYON_NUM_THREADS` → affinity
510/// is resolved here (R-performance-61).
511fn rayon_env_worker_count() -> usize {
512    let platform_default = || std::thread::available_parallelism().map_or(1, NonZeroUsize::get);
513    match std::env::var("RAYON_NUM_THREADS")
514        .ok()
515        .and_then(|value| value.parse::<usize>().ok())
516    {
517        Some(count) if count >= 1 => return count,
518        Some(_) => return platform_default(),
519        None => {}
520    }
521    match std::env::var("RAYON_RS_NUM_CPUS")
522        .ok()
523        .and_then(|value| value.parse::<usize>().ok())
524    {
525        Some(count) if count >= 1 => count,
526        _ => platform_default(),
527    }
528}
529
530/// `builder` with the spin budget of `wait`; the pool's default for [`CpuWaitPolicy::Active`].
531fn with_wait_policy(builder: PoolBuilder, wait: CpuWaitPolicy) -> PoolBuilder {
532    match wait.spin_polls() {
533        Some(polls) => builder.spin_polls(polls),
534        None => builder,
535    }
536}
537
538/// Build one worker pool on the calling thread.
539fn build_pool_here(
540    threads: Option<usize>,
541    wait: CpuWaitPolicy,
542) -> Result<(Pool, WorkerCpus), CpuThreadError> {
543    let threads = threads.unwrap_or_else(rayon_env_worker_count);
544    let (sender, receiver) = std::sync::mpsc::channel();
545    let builder = Pool::builder().threads(threads).on_start(move |worker| {
546        let _ = sender.send((worker, current_worker_cpus()));
547    });
548    let pool = with_wait_policy(builder, wait)
549        .build()
550        .map_err(|source| CpuThreadError::Pool {
551            source: Box::new(source),
552        })?;
553    // A worker runs its start hook in its own thread, so the reports arrive
554    // while and after the pool is built; collect one per worker before the
555    // backend exposes them.
556    let mut worker_cpus: WorkerCpus = vec![Vec::new(); pool.width().get()];
557    for _ in 0..worker_cpus.len() {
558        match receiver.recv_timeout(std::time::Duration::from_secs(1)) {
559            Ok((worker, cpus)) => {
560                if let Some(slot) = worker_cpus.get_mut(worker) {
561                    *slot = cpus;
562                }
563            }
564            // A worker that never reports leaves its entry empty rather than
565            // blocking the pool build.
566            Err(_) => break,
567        }
568    }
569    Ok((pool, worker_cpus))
570}
571
572/// tensor4all's owned CPU execution handle: one [`CpuEngine`].
573///
574/// tenferro no longer has a backend session or a `CpuContext`; a caller builds
575/// a Rayon pool, hands it to [`CpuEngine::new`], and keeps the engine for as
576/// long as it wants the thread policy, the buffer pool, the prepared-plan cache
577/// and the contraction scratch to stay warm. This type is that caller-held
578/// owner for tensor4all: the implicit frontend's resource owner during the
579/// transition to engine-explicit APIs, which will replace it later.
580///
581/// It is **not** [`Clone`]. Cloning an execution domain was how the previous
582/// session API shared one engine between callers; with a caller-held engine the
583/// caller owns it outright, and a second execution domain needs a second
584/// backend.
585///
586/// [`CpuBackend::new`] resolves its worker count in this order: the value
587/// [`set_default_cpu_threads`] fixed, else the Rayon environment
588/// (`RAYON_NUM_THREADS`, then the deprecated `RAYON_RS_NUM_CPUS`), else the
589/// platform's parallelism, which is evaluated under the main thread's mask and
590/// may be lower than that mask's CPU count because of cgroup quotas. A
591/// resolved count of one uses [`CpuEngine::serial`] and owns no pool. The count
592/// is fixed by the first successful [`CpuBackend::new`] of the process, or by the
593/// setter. If the pool or the Linux mask snapshot cannot be used,
594/// [`CpuBackend::new`] falls back to [`CpuEngine::serial`] for that backend only
595/// instead of panicking, and a later call retries.
596///
597/// Every pool this type builds on Linux confines its workers to the main
598/// thread's allowed CPU set, captured as a snapshot when the pool is built.
599/// This applies to the pool's workers only: operations that run inline on the
600/// caller (small operations, Rayon-worker entries that use the serial inline
601/// engine, and hataori lane threads) are not moved. A cpuset or CPU-hotplug
602/// change needs a new backend, not a continuous watcher.
603///
604/// [`CpuBackend::pinned`] is an opt-in per-worker pinning mode whose benefit is
605/// unmeasured. Disjoint pinned backends confine their *pools'* workers; they do
606/// not isolate all execution of their engines, because an entry from any Rayon
607/// worker runs inline on that worker. A provider team created from a pinned
608/// worker inherits that worker's one CPU.
609///
610/// # Examples
611///
612/// ```
613/// use tensor4all_tensorbackend::CpuBackend;
614/// use tenferro_cpu::CpuEngine;
615///
616/// // A serial backend runs every operation on the caller's thread.
617/// let backend = CpuBackend::with_threads(1)?;
618/// assert_eq!(backend.num_threads(), 1);
619///
620/// // A caller that already owns an engine can hand it in unchanged.
621/// let backend = CpuBackend::from(CpuEngine::serial());
622/// assert_eq!(backend.num_threads(), 1);
623/// # Ok::<(), Box<dyn std::error::Error>>(())
624/// ```
625pub struct CpuBackend {
626    engine: CpuEngine,
627    /// The CPU set each pool worker reported at start, by worker index; empty
628    /// for a serial engine or a caller-supplied engine.
629    worker_cpus: WorkerCpus,
630}
631
632impl std::fmt::Debug for CpuBackend {
633    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
634        f.debug_struct("CpuBackend")
635            .field("engine", &self.engine)
636            .field("worker_cpus", &self.worker_cpus())
637            .finish()
638    }
639}
640
641impl CpuBackend {
642    /// Build a backend with the default thread policy.
643    ///
644    /// The worker count is [`set_default_cpu_threads`] if it was called; else
645    /// `RAYON_NUM_THREADS` (then the deprecated `RAYON_RS_NUM_CPUS`); else the
646    /// platform's parallelism, evaluated under the main thread's mask, which may
647    /// be lower than that mask's CPU count because of cgroup quotas. The pool's
648    /// default 16 MiB worker stack, which the CPU providers need, is kept. The
649    /// count is fixed by the first successful call, or by the setter.
650    ///
651    /// On Linux the pool is built on a short-lived construction thread whose
652    /// mask is first set to the main thread's mask snapshot, so building from a
653    /// worker of a pinned pool does not inherit that worker's single CPU. The
654    /// pool's workers inherit the snapshot; inline operations of the caller are
655    /// not moved.
656    ///
657    /// The pool build and the Linux affinity snapshot are the fallible steps,
658    /// and this constructor does not report them: if either cannot be used,
659    /// this backend is [`CpuEngine::serial`] instead, which is correct but uses
660    /// no worker threads, and the failure is not stored as the default. A later
661    /// call retries the construction. Use [`CpuBackend::with_threads`] or
662    /// [`CpuBackend::pinned`] when that difference matters.
663    ///
664    /// # Examples
665    ///
666    /// ```
667    /// use tensor4all_tensorbackend::CpuBackend;
668    ///
669    /// let backend = CpuBackend::new();
670    /// assert!(backend.num_threads() >= 1);
671    /// ```
672    pub fn new() -> Self {
673        if let Some(&threads) = DEFAULT_CPU_THREADS.get() {
674            return Self::with_threads_or_serial(threads);
675        }
676        match build_pool(None, default_cpu_wait_policy()) {
677            Ok((pool, worker_cpus)) => {
678                let resolved = pool.width().get();
679                let fixed = match DEFAULT_CPU_THREADS.set(resolved) {
680                    Ok(()) => resolved,
681                    Err(_) => *DEFAULT_CPU_THREADS.get().unwrap_or(&resolved),
682                };
683                if fixed != resolved {
684                    return Self::with_threads_or_serial(fixed);
685                }
686                if resolved == 1 {
687                    // The pool is dropped with this branch: the backend is serial
688                    // and owns no workers, so it records none.
689                    Self {
690                        engine: CpuEngine::serial(),
691                        worker_cpus: WorkerCpus::default(),
692                    }
693                } else {
694                    Self {
695                        engine: CpuEngine::new(pool),
696                        worker_cpus,
697                    }
698                }
699            }
700            Err(_) => Self {
701                engine: CpuEngine::serial(),
702                worker_cpus: WorkerCpus::default(),
703            },
704        }
705    }
706
707    /// Build a backend with `threads` workers, or serial for one.
708    fn with_threads_or_serial(threads: usize) -> Self {
709        Self::with_threads(threads).unwrap_or_else(|_| Self {
710            engine: CpuEngine::serial(),
711            worker_cpus: WorkerCpus::default(),
712        })
713    }
714
715    /// Build a backend that may use `threads` workers.
716    ///
717    /// One worker builds a [`CpuEngine::serial`] engine, which owns no pool, so
718    /// it stays usable from inside another pool. Larger counts build a Rayon
719    /// pool with [`tenferro_cpu::Pool::builder`], so provider code gets the
720    /// worker stack it needs; the pool's workers are confined to the main
721    /// thread's allowed CPU set snapshot on Linux.
722    ///
723    /// # Errors
724    ///
725    /// Returns [`CpuThreadError::ZeroThreads`] for zero,
726    /// [`CpuThreadError::Affinity`] when the mask snapshot or its application
727    /// fails, [`CpuThreadError::ConstructionThread`] when the construction
728    /// thread cannot start, and [`CpuThreadError::Pool`] when the pool cannot be
729    /// built.
730    ///
731    /// # Examples
732    ///
733    /// ```
734    /// use tensor4all_tensorbackend::{CpuBackend, CpuThreadError};
735    ///
736    /// assert!(matches!(
737    ///     CpuBackend::with_threads(0),
738    ///     Err(CpuThreadError::ZeroThreads)
739    /// ));
740    /// let backend = CpuBackend::with_threads(2)?;
741    /// assert_eq!(backend.num_threads(), 2);
742    /// # Ok::<(), Box<dyn std::error::Error>>(())
743    /// ```
744    pub fn with_threads(threads: usize) -> Result<Self, CpuThreadError> {
745        Self::with_threads_and_wait_policy(threads, default_cpu_wait_policy())
746    }
747
748    /// Build a backend that may use `threads` workers, whose idle workers wait
749    /// as `wait` says.
750    ///
751    /// [`CpuBackend::with_threads`] is this with [`default_cpu_wait_policy`].
752    /// One worker builds a [`CpuEngine::serial`] engine, which owns no pool and
753    /// so has nothing to wait for; `wait` is then unused.
754    ///
755    /// # Errors
756    ///
757    /// The same errors as [`CpuBackend::with_threads`].
758    ///
759    /// # Examples
760    ///
761    /// ```
762    /// use tensor4all_tensorbackend::{CpuBackend, CpuWaitPolicy};
763    ///
764    /// let backend = CpuBackend::with_threads_and_wait_policy(2, CpuWaitPolicy::Passive)?;
765    /// assert_eq!(backend.num_threads(), 2);
766    /// # Ok::<(), Box<dyn std::error::Error>>(())
767    /// ```
768    pub fn with_threads_and_wait_policy(
769        threads: usize,
770        wait: CpuWaitPolicy,
771    ) -> Result<Self, CpuThreadError> {
772        if threads == 0 {
773            return Err(CpuThreadError::ZeroThreads);
774        }
775        if threads == 1 {
776            return Ok(Self {
777                engine: CpuEngine::serial(),
778                worker_cpus: WorkerCpus::default(),
779            });
780        }
781        let (pool, worker_cpus) = build_pool(Some(threads), wait)?;
782        Ok(Self {
783            engine: CpuEngine::new(pool),
784            worker_cpus,
785        })
786    }
787
788    /// Build a backend whose worker `i` is pinned to `cpus[i]`.
789    ///
790    /// This is opt-in per-worker pinning on Linux; its performance benefit is
791    /// unmeasured. Every listed CPU must be in the main thread's allowed set
792    /// ([`Self::allowed_cpus`]), which lets a caller split that set into
793    /// disjoint lists, one per engine. Each worker gets exactly its one CPU,
794    /// so a provider team created from a pinned worker inherits that one CPU.
795    /// A one-element list still builds a one-worker pool rather than a serial
796    /// engine; inline operations of a one-worker engine still run on the
797    /// caller.
798    ///
799    /// The mask is a snapshot: a cpuset or CPU-hotplug change needs a new
800    /// backend. Only the pool's workers are pinned; lane threads and inline
801    /// operations are not moved.
802    ///
803    /// # Errors
804    ///
805    /// Returns [`CpuThreadError::EmptyCpuList`] for an empty list,
806    /// [`CpuThreadError::DuplicateCpu`] for a repeated CPU,
807    /// [`CpuThreadError::CpuNotAllowed`] for a CPU outside the allowed set,
808    /// [`CpuThreadError::UnsupportedPlatform`] off Linux,
809    /// [`CpuThreadError::Affinity`] when the snapshot or a worker's mask fails,
810    /// [`CpuThreadError::WorkerAffinity`] when a worker cannot be pinned to
811    /// exactly its CPU, and [`CpuThreadError::Pool`] when the pool cannot be
812    /// built.
813    ///
814    /// # Examples
815    ///
816    /// ```
817    /// use tensor4all_tensorbackend::{CpuBackend, CpuThreadError};
818    ///
819    /// assert!(matches!(
820    ///     CpuBackend::pinned(&[]),
821    ///     Err(CpuThreadError::EmptyCpuList)
822    /// ));
823    /// # #[cfg(target_os = "linux")]
824    /// # {
825    /// let allowed = CpuBackend::allowed_cpus()?;
826    /// let backend = CpuBackend::pinned(&allowed[..1])?;
827    /// assert_eq!(backend.num_threads(), 1);
828    /// # }
829    /// # Ok::<(), Box<dyn std::error::Error>>(())
830    /// ```
831    pub fn pinned(cpus: &[usize]) -> Result<Self, CpuThreadError> {
832        if cpus.is_empty() {
833            return Err(CpuThreadError::EmptyCpuList);
834        }
835        #[cfg(not(target_os = "linux"))]
836        {
837            let _ = cpus;
838            Err(CpuThreadError::UnsupportedPlatform)
839        }
840        #[cfg(target_os = "linux")]
841        {
842            let allowed = affinity::main_thread_mask()?.cpus();
843            let mut seen = std::collections::HashSet::with_capacity(cpus.len());
844            for &cpu in cpus {
845                if !seen.insert(cpu) {
846                    return Err(CpuThreadError::DuplicateCpu { cpu });
847                }
848                if allowed.binary_search(&cpu).is_err() {
849                    return Err(CpuThreadError::CpuNotAllowed { cpu });
850                }
851            }
852
853            let workers = cpus.to_vec();
854            let (sender, receiver) = std::sync::mpsc::channel();
855            let builder = Pool::builder()
856                .threads(workers.len())
857                .on_start(move |worker| {
858                    let outcome = workers
859                        .get(worker)
860                        .copied()
861                        .ok_or_else(|| {
862                            std::io::Error::other("worker index outside the pinned CPU list")
863                        })
864                        .and_then(affinity::pin_current_thread);
865                    let _ = sender.send((worker, outcome, current_worker_cpus()));
866                });
867            let pool = with_wait_policy(builder, default_cpu_wait_policy())
868                .build()
869                .map_err(|source| CpuThreadError::Pool {
870                    source: Box::new(source),
871                })?;
872
873            // Every worker of a successfully built pool runs the start handler,
874            // which sends exactly one report, and the registry keeps its sender
875            // alive while the pool is alive. The loop therefore receives one
876            // report per worker. A closed channel is not expected, but it is
877            // still a worker affinity-handshake failure rather than a separate
878            // unreachable error case.
879            let mut worker_cpus: WorkerCpus = vec![Vec::new(); cpus.len()];
880            for _ in 0..cpus.len() {
881                let (worker, outcome, own_cpus) =
882                    receiver
883                        .recv()
884                        .map_err(|source| CpuThreadError::WorkerAffinity {
885                            worker: 0,
886                            source: std::io::Error::other(source),
887                        })?;
888                if let Some(slot) = worker_cpus.get_mut(worker) {
889                    *slot = own_cpus;
890                }
891                if let Err(source) = outcome {
892                    return Err(CpuThreadError::WorkerAffinity { worker, source });
893                }
894            }
895            Ok(Self {
896                engine: CpuEngine::new(pool),
897                worker_cpus,
898            })
899        }
900    }
901
902    /// The CPUs the main thread is allowed to run on right now, in ascending
903    /// order.
904    ///
905    /// This is a snapshot, read with `sched_getaffinity(getpid())` on Linux. A
906    /// caller can split it into disjoint lists and pass one list to each
907    /// [`CpuBackend::pinned`] engine. Off Linux this reports
908    /// [`CpuThreadError::UnsupportedPlatform`].
909    ///
910    /// # Errors
911    ///
912    /// Returns [`CpuThreadError::UnsupportedPlatform`] off Linux and
913    /// [`CpuThreadError::Affinity`] when the affinity query fails.
914    ///
915    /// # Examples
916    ///
917    /// ```
918    /// use tensor4all_tensorbackend::CpuBackend;
919    ///
920    /// # #[cfg(target_os = "linux")]
921    /// # {
922    /// let allowed = CpuBackend::allowed_cpus()?;
923    /// assert!(!allowed.is_empty());
924    /// assert!(allowed.windows(2).all(|pair| pair[0] < pair[1]));
925    /// # }
926    /// # Ok::<(), Box<dyn std::error::Error>>(())
927    /// ```
928    pub fn allowed_cpus() -> Result<Vec<usize>, CpuThreadError> {
929        #[cfg(not(target_os = "linux"))]
930        {
931            Err(CpuThreadError::UnsupportedPlatform)
932        }
933        #[cfg(target_os = "linux")]
934        {
935            Ok(affinity::main_thread_mask()?.cpus())
936        }
937    }
938
939    /// The number of workers operations on this backend may use.
940    ///
941    /// # Examples
942    ///
943    /// ```
944    /// use tensor4all_tensorbackend::CpuBackend;
945    ///
946    /// assert_eq!(CpuBackend::with_threads(1)?.num_threads(), 1);
947    /// # Ok::<(), Box<dyn std::error::Error>>(())
948    /// ```
949    pub fn num_threads(&self) -> usize {
950        self.engine.num_threads()
951    }
952
953    /// The CPU set each worker of this backend's pool reported when it started,
954    /// by worker index.
955    ///
956    /// Empty for a serial engine, for a caller-supplied engine
957    /// ([`CpuBackend::from`]) and off Linux. It is a snapshot taken at pool
958    /// build time, so it shows what the workers actually inherited — the main
959    /// thread's allowed set for [`CpuBackend::new`] and
960    /// [`CpuBackend::with_threads`], and its one CPU for
961    /// [`CpuBackend::pinned`].
962    ///
963    /// # Examples
964    ///
965    /// ```
966    /// use tensor4all_tensorbackend::CpuBackend;
967    ///
968    /// let backend = CpuBackend::with_threads(1)?;
969    /// assert!(backend.worker_cpus().is_empty());
970    /// # Ok::<(), Box<dyn std::error::Error>>(())
971    /// ```
972    pub fn worker_cpus(&self) -> Vec<Vec<usize>> {
973        self.worker_cpus.clone()
974    }
975
976    /// Borrow the engine this backend owns.
977    ///
978    /// # Examples
979    ///
980    /// ```
981    /// use tensor4all_tensorbackend::CpuBackend;
982    ///
983    /// let backend = CpuBackend::with_threads(1)?;
984    /// assert_eq!(backend.engine().buffer_pool_len(), 0);
985    /// # Ok::<(), Box<dyn std::error::Error>>(())
986    /// ```
987    pub fn engine(&self) -> &CpuEngine {
988        &self.engine
989    }
990
991    /// Borrow the engine mutably, for one operation or one session.
992    ///
993    /// # Examples
994    ///
995    /// ```
996    /// use tensor4all_tensorbackend::CpuBackend;
997    ///
998    /// let mut backend = CpuBackend::with_threads(1)?;
999    /// backend.engine_mut().reset_buffer_pool();
1000    /// assert_eq!(backend.engine().buffer_pool_len(), 0);
1001    /// # Ok::<(), Box<dyn std::error::Error>>(())
1002    /// ```
1003    pub fn engine_mut(&mut self) -> &mut CpuEngine {
1004        &mut self.engine
1005    }
1006
1007    /// Consume the backend and return the engine it owns.
1008    ///
1009    /// A caller that hands engines to another owner, such as a lane executor,
1010    /// uses this instead of borrowing and cloning: [`CpuBackend`] deliberately
1011    /// does not clone an execution domain.
1012    ///
1013    /// # Examples
1014    ///
1015    /// ```
1016    /// use tensor4all_tensorbackend::CpuBackend;
1017    ///
1018    /// let engine = CpuBackend::with_threads(1)?.into_engine();
1019    /// assert_eq!(engine.num_threads(), 1);
1020    /// # Ok::<(), Box<dyn std::error::Error>>(())
1021    /// ```
1022    pub fn into_engine(self) -> CpuEngine {
1023        self.engine
1024    }
1025}
1026
1027impl Default for CpuBackend {
1028    fn default() -> Self {
1029        Self::new()
1030    }
1031}
1032
1033impl Deref for CpuBackend {
1034    type Target = CpuEngine;
1035
1036    fn deref(&self) -> &Self::Target {
1037        &self.engine
1038    }
1039}
1040
1041impl DerefMut for CpuBackend {
1042    fn deref_mut(&mut self) -> &mut Self::Target {
1043        &mut self.engine
1044    }
1045}
1046
1047impl From<CpuEngine> for CpuBackend {
1048    fn from(engine: CpuEngine) -> Self {
1049        Self {
1050            engine,
1051            worker_cpus: WorkerCpus::default(),
1052        }
1053    }
1054}
1055
1056/// Caller-owned execution domain used by context-aware tensor algorithms.
1057///
1058/// Values are validated against the exact runtime represented by the selected
1059/// context; no implicit host/device transfer is performed by this enum.
1060#[derive(Clone, Debug)]
1061pub enum ExecutionContext {
1062    /// Host execution through one caller-owned CPU context.
1063    Cpu(Arc<CpuExecutionContext>),
1064    /// CUDA execution through one caller-owned CUDA context.
1065    #[cfg(feature = "tenferro-cuda")]
1066    Cuda(Arc<crate::cuda::CudaExecutionContext>),
1067}
1068
1069impl ExecutionContext {
1070    /// Check whether this is the process-global CPU context.
1071    ///
1072    /// Compatibility boundary: legacy CPU-global callers route through the
1073    /// historical host code paths (bitwise-identical numerics) while explicit
1074    /// contexts use the scoped primitives. Without the global-defaults
1075    /// feature there is no global context, so this always reports false.
1076    pub fn is_global_default_cpu(&self) -> bool {
1077        #[cfg(feature = "global-defaults")]
1078        {
1079            match self {
1080                ExecutionContext::Cpu(context) => {
1081                    let own = context.eager_runtime().map(|runtime| runtime.id());
1082                    let global = defaults::default_eager_ctx().map(|runtime| runtime.id());
1083                    matches!((own, global), (Ok(a), Ok(b)) if a == b)
1084                }
1085                #[cfg(feature = "tenferro-cuda")]
1086                ExecutionContext::Cuda(_) => false,
1087            }
1088        }
1089        #[cfg(not(feature = "global-defaults"))]
1090        {
1091            let _ = self;
1092            false
1093        }
1094    }
1095}
1096
1097/// Error returned by explicit CPU context graph or eager-runtime operations.
1098///
1099/// The original tenferro diagnostic is retained as the error source.
1100///
1101/// # Examples
1102///
1103/// ```
1104/// use std::error::Error;
1105/// use std::sync::Arc;
1106/// use tensor4all_tensorbackend::CpuExecutionContextError;
1107///
1108/// let error = CpuExecutionContextError::Initialization {
1109///     component: "graph runtime",
1110///     source: Arc::new(std::io::Error::other("registration failed")),
1111/// };
1112/// assert!(error.source().is_some());
1113/// ```
1114#[derive(Debug, Clone, thiserror::Error)]
1115pub enum CpuExecutionContextError {
1116    /// A context-owned graph or eager runtime could not be initialized.
1117    #[error("failed to initialize {component}: {source}")]
1118    Initialization {
1119        /// Context component being initialized.
1120        component: &'static str,
1121        /// Original tenferro diagnostic.
1122        #[source]
1123        source: Arc<dyn std::error::Error + Send + Sync + 'static>,
1124    },
1125    /// The calling thread is already inside a session of this context.
1126    ///
1127    /// One context owns one engine and lends it to one caller at a time. A
1128    /// session entered from inside another session on the same thread would
1129    /// wait for a lock that thread already holds, so it is reported as this
1130    /// typed error before any lock is taken.
1131    #[error("recursive CPU execution-context session entry on one thread")]
1132    Reentered,
1133    /// Graph compilation or execution failed.
1134    #[error("CPU graph {operation} failed: {source}")]
1135    Graph {
1136        /// Graph operation that failed.
1137        operation: &'static str,
1138        /// Original tenferro diagnostic.
1139        #[source]
1140        source: Arc<dyn std::error::Error + Send + Sync + 'static>,
1141    },
1142}
1143
1144thread_local! {
1145    /// Identities of the execution contexts whose engines this thread holds.
1146    ///
1147    /// Re-entry is per context: holding one context's engine must not block an
1148    /// implicit entry that uses another context's engine.
1149    static ACTIVE_CONTEXTS: RefCell<Vec<u64>> = const { RefCell::new(Vec::new()) };
1150}
1151
1152static NEXT_CONTEXT_ID: AtomicU64 = AtomicU64::new(1);
1153
1154pub(crate) fn next_context_id() -> u64 {
1155    NEXT_CONTEXT_ID.fetch_add(1, Ordering::Relaxed)
1156}
1157
1158pub(crate) struct CanonicalSessionGuard {
1159    context_id: u64,
1160}
1161
1162impl CanonicalSessionGuard {
1163    /// Whether this context's engine is already held on this thread.
1164    pub(crate) fn is_active(context_id: u64) -> bool {
1165        ACTIVE_CONTEXTS.with(|active| active.borrow().contains(&context_id))
1166    }
1167
1168    pub(crate) fn enter(context_id: u64) -> Self {
1169        ACTIVE_CONTEXTS.with(|active| active.borrow_mut().push(context_id));
1170        Self { context_id }
1171    }
1172}
1173
1174impl Drop for CanonicalSessionGuard {
1175    fn drop(&mut self) {
1176        ACTIVE_CONTEXTS.with(|active| {
1177            let mut active = active.borrow_mut();
1178            if active.last() == Some(&self.context_id) {
1179                active.pop();
1180            } else {
1181                active.retain(|context_id| *context_id != self.context_id);
1182            }
1183        });
1184    }
1185}
1186
1187impl CpuExecutionContextError {
1188    fn initialization(
1189        component: &'static str,
1190        source: impl std::error::Error + Send + Sync + 'static,
1191    ) -> Self {
1192        Self::Initialization {
1193            component,
1194            source: Arc::new(source),
1195        }
1196    }
1197
1198    fn graph(
1199        operation: &'static str,
1200        source: impl std::error::Error + Send + Sync + 'static,
1201    ) -> Self {
1202        Self::Graph {
1203            operation,
1204            source: Arc::new(source),
1205        }
1206    }
1207}
1208
1209/// Failure entering the engine of the context that owns an eager runtime.
1210///
1211/// The implicit frontend resolves the engine from the runtime of the operand it
1212/// acts on; the owner may be a CPU or a CUDA execution context, so this error
1213/// keeps the owning context's typed diagnostic instead of flattening both into
1214/// one string.
1215///
1216/// # Examples
1217///
1218/// ```
1219/// use tensor4all_tensorbackend::{
1220///     CpuExecutionContextError, ExecutionContextEntryError,
1221/// };
1222///
1223/// let error = ExecutionContextEntryError::from(CpuExecutionContextError::Reentered);
1224/// assert!(matches!(
1225///     error,
1226///     ExecutionContextEntryError::Cpu(CpuExecutionContextError::Reentered)
1227/// ));
1228/// ```
1229#[cfg(feature = "global-defaults")]
1230#[derive(Debug, Clone, thiserror::Error)]
1231pub enum ExecutionContextEntryError {
1232    /// The owning CPU context rejected the session entry.
1233    #[error(transparent)]
1234    Cpu(#[from] CpuExecutionContextError),
1235    /// The owning CUDA context rejected the session entry.
1236    #[cfg(feature = "tenferro-cuda")]
1237    #[error(transparent)]
1238    Cuda(#[from] crate::cuda::CudaExecutionContextError),
1239}
1240
1241/// The engine a context owns, plus the graph runtime derived from it.
1242///
1243/// The registration the graph runtime holds is derived from the engine, and
1244/// `Runtime::run` needs `&mut engine`, so both live under one lock. The
1245/// [`GraphCompiler`] is not here: it never touches the engine, so it has its own
1246/// lock and a compile never waits for a running graph.
1247struct EngineState {
1248    backend: CpuBackend,
1249    runtime: Option<Runtime>,
1250}
1251
1252impl EngineState {
1253    fn new(backend: CpuBackend) -> Self {
1254        Self {
1255            backend,
1256            runtime: None,
1257        }
1258    }
1259
1260    /// The graph runtime of this context, built from the engine on first use.
1261    fn runtime(&mut self) -> Result<&mut Runtime, CpuExecutionContextError> {
1262        let Self { backend, runtime } = self;
1263        match runtime {
1264            Some(runtime) => Ok(runtime),
1265            None => {
1266                let built = build_graph_runtime(backend)?;
1267                Ok(runtime.insert(built))
1268            }
1269        }
1270    }
1271}
1272
1273/// The runtime engine id `engine` is registered under.
1274///
1275/// An engine built by [`CpuEngine::with_runtime_engine_id`] runs under its own
1276/// id, so every registration and extension module installed on its behalf must
1277/// use that id rather than the canonical [`tenferro_cpu::runtime_engine_id`].
1278fn engine_runtime_id(engine: &CpuEngine) -> tenferro::EngineId {
1279    ExecutionEngine::target(engine).engine_id().clone()
1280}
1281
1282/// The engine state one context owns, behind a handle that can also be reached
1283/// through the eager-runtime owner table.
1284///
1285/// A [`CpuExecutionContext`] holds this in an [`Arc`]; the process-wide owner
1286/// table holds only a [`Weak`](std::sync::Weak), so the entry dangles as soon as the context is
1287/// dropped instead of keeping the engine alive.
1288pub(crate) struct EngineOwner {
1289    id: u64,
1290    /// The engine's registered runtime id, captured once at construction.
1291    ///
1292    /// The eager runtime installs its extension modules under this id, and it is
1293    /// read without taking the engine lock so building the eager runtime can
1294    /// never wait for a running session.
1295    engine_id: tenferro::EngineId,
1296    /// The engine's prepared-plan cache, captured once at construction.
1297    ///
1298    /// A session entered from a Rayon worker runs inline on a fresh serial
1299    /// engine that shares this cache (tensor4all-rs#830). Holding the handle
1300    /// here keeps that entry off the engine lock: the worker never waits for an
1301    /// engine its own pool may be holding.
1302    plans: tenferro_cpu::CpuPlanCache,
1303    engine: Mutex<EngineState>,
1304}
1305
1306impl EngineOwner {
1307    fn new(backend: CpuBackend) -> Self {
1308        Self {
1309            id: next_context_id(),
1310            engine_id: engine_runtime_id(backend.engine()),
1311            plans: backend.engine().plan_cache().clone(),
1312            engine: Mutex::new(EngineState::new(backend)),
1313        }
1314    }
1315
1316    /// Lock the engine state, recovering a lock poisoned by a panic.
1317    ///
1318    /// A panic while the engine was borrowed leaves tenferro's engine marked
1319    /// broken, and the lock poisoned. The engine is kept: it carries the
1320    /// caller's pool, allocation domain, batch policy and runtime engine id, so
1321    /// replacing it with a default engine would silently change the caller's
1322    /// execution policy. Recovering the poison only makes the lock usable again;
1323    /// operations on the broken engine are refused with tenferro's typed
1324    /// `CpuEngineError::Broken`, and the caller builds a new context for a new
1325    /// engine.
1326    fn lock_engine(&self) -> std::sync::MutexGuard<'_, EngineState> {
1327        match self.engine.lock() {
1328            Ok(guard) => guard,
1329            Err(poisoned) => {
1330                let guard = poisoned.into_inner();
1331                self.engine.clear_poison();
1332                guard
1333            }
1334        }
1335    }
1336
1337    /// Check the per-context re-entry guard and install it for one entry.
1338    ///
1339    /// Every fallible entry checks before it locks: a nested entry would wait
1340    /// for an engine lock this thread already holds, so it is reported as
1341    /// [`CpuExecutionContextError::Reentered`] instead.
1342    fn enter_session(&self) -> Result<CanonicalSessionGuard, CpuExecutionContextError> {
1343        if CanonicalSessionGuard::is_active(self.id) {
1344            return Err(CpuExecutionContextError::Reentered);
1345        }
1346        Ok(CanonicalSessionGuard::enter(self.id))
1347    }
1348
1349    /// The single guarded entry that locks this owner's engine.
1350    ///
1351    /// This is the only way to lock the engine from a fallible entry: the
1352    /// re-entry guard is checked and installed here, so no caller can hold the
1353    /// lock without holding the guard too. [`CpuExecutionContext::with_backend`]
1354    /// is the one remaining locking entry, and it installs the same guard
1355    /// without being able to report the rejection.
1356    fn begin_session(&self) -> Result<EngineEntry<'_>, CpuExecutionContextError> {
1357        let guard = self.enter_session()?;
1358        Ok(EngineEntry {
1359            state: self.lock_engine(),
1360            _guard: guard,
1361        })
1362    }
1363
1364    /// Run `f` in one session of this owner's engine.
1365    ///
1366    /// This is the single session-entry implementation shared by the owning
1367    /// context and the eager-runtime route: the same per-context re-entry guard
1368    /// and the same inline-serial rule for Rayon workers apply either way.
1369    fn with_session<R>(
1370        &self,
1371        f: impl FnOnce(&mut CpuEngine) -> R,
1372    ) -> Result<R, CpuExecutionContextError> {
1373        // A worker runs inline on its own serial engine instead of waiting for a
1374        // pool it belongs to (tensor4all-rs#830). The fresh engine shares the
1375        // plan cache, whose entries are immutable prepared plans, so a stage
1376        // entered per item does not re-plan. It does **not** share the context
1377        // engine's buffer pool, extension caches, batch policy or allocation
1378        // domain: an entry from a worker therefore allocates and batches
1379        // independently, and only while it runs.
1380        if rayon::current_thread_index().is_some() {
1381            let _guard = self.enter_session()?;
1382            let mut engine = CpuEngine::serial().with_plan_cache(self.plans.clone());
1383            return Ok(f(&mut engine));
1384        }
1385        let mut entry = self.begin_session()?;
1386        Ok(f(entry.backend.engine_mut()))
1387    }
1388}
1389
1390/// One held engine session: the re-entry guard and the locked engine state.
1391///
1392/// The guard lives exactly as long as the lock it makes safe, so dropping the
1393/// entry releases the engine before the thread is allowed to enter again.
1394struct EngineEntry<'a> {
1395    state: std::sync::MutexGuard<'a, EngineState>,
1396    _guard: CanonicalSessionGuard,
1397}
1398
1399impl Deref for EngineEntry<'_> {
1400    type Target = EngineState;
1401
1402    fn deref(&self) -> &EngineState {
1403        &self.state
1404    }
1405}
1406
1407impl DerefMut for EngineEntry<'_> {
1408    fn deref_mut(&mut self) -> &mut EngineState {
1409        &mut self.state
1410    }
1411}
1412
1413/// Caller-owned CPU execution domain for plain, graph, and eager-AD work.
1414///
1415/// The context owns one [`CpuBackend`] and lends its engine to one caller at a
1416/// time. A session entered from a Rayon worker runs inline on a fresh serial
1417/// engine that shares the context engine's prepared-plan cache instead of
1418/// waiting for the context engine: a worker that waits for a pool install is
1419/// handed more of the enclosing pool's work, and that work may enter a session
1420/// itself (tensor4all-rs#830). Graph preparation caches, the engine's buffer
1421/// pool and the eager runtime are owned by this context and are released when
1422/// it is dropped.
1423///
1424/// # Examples
1425///
1426/// ```
1427/// use tensor4all_tensorbackend::CpuExecutionContext;
1428/// use tensor4all_tensorbackend::CpuBackend;
1429///
1430/// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1431/// let threads = context.with_backend(|backend| backend.num_threads());
1432/// assert_eq!(threads, 1);
1433/// # Ok::<(), Box<dyn std::error::Error>>(())
1434/// ```
1435pub struct CpuExecutionContext {
1436    owner: Arc<EngineOwner>,
1437    compiler: Mutex<GraphCompiler>,
1438    eager: OnceLock<Result<Arc<EagerRuntime>, CpuExecutionContextError>>,
1439}
1440
1441impl std::fmt::Debug for CpuExecutionContext {
1442    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1443        f.debug_struct("CpuExecutionContext")
1444            .field("graph_initialized", &self.engine_initialized())
1445            .field("eager_initialized", &self.eager.get().is_some())
1446            .finish_non_exhaustive()
1447    }
1448}
1449
1450impl CpuExecutionContext {
1451    /// Create an execution context from a caller-selected CPU backend.
1452    ///
1453    /// Runtime construction is lazy, so creating a context cannot fail and does
1454    /// not allocate another executor or consult environment configuration.
1455    pub fn from_backend(backend: CpuBackend) -> Self {
1456        Self {
1457            owner: Arc::new(EngineOwner::new(backend)),
1458            compiler: Mutex::new(GraphCompiler::new()),
1459            eager: OnceLock::new(),
1460        }
1461    }
1462
1463    fn engine_initialized(&self) -> bool {
1464        self.owner
1465            .engine
1466            .try_lock()
1467            .is_ok_and(|state| state.runtime.is_some())
1468    }
1469
1470    /// Lock the compiler, discarding one a panic left behind.
1471    ///
1472    /// A `GraphCompiler` holds planning state that a panic may have left half
1473    /// written; rebuilding it costs nothing a compile would not pay for the next
1474    /// graph anyway. The poison is cleared after the rebuild so later compiles
1475    /// reuse the fresh compiler instead of rebuilding it on every access.
1476    fn lock_compiler(&self) -> std::sync::MutexGuard<'_, GraphCompiler> {
1477        match self.compiler.lock() {
1478            Ok(guard) => guard,
1479            Err(poisoned) => {
1480                let mut guard = poisoned.into_inner();
1481                *guard = GraphCompiler::new();
1482                self.compiler.clear_poison();
1483                guard
1484            }
1485        }
1486    }
1487
1488    /// Run a plain tensor operation with this context's engine.
1489    ///
1490    /// The closure runs while the context-local engine lock is held. A lock
1491    /// poisoned by a panic is recovered, and the engine is kept: it carries the
1492    /// caller's pool, allocation domain, batch policy and runtime engine id, so
1493    /// a broken engine keeps refusing operations with tenferro's typed
1494    /// `CpuEngineError::Broken` instead of being silently replaced by a default
1495    /// one. The caller builds a new context when it needs a new engine. The
1496    /// session entries recover the same way.
1497    ///
1498    /// This borrows the engine for the whole closure, so a session entry from
1499    /// inside `f` on this thread is rejected with
1500    /// [`CpuExecutionContextError::Reentered`] instead of waiting for the engine.
1501    /// Calling `with_backend` from inside a session of this context is the
1502    /// caller's obligation not to do: this entry cannot report an error, so it
1503    /// would wait for the engine the session already holds.
1504    pub fn with_backend<R>(&self, f: impl FnOnce(&mut CpuBackend) -> R) -> R {
1505        // The guard makes a nested session entry of this context a typed error,
1506        // not a wait; another context's engine stays independently enterable.
1507        let _guard = CanonicalSessionGuard::enter(self.owner.id);
1508        f(&mut self.owner.lock_engine().backend)
1509    }
1510
1511    /// Run `f` in one session of this context's engine.
1512    ///
1513    /// This is the engine-borrow entry for callers that need the concrete
1514    /// engine (for example to hand it to an eager AD session). The context lends
1515    /// its engine to one caller at a time for the whole of `f`.
1516    ///
1517    /// # Errors
1518    ///
1519    /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1520    /// already inside a session of this context.
1521    ///
1522    /// # Examples
1523    ///
1524    /// ```
1525    /// use tensor4all_tensorbackend::{CpuBackend, CpuExecutionContext};
1526    ///
1527    /// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1528    /// let threads = context.with_session(|engine| engine.num_threads())?;
1529    /// assert_eq!(threads, 1);
1530    /// # Ok::<(), Box<dyn std::error::Error>>(())
1531    /// ```
1532    pub fn with_session<R>(
1533        &self,
1534        f: impl FnOnce(&mut CpuEngine) -> R,
1535    ) -> Result<R, CpuExecutionContextError> {
1536        self.owner.with_session(f)
1537    }
1538
1539    /// Compile a backend-neutral traced graph using this context's compiler cache.
1540    ///
1541    /// # Errors
1542    ///
1543    /// Returns [`CpuExecutionContextError::Graph`] when graph compilation fails.
1544    pub fn compile_graph(
1545        &self,
1546        graph: &TracedGraph,
1547    ) -> Result<CompiledGraph, CpuExecutionContextError> {
1548        self.lock_compiler()
1549            .compile_traced_graph(graph)
1550            .map_err(|source| CpuExecutionContextError::graph("compilation", source))
1551    }
1552
1553    /// Execute a compiled graph in this context's runtime and prepared-plan cache.
1554    ///
1555    /// `CompiledGraph` is backend-neutral. Backend-prepared executables and
1556    /// workspaces never leave this context-owned runtime.
1557    ///
1558    /// # Errors
1559    ///
1560    /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1561    /// already inside a session of this context, and
1562    /// [`CpuExecutionContextError::Initialization`], [`CpuExecutionContextError::Graph`]
1563    /// when runtime initialization, preparation, or execution fails.
1564    pub fn run_graph(
1565        &self,
1566        graph: &CompiledGraph,
1567        inputs: &[&Tensor],
1568    ) -> Result<Vec<Tensor>, CpuExecutionContextError> {
1569        let reads = inputs
1570            .iter()
1571            .map(|input| TensorRead::from_tensor(input))
1572            .collect::<Vec<_>>();
1573        self.run_graph_reads(graph, &reads)
1574    }
1575
1576    /// Execute a compiled graph on borrowed tensor views.
1577    ///
1578    /// The read form is what the bridge uses: an operand that is already a
1579    /// borrowed view stays borrowed instead of being copied into an owner first.
1580    ///
1581    /// # Errors
1582    ///
1583    /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1584    /// already inside a session of this context, and
1585    /// [`CpuExecutionContextError::Initialization`], [`CpuExecutionContextError::Graph`]
1586    /// when runtime initialization, preparation, or execution fails.
1587    pub(crate) fn run_graph_reads(
1588        &self,
1589        graph: &CompiledGraph,
1590        inputs: &[TensorRead<'_>],
1591    ) -> Result<Vec<Tensor>, CpuExecutionContextError> {
1592        let mut state = self.owner.begin_session()?;
1593        let EngineState { backend, runtime } = &mut *state;
1594        let runtime = match runtime {
1595            Some(runtime) => &*runtime,
1596            None => &*runtime.insert(build_graph_runtime(backend)?),
1597        };
1598        let target = backend.engine().target();
1599        let prepared = runtime
1600            .prepare(graph, inputs, std::slice::from_ref(&target))
1601            .map_err(|source| CpuExecutionContextError::graph("preparation", source))?;
1602        let outputs = {
1603            let mut engines: [&mut dyn ExecutionEngine; 1] = [backend.engine_mut()];
1604            runtime
1605                .run(&mut engines, &prepared, inputs)
1606                .map_err(|source| CpuExecutionContextError::graph("execution", source))?
1607        };
1608        outputs
1609            .into_iter()
1610            .map(|output| {
1611                output
1612                    .into_tensor(backend.engine_mut())
1613                    .map_err(|source| CpuExecutionContextError::graph("output", source))
1614            })
1615            .collect()
1616    }
1617
1618    /// Return this context's eager reverse-AD runtime.
1619    ///
1620    /// The runtime is engine-independent: it is built once from an
1621    /// [`AdContext`] with the linalg AD rules and the einsum/linalg extension
1622    /// modules installed under this context engine's runtime id, and every call
1623    /// takes the caller's engine ([`EagerRuntime::with_eager_session`]). Repeated
1624    /// calls return the same runtime and therefore the same eager compilation
1625    /// cache. The runtime registers itself with the eager-runtime owner table once,
1626    /// when it is built.
1627    ///
1628    /// # Errors
1629    ///
1630    /// Returns [`CpuExecutionContextError`] when linalg AD-rule or eager-runtime
1631    /// registration fails.
1632    pub fn eager_runtime(&self) -> Result<Arc<EagerRuntime>, CpuExecutionContextError> {
1633        self.eager
1634            .get_or_init(|| {
1635                let runtime = build_eager_runtime(&self.owner.engine_id)?;
1636                #[cfg(feature = "global-defaults")]
1637                register_eager_runtime_owner(
1638                    &runtime,
1639                    RuntimeOwner::Cpu(Arc::downgrade(&self.owner)),
1640                );
1641                Ok(runtime)
1642            })
1643            .as_ref()
1644            .map(Arc::clone)
1645            .map_err(Clone::clone)
1646    }
1647
1648    /// Return statistics for this context's runtime-owned graph caches.
1649    ///
1650    /// # Errors
1651    ///
1652    /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1653    /// already inside a session of this context, and
1654    /// [`CpuExecutionContextError`] when graph initialization or the cache
1655    /// statistics query fails.
1656    pub fn graph_cache_stats(
1657        &self,
1658    ) -> Result<tenferro::RuntimeCacheStats, CpuExecutionContextError> {
1659        let mut state = self.owner.begin_session()?;
1660        let runtime = state.runtime()?;
1661        runtime
1662            .cache_stats()
1663            .map_err(|source| CpuExecutionContextError::graph("cache statistics", source))
1664    }
1665
1666    /// Return retained-buffer statistics for this context's engine.
1667    ///
1668    /// `buffers` counts the retained allocations and `capacity_bytes` their
1669    /// total element capacity. A context whose pool has just been released by
1670    /// [`Self::reset_graph_buffer_pool`] reports zero.
1671    ///
1672    /// # Errors
1673    ///
1674    /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1675    /// already inside a session of this context.
1676    ///
1677    /// # Examples
1678    ///
1679    /// ```
1680    /// use tensor4all_tensorbackend::CpuExecutionContext;
1681    /// use tenferro::program::{CoreSemanticOp, ProgramInputSpec};
1682    /// use tenferro::{DType, Tensor, TraceContext};
1683    /// use tensor4all_tensorbackend::CpuBackend;
1684    ///
1685    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1686    /// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1687    ///
1688    /// let trace = TraceContext::new();
1689    /// let input = trace.input(ProgramInputSpec::new(DType::F64, [2_usize.into()]))?;
1690    /// let output = trace.add_op(CoreSemanticOp::Neg, &[input])?[0];
1691    /// let graph = trace.finish(&[output])?;
1692    /// let compiled = context.compile_graph(&graph)?;
1693    /// let values = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, -2.0])?;
1694    /// let output = context.run_graph(&compiled, &[&values])?;
1695    /// assert_eq!(output[0].as_slice::<f64>()?, &[-1.0, 2.0]);
1696    ///
1697    /// // Releasing the pool empties it.
1698    /// context.reset_graph_buffer_pool()?;
1699    /// let after = context.graph_buffer_pool_stats()?;
1700    /// assert_eq!(after.buffers, 0);
1701    /// assert_eq!(after.capacity_bytes, 0);
1702    /// # Ok(())
1703    /// # }
1704    /// ```
1705    pub fn graph_buffer_pool_stats(&self) -> Result<BufferPoolStats, CpuExecutionContextError> {
1706        Ok(self
1707            .owner
1708            .begin_session()?
1709            .backend
1710            .engine()
1711            .buffer_pool_stats())
1712    }
1713
1714    /// Release retained buffers owned by this context's engine.
1715    ///
1716    /// The graph runtime, its compiled plans and its prepared-plan cache are
1717    /// kept, so callers can keep running previously compiled graphs afterwards.
1718    ///
1719    /// # Errors
1720    ///
1721    /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1722    /// already inside a session of this context.
1723    ///
1724    /// # Examples
1725    ///
1726    /// ```
1727    /// use tensor4all_tensorbackend::CpuExecutionContext;
1728    /// use tensor4all_tensorbackend::CpuBackend;
1729    ///
1730    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1731    /// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1732    /// context.reset_graph_buffer_pool()?;
1733    ///
1734    /// // The pool retains nothing afterwards, and the runtime stays usable.
1735    /// let stats = context.graph_buffer_pool_stats()?;
1736    /// assert_eq!(stats.buffers, 0);
1737    /// assert_eq!(stats.capacity_bytes, 0);
1738    /// # Ok(())
1739    /// # }
1740    /// ```
1741    pub fn reset_graph_buffer_pool(&self) -> Result<(), CpuExecutionContextError> {
1742        self.owner
1743            .begin_session()?
1744            .backend
1745            .engine_mut()
1746            .reset_buffer_pool();
1747        Ok(())
1748    }
1749
1750    /// Recreate this context's graph runtime and release its prepared caches.
1751    ///
1752    /// # Errors
1753    ///
1754    /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1755    /// already inside a session of this context.
1756    pub fn reset_graph_runtime(&self) -> Result<(), CpuExecutionContextError> {
1757        {
1758            let mut state = self.owner.begin_session()?;
1759            state.runtime = None;
1760            state.backend.engine_mut().reset_buffer_pool();
1761        }
1762        *self.lock_compiler() = GraphCompiler::new();
1763        Ok(())
1764    }
1765}
1766
1767fn build_graph_runtime(backend: &CpuBackend) -> Result<Runtime, CpuExecutionContextError> {
1768    // Register and extend the runtime under the engine's *own* runtime id: an
1769    // engine built with `CpuEngine::with_runtime_engine_id` would not be found
1770    // by a preparation targeting the canonical CPU id.
1771    let engine_id = engine_runtime_id(backend.engine());
1772    let mut builder = Runtime::builder();
1773    builder
1774        .register_engine(
1775            tenferro_cpu::runtime_engine_registration_with_id(backend.engine(), engine_id.clone())
1776                .map_err(|source| {
1777                    CpuExecutionContextError::initialization("graph CPU engine", source)
1778                })?,
1779        )
1780        .map_err(|source| CpuExecutionContextError::initialization("graph CPU engine", source))?;
1781    builder
1782        .install_extension_module(
1783            tenferro_einsum::extension_module::<CpuEngine>(engine_id).map_err(|source| {
1784                CpuExecutionContextError::initialization("einsum extension", source)
1785            })?,
1786        )
1787        .map_err(|source| CpuExecutionContextError::initialization("einsum extension", source))?;
1788    builder
1789        .build()
1790        .map_err(|source| CpuExecutionContextError::initialization("graph runtime", source))
1791}
1792
1793/// Build an AD context with the built-in linalg semantic rules installed.
1794///
1795/// The context is backend-neutral: CPU and CUDA eager runtimes share the same
1796/// rule set and differ only in the execution extension modules installed on the
1797/// runtime.
1798pub(crate) fn build_ad_context(
1799) -> Result<AdContext, tenferro_ad::semantic_extension::SemanticExtensionRegistryError> {
1800    let rules = tenferro_linalg::semantic_ad_rules()?;
1801    let builder = AdContext::builder().with_semantic_extension_rules(rules)?;
1802    Ok(match builder.build() {
1803        Ok(context) => context,
1804        Err(never) => match never {},
1805    })
1806}
1807
1808fn build_eager_runtime(
1809    engine_id: &tenferro::EngineId,
1810) -> Result<Arc<EagerRuntime>, CpuExecutionContextError> {
1811    let ad_context = build_ad_context()
1812        .map_err(|source| CpuExecutionContextError::initialization("linalg AD rules", source))?;
1813    let runtime = EagerRuntime::with_ad_context(&ad_context)
1814        .map_err(|source| CpuExecutionContextError::initialization("eager runtime", source))?;
1815    // [AI Supplied] Install the built-in extension modules before publishing
1816    // the shared context. Lazy first use reconfigures the runtime and advances
1817    // its epoch; doing that after tensors have prepared AD derivatives can
1818    // invalidate those prepared programs under parallel first use.
1819    //
1820    // The modules are installed under the *engine's own* runtime id: an engine
1821    // built with `CpuEngine::with_runtime_engine_id` would otherwise not find
1822    // its einsum/linalg extensions.
1823    let einsum_module = tenferro_einsum::extension_module::<CpuEngine>(engine_id.clone())
1824        .map_err(|source| CpuExecutionContextError::initialization("einsum extension", source))?;
1825    runtime
1826        .install_extension_module(einsum_module)
1827        .map_err(|source| CpuExecutionContextError::initialization("einsum runtime", source))?;
1828    let linalg_module = tenferro_linalg::extension_module::<CpuEngine>(engine_id.clone())
1829        .map_err(|source| CpuExecutionContextError::initialization("linalg extension", source))?;
1830    runtime
1831        .install_extension_module(linalg_module)
1832        .map_err(|source| CpuExecutionContextError::initialization("linalg runtime", source))?;
1833    Ok(runtime)
1834}
1835
1836/// A context that owns an eager runtime, held weakly by the process-wide table.
1837#[cfg(feature = "global-defaults")]
1838pub(crate) enum RuntimeOwner {
1839    Cpu(Weak<EngineOwner>),
1840    #[cfg(feature = "tenferro-cuda")]
1841    Cuda(Weak<crate::cuda::CudaEngineOwner>),
1842}
1843
1844#[cfg(feature = "global-defaults")]
1845impl RuntimeOwner {
1846    fn upgrade(&self) -> Option<LiveRuntimeOwner> {
1847        match self {
1848            Self::Cpu(owner) => owner.upgrade().map(LiveRuntimeOwner::Cpu),
1849            #[cfg(feature = "tenferro-cuda")]
1850            Self::Cuda(owner) => owner.upgrade().map(LiveRuntimeOwner::Cuda),
1851        }
1852    }
1853
1854    fn is_live(&self) -> bool {
1855        match self {
1856            Self::Cpu(owner) => owner.strong_count() > 0,
1857            #[cfg(feature = "tenferro-cuda")]
1858            Self::Cuda(owner) => owner.strong_count() > 0,
1859        }
1860    }
1861}
1862
1863/// An upgraded [`RuntimeOwner`] that keeps its context alive for one session.
1864#[cfg(feature = "global-defaults")]
1865enum LiveRuntimeOwner {
1866    Cpu(Arc<EngineOwner>),
1867    #[cfg(feature = "tenferro-cuda")]
1868    Cuda(Arc<crate::cuda::CudaEngineOwner>),
1869}
1870
1871/// Process-wide map from an eager runtime's id to the context engine that owns it.
1872///
1873/// The entry holds a [`Weak`] handle: a context registers its runtime when it
1874/// builds it, and the entry dangles when the context (the only strong owner) is
1875/// dropped. Lookups prune the dead entry and fall back to the process-global
1876/// context, so no background cleanup is needed.
1877#[cfg(feature = "global-defaults")]
1878static EAGER_RUNTIME_OWNERS: OnceLock<Mutex<HashMap<ContextId, RuntimeOwner>>> = OnceLock::new();
1879
1880#[cfg(feature = "global-defaults")]
1881fn eager_runtime_owners() -> std::sync::MutexGuard<'static, HashMap<ContextId, RuntimeOwner>> {
1882    EAGER_RUNTIME_OWNERS
1883        .get_or_init(|| Mutex::new(HashMap::new()))
1884        .lock()
1885        .unwrap_or_else(|poisoned| poisoned.into_inner())
1886}
1887
1888#[cfg(feature = "global-defaults")]
1889pub(crate) fn register_eager_runtime_owner(runtime: &EagerRuntime, owner: RuntimeOwner) {
1890    let mut owners = eager_runtime_owners();
1891    owners.retain(|_, owner| owner.is_live());
1892    owners.insert(runtime.id(), owner);
1893}
1894
1895/// Run `f` on the engine of the context that owns `runtime`'s eager runtime.
1896///
1897/// This is the transitional implicit route: an implicit (context-free) eager
1898/// operation has no context argument, so it resolves the engine from the
1899/// runtime of the tensor it acts on instead of always taking the
1900/// process-global default context's engine. That keeps a caller-owned context's
1901/// thread policy, device domain, and engine serialization intact for
1902/// context-free readouts and intermediate operations, including on a context
1903/// that is not the global one.
1904///
1905/// The closure receives the owning engine as [`ExecutionEngine`]: the owner may
1906/// be a CPU context ([`CpuEngine`]) or, with the `tenferro-cuda` feature, a CUDA
1907/// context (`tenferro_gpu::cuda::CudaEngine`). The closure runs through the
1908/// owning context's normal session entry, so a nested entry on the same thread
1909/// is reported as [`ExecutionContextEntryError`]'s typed CPU or CUDA re-entry
1910/// variant instead of waiting for the engine. An entry from a Rayon worker runs
1911/// inline on a serial CPU engine that shares the context's prepared-plan cache;
1912/// a CUDA context keeps its single engine and never opens a second one.
1913/// When no live context owns `runtime` (for example after the context was
1914/// dropped), the entry falls back to the process-global CPU context; the global
1915/// context's own runtime resolves to itself.
1916///
1917/// # Errors
1918///
1919/// Returns [`ExecutionContextEntryError`] when the owning context's session
1920/// entry is rejected.
1921///
1922/// # Examples
1923///
1924/// ```
1925/// use tensor4all_tensorbackend::{with_runtime_engine, CpuBackend, CpuExecutionContext};
1926/// use tenferro_cpu::CpuEngine;
1927///
1928/// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1929/// let runtime = context.eager_runtime()?;
1930/// let threads = with_runtime_engine(&runtime, |engine| {
1931///     engine
1932///         .as_any_mut()
1933///         .downcast_mut::<CpuEngine>()
1934///         .map(|engine| engine.num_threads())
1935/// })?;
1936/// assert_eq!(threads, Some(1));
1937/// # Ok::<(), Box<dyn std::error::Error>>(())
1938/// ```
1939#[cfg(feature = "global-defaults")]
1940pub fn with_runtime_engine<R>(
1941    runtime: &EagerRuntime,
1942    f: impl FnOnce(&mut dyn ExecutionEngine) -> R,
1943) -> Result<R, ExecutionContextEntryError> {
1944    let owner = {
1945        let mut owners = eager_runtime_owners();
1946        match owners.get(&runtime.id()).and_then(RuntimeOwner::upgrade) {
1947            Some(owner) => owner,
1948            None => {
1949                owners.remove(&runtime.id());
1950                LiveRuntimeOwner::Cpu(Arc::clone(&defaults::default_context().owner))
1951            }
1952        }
1953    };
1954    match owner {
1955        LiveRuntimeOwner::Cpu(owner) => owner
1956            .with_session(|engine| f(engine))
1957            .map_err(ExecutionContextEntryError::Cpu),
1958        #[cfg(feature = "tenferro-cuda")]
1959        LiveRuntimeOwner::Cuda(owner) => owner
1960            .with_execution_engine(f)
1961            .map_err(ExecutionContextEntryError::Cuda),
1962    }
1963}
1964
1965#[cfg(feature = "global-defaults")]
1966mod defaults {
1967    use super::*;
1968
1969    static DEFAULT_CONTEXT: OnceLock<Arc<CpuExecutionContext>> = OnceLock::new();
1970
1971    #[cfg(test)]
1972    thread_local! {
1973        static FORCE_EAGER_CONTEXT_FAILURE: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
1974    }
1975
1976    /// Borrow the process-global CPU execution context.
1977    ///
1978    /// New code must take a caller-owned context; this accessor exists for the
1979    /// process-global convenience path and for callers that map the session-entry
1980    /// rejection into their own error type.
1981    pub(crate) fn default_context() -> &'static Arc<CpuExecutionContext> {
1982        DEFAULT_CONTEXT
1983            .get_or_init(|| Arc::new(CpuExecutionContext::from_backend(CpuBackend::new())))
1984    }
1985
1986    /// Error returned when the process-global eager AD runtime cannot be initialized.
1987    ///
1988    /// # Examples
1989    ///
1990    /// ```
1991    /// use std::error::Error;
1992    /// use std::sync::Arc;
1993    /// use tensor4all_tensorbackend::EagerContextError;
1994    ///
1995    /// let error = EagerContextError::Registration {
1996    ///     source: Arc::new(std::io::Error::other("registration failed")),
1997    /// };
1998    /// assert!(error.source().is_some());
1999    /// ```
2000    #[derive(Debug, Clone, thiserror::Error)]
2001    pub enum EagerContextError {
2002        /// The tenferro linalg AD extension rule could not be registered.
2003        #[error("failed to register tenferro linalg AD rule: {source}")]
2004        Registration {
2005            /// Original diagnostic returned by tenferro.
2006            #[source]
2007            source: Arc<dyn std::error::Error + Send + Sync + 'static>,
2008        },
2009    }
2010
2011    /// Run a closure against the optional process-global CPU backend.
2012    pub fn with_default_backend<R>(f: impl FnOnce(&mut CpuBackend) -> R) -> R {
2013        default_context().with_backend(f)
2014    }
2015
2016    /// Run `f` in one session of the process-global context's engine.
2017    ///
2018    /// The entry rejection stays inside tenferro's error type: it is reported as
2019    /// a `tenferro_tensor::Error` classified as runtime state, so callers keep
2020    /// one error type for session work.
2021    ///
2022    /// # Errors
2023    ///
2024    /// Returns the tenferro diagnostic when the entry is rejected or when `f`
2025    /// fails.
2026    pub(crate) fn with_default_session<T: Send>(
2027        f: impl FnOnce(&mut CpuEngine) -> Result<T, tenferro_tensor::Error> + Send,
2028    ) -> Result<T, tenferro_tensor::Error> {
2029        default_context()
2030            .with_session(f)
2031            .map_err(|source| {
2032                tenferro_tensor::Error::runtime_state_source("canonical session entry", source)
2033            })
2034            .and_then(std::convert::identity)
2035    }
2036
2037    /// Run `f` against the process-global graph compiler.
2038    ///
2039    /// The compiler lock is released before the caller runs the compiled graph,
2040    /// so a compile never waits for a context engine that is busy running one.
2041    pub(crate) fn with_default_graph_compiler<R>(f: impl FnOnce(&mut GraphCompiler) -> R) -> R {
2042        f(&mut default_context().lock_compiler())
2043    }
2044
2045    pub(crate) fn default_engine_buffer_pool_stats() -> anyhow::Result<BufferPoolStats> {
2046        default_context()
2047            .graph_buffer_pool_stats()
2048            .map_err(anyhow::Error::new)
2049    }
2050
2051    pub(crate) fn reset_default_engine_buffer_pool() -> anyhow::Result<()> {
2052        default_context()
2053            .reset_graph_buffer_pool()
2054            .map_err(anyhow::Error::new)
2055    }
2056
2057    pub(crate) fn reset_default_engine() -> anyhow::Result<()> {
2058        default_context()
2059            .reset_graph_runtime()
2060            .map_err(anyhow::Error::new)
2061    }
2062
2063    /// Return the optional process-global eager context used by convenience APIs.
2064    ///
2065    /// # Errors
2066    ///
2067    /// Returns [`EagerContextError::Registration`] when eager runtime
2068    /// initialization fails.
2069    ///
2070    /// # Examples
2071    ///
2072    /// ```
2073    /// use std::sync::Arc;
2074    /// use tensor4all_tensorbackend::default_eager_ctx;
2075    ///
2076    /// let first = default_eager_ctx().unwrap();
2077    /// let second = default_eager_ctx().unwrap();
2078    /// assert!(Arc::ptr_eq(&first, &second));
2079    /// ```
2080    pub fn default_eager_ctx() -> Result<Arc<EagerRuntime>, EagerContextError> {
2081        #[cfg(test)]
2082        if FORCE_EAGER_CONTEXT_FAILURE.with(std::cell::Cell::get) {
2083            return Err(EagerContextError::Registration {
2084                source: Arc::new(std::io::Error::other(
2085                    "forced default eager context registration failure",
2086                )),
2087            });
2088        }
2089        default_context()
2090            .eager_runtime()
2091            .map_err(|source| EagerContextError::Registration {
2092                source: Arc::new(source),
2093            })
2094    }
2095
2096    /// Borrow the process-global CPU execution context.
2097    ///
2098    /// Compatibility entry for CPU-global convenience APIs (e.g. the legacy
2099    /// context-free SRC entry): host tensors constructed through the global
2100    /// default belong to this exact context, so they validate against it.
2101    /// New code must take a caller-owned context instead of consulting this.
2102    ///
2103    /// # Examples
2104    ///
2105    /// ```
2106    /// use tensor4all_tensorbackend::{default_cpu_execution_context, ExecutionContext};
2107    ///
2108    /// let context = ExecutionContext::Cpu(default_cpu_execution_context());
2109    /// assert!(matches!(context, ExecutionContext::Cpu(_)));
2110    /// ```
2111    pub fn default_cpu_execution_context() -> Arc<CpuExecutionContext> {
2112        Arc::clone(default_context())
2113    }
2114
2115    #[cfg(test)]
2116    pub(crate) fn with_forced_eager_context_failure<T>(f: impl FnOnce() -> T) -> T {
2117        let previous = FORCE_EAGER_CONTEXT_FAILURE.with(|failure| failure.replace(true));
2118        let result = f();
2119        FORCE_EAGER_CONTEXT_FAILURE.with(|failure| failure.set(previous));
2120        result
2121    }
2122}
2123
2124#[cfg(feature = "global-defaults")]
2125pub(crate) use defaults::default_context;
2126#[cfg(all(test, feature = "global-defaults"))]
2127pub(crate) use defaults::with_forced_eager_context_failure;
2128#[cfg(feature = "global-defaults")]
2129pub use defaults::{
2130    default_cpu_execution_context, default_eager_ctx, with_default_backend, EagerContextError,
2131};
2132#[cfg(feature = "global-defaults")]
2133pub(crate) use defaults::{
2134    default_engine_buffer_pool_stats, reset_default_engine, reset_default_engine_buffer_pool,
2135    with_default_graph_compiler, with_default_session,
2136};
2137
2138#[cfg(test)]
2139mod tests;