tensor4all_tensorbackend/context.rs
1//! Explicit and optional process-global tenferro CPU execution contexts.
2
3use std::cell::RefCell;
4#[cfg(feature = "global-defaults")]
5use std::collections::HashMap;
6use std::num::NonZeroUsize;
7use std::ops::{Deref, DerefMut};
8use std::sync::atomic::{AtomicU64, Ordering};
9#[cfg(feature = "global-defaults")]
10use std::sync::Weak;
11use std::sync::{Arc, Mutex, OnceLock};
12
13use tenferro::{
14 CompiledGraph, ExecutionEngine, GraphCompiler, Runtime, Tensor, TensorRead, TracedGraph,
15};
16#[cfg(feature = "global-defaults")]
17use tenferro_ad::ContextId;
18use tenferro_ad::{AdContext, EagerRuntime};
19use tenferro_cpu::{BufferPoolStats, CpuEngine, Pool, PoolBuilder};
20
21/// Linux CPU-affinity FFI, kept in one place so every `unsafe` block has one
22/// auditable owner.
23///
24/// The mask is a growable word buffer in the kernel's `sched_*affinity` layout.
25/// No fixed `cpu_set_t` is used: a mask larger than glibc's 1024-CPU set is
26/// retried with a larger buffer instead of failing, and CPU ids are validated
27/// before they index the buffer.
28#[cfg(target_os = "linux")]
29mod affinity {
30 use std::io;
31
32 use super::CpuThreadError;
33
34 const BITS_PER_WORD: usize = libc::c_ulong::BITS as usize;
35 const INITIAL_WORDS: usize = 1;
36 /// 512 KiB of mask on a 64-bit host, far beyond any current CPU count.
37 const MAX_WORDS: usize = 1 << 16;
38
39 /// One CPU set in the layout `sched_getaffinity` and `sched_setaffinity`
40 /// expect. The words are owned by the `Vec`, so the buffer is word-aligned.
41 #[derive(Clone, Debug)]
42 pub(super) struct CpuMask {
43 words: Vec<libc::c_ulong>,
44 }
45
46 impl CpuMask {
47 /// The mask holding exactly `cpu`, with at least `min_words` words.
48 ///
49 /// Callers validate `cpu` against the allowed set first, so the small
50 /// allocation is bounded by the current kernel mask. `min_words` keeps
51 /// the buffer at the kernel's mask size on hosts where the highest
52 /// allowed CPU is in a low word.
53 pub(super) fn single(cpu: usize, min_words: usize) -> Self {
54 let word = cpu / BITS_PER_WORD;
55 let bit = cpu % BITS_PER_WORD;
56 let mut words = vec![0; (word + 1).max(min_words)];
57 words[word] |= 1 << bit;
58 Self { words }
59 }
60
61 /// The number of words in this mask's kernel buffer.
62 pub(super) fn words_len(&self) -> usize {
63 self.words.len()
64 }
65
66 /// The CPU ids in this mask, in ascending order.
67 pub(super) fn cpus(&self) -> Vec<usize> {
68 let mut cpus = Vec::new();
69 for (index, &word) in self.words.iter().enumerate() {
70 let mut bits = word;
71 while bits != 0 {
72 let bit = bits.trailing_zeros() as usize;
73 cpus.push(index * BITS_PER_WORD + bit);
74 bits &= bits - 1;
75 }
76 }
77 cpus
78 }
79
80 /// Whether this mask holds exactly the same CPUs as `other`.
81 pub(super) fn equals(&self, other: &Self) -> bool {
82 let len = self.words.len().max(other.words.len());
83 (0..len).all(|index| self.word(index) == other.word(index))
84 }
85
86 fn word(&self, index: usize) -> libc::c_ulong {
87 self.words.get(index).copied().unwrap_or(0)
88 }
89
90 fn as_bytes(&self) -> usize {
91 std::mem::size_of_val(self.words.as_slice())
92 }
93
94 fn as_ptr(&self) -> *const libc::c_void {
95 self.words.as_ptr().cast()
96 }
97 }
98
99 /// Read the mask of the thread identified by `pid` (`0` is the calling
100 /// thread).
101 fn get_mask(pid: libc::pid_t) -> io::Result<CpuMask> {
102 let mut words = vec![0 as libc::c_ulong; INITIAL_WORDS];
103 loop {
104 let bytes = std::mem::size_of_val(words.as_slice());
105 // SAFETY: `words` is an owned, word-aligned buffer and `bytes` is
106 // its exact byte length; the kernel writes at most that many bytes.
107 let result = unsafe { libc::sched_getaffinity(pid, bytes, words.as_mut_ptr().cast()) };
108 if result == 0 {
109 // Keep the successful kernel buffer size: `sched_setaffinity`
110 // requires at least that many bytes even when the high words
111 // are empty.
112 return Ok(CpuMask { words });
113 }
114 let error = io::Error::last_os_error();
115 if error.raw_os_error() != Some(libc::EINVAL) {
116 return Err(error);
117 }
118 let next = words.len().checked_mul(2).filter(|&len| len <= MAX_WORDS);
119 let Some(next) = next else {
120 return Err(io::Error::other(
121 "CPU affinity mask is larger than the supported buffer",
122 ));
123 };
124 words.resize(next, 0);
125 }
126 }
127
128 /// Replace the mask of the thread identified by `pid` (`0` is the calling
129 /// thread) with `mask`.
130 fn set_mask(pid: libc::pid_t, mask: &CpuMask) -> io::Result<()> {
131 // SAFETY: `mask` owns a word-aligned buffer and `as_bytes` is its exact
132 // byte length; the kernel reads at most that many bytes.
133 let result = unsafe { libc::sched_setaffinity(pid, mask.as_bytes(), mask.as_ptr().cast()) };
134 if result == 0 {
135 Ok(())
136 } else {
137 Err(io::Error::last_os_error())
138 }
139 }
140
141 /// The main thread's mask at construction, a snapshot taken with
142 /// `sched_getaffinity(getpid())`.
143 ///
144 /// The process id names the thread-group leader, so a pool built lazily
145 /// from a worker of an already-pinned pool does not inherit that worker's
146 /// single CPU.
147 pub(super) fn main_thread_mask() -> Result<CpuMask, CpuThreadError> {
148 get_mask(std::process::id() as libc::pid_t).map_err(|source| CpuThreadError::Affinity {
149 operation: "sched_getaffinity",
150 source,
151 })
152 }
153
154 /// Set the calling thread's mask and verify the kernel kept all of it.
155 fn set_and_verify_current_thread(mask: &CpuMask) -> io::Result<()> {
156 set_mask(0, mask)?;
157 let actual = get_mask(0)?;
158 if actual.equals(mask) {
159 Ok(())
160 } else {
161 Err(io::Error::other(
162 "the effective CPU mask differs from the requested mask",
163 ))
164 }
165 }
166
167 /// Set the calling thread's mask to `mask`.
168 ///
169 /// A threaded cgroup may silently intersect the request with a narrower
170 /// cpuset, so the read-back is what turns a partially applied mask into a
171 /// typed failure before Rayon builds any worker.
172 pub(super) fn set_current_thread_to(mask: &CpuMask) -> Result<(), CpuThreadError> {
173 set_and_verify_current_thread(mask).map_err(|source| CpuThreadError::Affinity {
174 operation: "sched_setaffinity",
175 source,
176 })
177 }
178
179 /// Pin the calling thread to exactly `cpu` and verify the effective mask.
180 pub(super) fn pin_current_thread(cpu: usize) -> io::Result<()> {
181 let min_words = get_mask(0)?.words_len();
182 let expected = CpuMask::single(cpu, min_words);
183 set_and_verify_current_thread(&expected)
184 }
185
186 /// The calling thread's effective CPU ids.
187 pub(super) fn current_thread_cpus() -> Result<Vec<usize>, CpuThreadError> {
188 get_mask(0)
189 .map(|mask| mask.cpus())
190 .map_err(|source| CpuThreadError::Affinity {
191 operation: "sched_getaffinity",
192 source,
193 })
194 }
195}
196
197/// Failure reported when building a [`CpuBackend`] pool or applying its
198/// affinity policy.
199///
200/// A pool build and, on Linux, the affinity snapshot are the fallible steps of a
201/// [`CpuBackend`]: tenferro itself never builds threads, so the caller's thread
202/// policy is the caller's error to
203/// carry.
204///
205/// # Examples
206///
207/// ```
208/// use tensor4all_tensorbackend::CpuThreadError;
209///
210/// let error = CpuThreadError::ZeroThreads;
211/// assert!(error.to_string().contains("at least one"));
212/// ```
213#[derive(Debug, thiserror::Error)]
214pub enum CpuThreadError {
215 /// A [`CpuBackend`] was asked for zero workers.
216 #[error("a CPU backend needs at least one worker thread")]
217 ZeroThreads,
218 /// The worker pool for the requested worker count could not be built.
219 #[error("failed to build the CPU worker pool: {source}")]
220 Pool {
221 /// Original pool diagnostic.
222 #[source]
223 source: Box<tenferro_cpu::threadteam::Error>,
224 },
225 /// [`CpuBackend::pinned`] was given an empty CPU list.
226 #[error("a pinned CPU backend needs at least one CPU")]
227 EmptyCpuList,
228 /// [`CpuBackend::pinned`] was given the same CPU twice.
229 #[error("the pinned CPU list repeats CPU {cpu}")]
230 DuplicateCpu {
231 /// The repeated CPU id.
232 cpu: usize,
233 },
234 /// [`CpuBackend::pinned`] named a CPU outside the main thread's mask.
235 #[error("CPU {cpu} is outside the main thread's allowed CPU set")]
236 CpuNotAllowed {
237 /// The rejected CPU id.
238 cpu: usize,
239 },
240 /// CPU affinity is only implemented on Linux.
241 #[error("CPU affinity is not supported on this platform")]
242 UnsupportedPlatform,
243 /// An affinity system call failed.
244 #[error("{operation} failed: {source}")]
245 Affinity {
246 /// The system call that failed.
247 operation: &'static str,
248 /// Original operating-system diagnostic.
249 #[source]
250 source: std::io::Error,
251 },
252 /// The short-lived thread that builds a pool could not be started.
253 #[error("failed to start the CPU pool construction thread: {source}")]
254 ConstructionThread {
255 /// Original thread-spawn diagnostic.
256 #[source]
257 source: std::io::Error,
258 },
259 /// The CPU pool construction thread panicked before returning the pool.
260 #[error("the CPU pool construction thread panicked")]
261 ConstructionThreadPanic,
262 /// A pinned worker could not be confined to its CPU, or its startup
263 /// handshake failed.
264 #[error("CPU worker {worker} failed its affinity handshake: {source}")]
265 WorkerAffinity {
266 /// Rayon worker index, or zero for a closed startup channel.
267 worker: usize,
268 /// Original operating-system or verification diagnostic.
269 #[source]
270 source: std::io::Error,
271 },
272}
273
274/// Failure reported by [`set_default_cpu_threads`] when the default is fixed.
275///
276/// # Examples
277///
278/// ```
279/// use tensor4all_tensorbackend::AlreadyInitialized;
280///
281/// let error = AlreadyInitialized;
282/// assert_eq!(
283/// error.to_string(),
284/// "the process-wide default CPU thread count is already initialized"
285/// );
286/// ```
287#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
288#[error("the process-wide default CPU thread count is already initialized")]
289pub struct AlreadyInitialized;
290
291static DEFAULT_CPU_THREADS: OnceLock<usize> = OnceLock::new();
292
293/// Fix the worker count [`CpuBackend::new`] uses for the rest of the process.
294///
295/// The value is read once, by the first [`CpuBackend::new`] call that resolves a
296/// count; a backend built later keeps whatever policy it was built with. A
297/// first [`CpuBackend::new`] that cannot build its pool falls back to serial for
298/// that backend only and does not fix the default, so a later call tries again.
299/// Call this before any other thread can construct a default backend.
300///
301/// # Errors
302///
303/// Returns [`AlreadyInitialized`] when the default thread configuration was
304/// already fixed successfully, so a change is unsupported: either an earlier
305/// call or a [`CpuBackend::new`] that resolved a count fixed it. A failed first
306/// construction does not fix it.
307///
308/// # Examples
309///
310/// ```
311/// use std::num::NonZeroUsize;
312/// use tensor4all_tensorbackend::{set_default_cpu_threads, AlreadyInitialized};
313///
314/// // Whichever call sets the default first wins, and every later one is
315/// // rejected typed instead of panicking.
316/// let _ = set_default_cpu_threads(NonZeroUsize::new(3).unwrap());
317/// assert_eq!(
318/// set_default_cpu_threads(NonZeroUsize::MAX),
319/// Err(AlreadyInitialized)
320/// );
321/// ```
322pub fn set_default_cpu_threads(threads: NonZeroUsize) -> Result<(), AlreadyInitialized> {
323 DEFAULT_CPU_THREADS
324 .set(threads.get())
325 .map_err(|_| AlreadyInitialized)
326}
327
328/// How the idle workers of a CPU pool wait for the next parallel section.
329///
330/// A worker that has nothing to do polls for a while before it sleeps. With the
331/// default budget it keeps polling through a serial phase of the caller that is
332/// shorter than that budget, so a pool of `N` workers can use `N` times the wall
333/// time in CPU time. A smaller budget trades that CPU time for a longer wake-up
334/// of the next parallel section. The names follow OpenMP's `OMP_WAIT_POLICY`.
335///
336/// The budget is threadteam's `PoolBuilder::spin_polls`, a count of polls and
337/// not a duration, so how long it lasts depends on the CPU.
338///
339/// # Examples
340///
341/// ```
342/// use tensor4all_tensorbackend::CpuWaitPolicy;
343///
344/// assert_eq!(CpuWaitPolicy::default(), CpuWaitPolicy::Active);
345/// assert_eq!(CpuWaitPolicy::Passive.spin_polls(), Some(0));
346/// assert_eq!(CpuWaitPolicy::SpinPolls(4096).spin_polls(), Some(4096));
347/// ```
348#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
349pub enum CpuWaitPolicy {
350 /// Keep the pool's default spin budget (the behaviour before this type
351 /// existed).
352 #[default]
353 Active,
354 /// Do not spin: a waiting worker yields a few times and then sleeps, for a
355 /// shared machine or parallel test processes.
356 Passive,
357 /// Spin for this many polls before the yields and the sleep.
358 SpinPolls(u32),
359}
360
361impl CpuWaitPolicy {
362 /// The spin budget this policy sets on the pool, or `None` for the pool's
363 /// default.
364 ///
365 /// # Examples
366 ///
367 /// ```
368 /// use tensor4all_tensorbackend::CpuWaitPolicy;
369 ///
370 /// assert_eq!(CpuWaitPolicy::Active.spin_polls(), None);
371 /// assert_eq!(CpuWaitPolicy::SpinPolls(7).spin_polls(), Some(7));
372 /// ```
373 pub fn spin_polls(self) -> Option<u32> {
374 match self {
375 Self::Active => None,
376 Self::Passive => Some(0),
377 Self::SpinPolls(polls) => Some(polls),
378 }
379 }
380}
381
382/// Failure reported by [`set_default_cpu_wait_policy`] when the default is fixed.
383///
384/// # Examples
385///
386/// ```
387/// use tensor4all_tensorbackend::WaitPolicyAlreadyInitialized;
388///
389/// assert_eq!(
390/// WaitPolicyAlreadyInitialized.to_string(),
391/// "the process-wide default CPU wait policy is already initialized"
392/// );
393/// ```
394#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
395#[error("the process-wide default CPU wait policy is already initialized")]
396pub struct WaitPolicyAlreadyInitialized;
397
398static DEFAULT_CPU_WAIT_POLICY: OnceLock<CpuWaitPolicy> = OnceLock::new();
399
400/// Fix the [`CpuWaitPolicy`] of the pools that [`CpuBackend::new`] and
401/// [`CpuBackend::with_threads`] build for the rest of the process.
402///
403/// Only pools built after the call use it; a backend that already exists keeps
404/// the policy it was built with. Call it before any other thread can construct a
405/// backend. Without a call the policy is [`CpuWaitPolicy::Active`].
406///
407/// # Errors
408///
409/// Returns [`WaitPolicyAlreadyInitialized`] when the process-wide wait-policy
410/// configuration was already fixed by an earlier call, so a change is
411/// unsupported and the first value stays. Build a backend with
412/// [`CpuBackend::with_threads_and_wait_policy`] to use another policy for one
413/// backend.
414///
415/// # Examples
416///
417/// ```
418/// use tensor4all_tensorbackend::{
419/// default_cpu_wait_policy, set_default_cpu_wait_policy, CpuWaitPolicy,
420/// WaitPolicyAlreadyInitialized,
421/// };
422///
423/// // Whichever call sets the default first wins; a later one is rejected typed.
424/// let _ = set_default_cpu_wait_policy(CpuWaitPolicy::Passive);
425/// assert_eq!(
426/// set_default_cpu_wait_policy(CpuWaitPolicy::SpinPolls(1)),
427/// Err(WaitPolicyAlreadyInitialized)
428/// );
429/// assert_eq!(default_cpu_wait_policy(), CpuWaitPolicy::Passive);
430/// ```
431pub fn set_default_cpu_wait_policy(
432 policy: CpuWaitPolicy,
433) -> Result<(), WaitPolicyAlreadyInitialized> {
434 DEFAULT_CPU_WAIT_POLICY
435 .set(policy)
436 .map_err(|_| WaitPolicyAlreadyInitialized)
437}
438
439/// The [`CpuWaitPolicy`] new default pools use: the one
440/// [`set_default_cpu_wait_policy`] fixed, else [`CpuWaitPolicy::Active`].
441///
442/// # Examples
443///
444/// ```
445/// use tensor4all_tensorbackend::{default_cpu_wait_policy, CpuWaitPolicy};
446///
447/// // A process that never set it keeps today's behaviour.
448/// assert_eq!(default_cpu_wait_policy(), CpuWaitPolicy::Active);
449/// ```
450pub fn default_cpu_wait_policy() -> CpuWaitPolicy {
451 DEFAULT_CPU_WAIT_POLICY.get().copied().unwrap_or_default()
452}
453
454/// The CPU set each worker of a pool reported when it started, by worker index.
455type WorkerCpus = Vec<Vec<usize>>;
456
457/// The calling thread's CPU set, empty off Linux.
458#[cfg(target_os = "linux")]
459fn current_worker_cpus() -> Vec<usize> {
460 affinity::current_thread_cpus().unwrap_or_default()
461}
462
463/// The calling thread's CPU set, empty off Linux.
464#[cfg(not(target_os = "linux"))]
465fn current_worker_cpus() -> Vec<usize> {
466 Vec::new()
467}
468
469/// Build one worker pool with `threads` (or the environment's count when
470/// `None`), recording the CPU set each worker starts on.
471///
472/// On Linux the build runs on a short-lived construction thread whose mask is
473/// first set to the main thread's mask snapshot, so a pool built from a singly
474/// pinned worker still spans the main thread's CPUs. The workers inherit the
475/// construction thread's mask.
476///
477/// Off Linux the pool is built on the calling thread and no affinity call is
478/// made.
479fn build_pool(
480 threads: Option<usize>,
481 wait: CpuWaitPolicy,
482) -> Result<(Pool, WorkerCpus), CpuThreadError> {
483 #[cfg(target_os = "linux")]
484 {
485 let mask = affinity::main_thread_mask()?;
486 std::thread::Builder::new()
487 .name("tensor4all-cpu-pool".to_owned())
488 .spawn(move || {
489 affinity::set_current_thread_to(&mask)?;
490 build_pool_here(threads, wait)
491 })
492 .map_err(|source| CpuThreadError::ConstructionThread { source })?
493 .join()
494 .map_err(|_| CpuThreadError::ConstructionThreadPanic)?
495 }
496 #[cfg(not(target_os = "linux"))]
497 {
498 build_pool_here(threads, wait)
499 }
500}
501
502/// The worker count Rayon itself resolves from the environment, in Rayon's
503/// order: `RAYON_NUM_THREADS`, the deprecated `RAYON_RS_NUM_CPUS`, then
504/// [`std::thread::available_parallelism`] (which already accounts for the
505/// process affinity mask and cgroup quotas). `RAYON_NUM_THREADS=0` or a
506/// malformed value means "use the platform default", as in Rayon.
507///
508/// The pool builder this type uses does not read the environment, so the
509/// precedence of [`set_default_cpu_threads`] → `RAYON_NUM_THREADS` → affinity
510/// is resolved here (R-performance-61).
511fn rayon_env_worker_count() -> usize {
512 let platform_default = || std::thread::available_parallelism().map_or(1, NonZeroUsize::get);
513 match std::env::var("RAYON_NUM_THREADS")
514 .ok()
515 .and_then(|value| value.parse::<usize>().ok())
516 {
517 Some(count) if count >= 1 => return count,
518 Some(_) => return platform_default(),
519 None => {}
520 }
521 match std::env::var("RAYON_RS_NUM_CPUS")
522 .ok()
523 .and_then(|value| value.parse::<usize>().ok())
524 {
525 Some(count) if count >= 1 => count,
526 _ => platform_default(),
527 }
528}
529
530/// `builder` with the spin budget of `wait`; the pool's default for [`CpuWaitPolicy::Active`].
531fn with_wait_policy(builder: PoolBuilder, wait: CpuWaitPolicy) -> PoolBuilder {
532 match wait.spin_polls() {
533 Some(polls) => builder.spin_polls(polls),
534 None => builder,
535 }
536}
537
538/// Build one worker pool on the calling thread.
539fn build_pool_here(
540 threads: Option<usize>,
541 wait: CpuWaitPolicy,
542) -> Result<(Pool, WorkerCpus), CpuThreadError> {
543 let threads = threads.unwrap_or_else(rayon_env_worker_count);
544 let (sender, receiver) = std::sync::mpsc::channel();
545 let builder = Pool::builder().threads(threads).on_start(move |worker| {
546 let _ = sender.send((worker, current_worker_cpus()));
547 });
548 let pool = with_wait_policy(builder, wait)
549 .build()
550 .map_err(|source| CpuThreadError::Pool {
551 source: Box::new(source),
552 })?;
553 // A worker runs its start hook in its own thread, so the reports arrive
554 // while and after the pool is built; collect one per worker before the
555 // backend exposes them.
556 let mut worker_cpus: WorkerCpus = vec![Vec::new(); pool.width().get()];
557 for _ in 0..worker_cpus.len() {
558 match receiver.recv_timeout(std::time::Duration::from_secs(1)) {
559 Ok((worker, cpus)) => {
560 if let Some(slot) = worker_cpus.get_mut(worker) {
561 *slot = cpus;
562 }
563 }
564 // A worker that never reports leaves its entry empty rather than
565 // blocking the pool build.
566 Err(_) => break,
567 }
568 }
569 Ok((pool, worker_cpus))
570}
571
572/// tensor4all's owned CPU execution handle: one [`CpuEngine`].
573///
574/// tenferro no longer has a backend session or a `CpuContext`; a caller builds
575/// a Rayon pool, hands it to [`CpuEngine::new`], and keeps the engine for as
576/// long as it wants the thread policy, the buffer pool, the prepared-plan cache
577/// and the contraction scratch to stay warm. This type is that caller-held
578/// owner for tensor4all: the implicit frontend's resource owner during the
579/// transition to engine-explicit APIs, which will replace it later.
580///
581/// It is **not** [`Clone`]. Cloning an execution domain was how the previous
582/// session API shared one engine between callers; with a caller-held engine the
583/// caller owns it outright, and a second execution domain needs a second
584/// backend.
585///
586/// [`CpuBackend::new`] resolves its worker count in this order: the value
587/// [`set_default_cpu_threads`] fixed, else the Rayon environment
588/// (`RAYON_NUM_THREADS`, then the deprecated `RAYON_RS_NUM_CPUS`), else the
589/// platform's parallelism, which is evaluated under the main thread's mask and
590/// may be lower than that mask's CPU count because of cgroup quotas. A
591/// resolved count of one uses [`CpuEngine::serial`] and owns no pool. The count
592/// is fixed by the first successful [`CpuBackend::new`] of the process, or by the
593/// setter. If the pool or the Linux mask snapshot cannot be used,
594/// [`CpuBackend::new`] falls back to [`CpuEngine::serial`] for that backend only
595/// instead of panicking, and a later call retries.
596///
597/// Every pool this type builds on Linux confines its workers to the main
598/// thread's allowed CPU set, captured as a snapshot when the pool is built.
599/// This applies to the pool's workers only: operations that run inline on the
600/// caller (small operations, Rayon-worker entries that use the serial inline
601/// engine, and hataori lane threads) are not moved. A cpuset or CPU-hotplug
602/// change needs a new backend, not a continuous watcher.
603///
604/// [`CpuBackend::pinned`] is an opt-in per-worker pinning mode whose benefit is
605/// unmeasured. Disjoint pinned backends confine their *pools'* workers; they do
606/// not isolate all execution of their engines, because an entry from any Rayon
607/// worker runs inline on that worker. A provider team created from a pinned
608/// worker inherits that worker's one CPU.
609///
610/// # Examples
611///
612/// ```
613/// use tensor4all_tensorbackend::CpuBackend;
614/// use tenferro_cpu::CpuEngine;
615///
616/// // A serial backend runs every operation on the caller's thread.
617/// let backend = CpuBackend::with_threads(1)?;
618/// assert_eq!(backend.num_threads(), 1);
619///
620/// // A caller that already owns an engine can hand it in unchanged.
621/// let backend = CpuBackend::from(CpuEngine::serial());
622/// assert_eq!(backend.num_threads(), 1);
623/// # Ok::<(), Box<dyn std::error::Error>>(())
624/// ```
625pub struct CpuBackend {
626 engine: CpuEngine,
627 /// The CPU set each pool worker reported at start, by worker index; empty
628 /// for a serial engine or a caller-supplied engine.
629 worker_cpus: WorkerCpus,
630}
631
632impl std::fmt::Debug for CpuBackend {
633 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
634 f.debug_struct("CpuBackend")
635 .field("engine", &self.engine)
636 .field("worker_cpus", &self.worker_cpus())
637 .finish()
638 }
639}
640
641impl CpuBackend {
642 /// Build a backend with the default thread policy.
643 ///
644 /// The worker count is [`set_default_cpu_threads`] if it was called; else
645 /// `RAYON_NUM_THREADS` (then the deprecated `RAYON_RS_NUM_CPUS`); else the
646 /// platform's parallelism, evaluated under the main thread's mask, which may
647 /// be lower than that mask's CPU count because of cgroup quotas. The pool's
648 /// default 16 MiB worker stack, which the CPU providers need, is kept. The
649 /// count is fixed by the first successful call, or by the setter.
650 ///
651 /// On Linux the pool is built on a short-lived construction thread whose
652 /// mask is first set to the main thread's mask snapshot, so building from a
653 /// worker of a pinned pool does not inherit that worker's single CPU. The
654 /// pool's workers inherit the snapshot; inline operations of the caller are
655 /// not moved.
656 ///
657 /// The pool build and the Linux affinity snapshot are the fallible steps,
658 /// and this constructor does not report them: if either cannot be used,
659 /// this backend is [`CpuEngine::serial`] instead, which is correct but uses
660 /// no worker threads, and the failure is not stored as the default. A later
661 /// call retries the construction. Use [`CpuBackend::with_threads`] or
662 /// [`CpuBackend::pinned`] when that difference matters.
663 ///
664 /// # Examples
665 ///
666 /// ```
667 /// use tensor4all_tensorbackend::CpuBackend;
668 ///
669 /// let backend = CpuBackend::new();
670 /// assert!(backend.num_threads() >= 1);
671 /// ```
672 pub fn new() -> Self {
673 if let Some(&threads) = DEFAULT_CPU_THREADS.get() {
674 return Self::with_threads_or_serial(threads);
675 }
676 match build_pool(None, default_cpu_wait_policy()) {
677 Ok((pool, worker_cpus)) => {
678 let resolved = pool.width().get();
679 let fixed = match DEFAULT_CPU_THREADS.set(resolved) {
680 Ok(()) => resolved,
681 Err(_) => *DEFAULT_CPU_THREADS.get().unwrap_or(&resolved),
682 };
683 if fixed != resolved {
684 return Self::with_threads_or_serial(fixed);
685 }
686 if resolved == 1 {
687 // The pool is dropped with this branch: the backend is serial
688 // and owns no workers, so it records none.
689 Self {
690 engine: CpuEngine::serial(),
691 worker_cpus: WorkerCpus::default(),
692 }
693 } else {
694 Self {
695 engine: CpuEngine::new(pool),
696 worker_cpus,
697 }
698 }
699 }
700 Err(_) => Self {
701 engine: CpuEngine::serial(),
702 worker_cpus: WorkerCpus::default(),
703 },
704 }
705 }
706
707 /// Build a backend with `threads` workers, or serial for one.
708 fn with_threads_or_serial(threads: usize) -> Self {
709 Self::with_threads(threads).unwrap_or_else(|_| Self {
710 engine: CpuEngine::serial(),
711 worker_cpus: WorkerCpus::default(),
712 })
713 }
714
715 /// Build a backend that may use `threads` workers.
716 ///
717 /// One worker builds a [`CpuEngine::serial`] engine, which owns no pool, so
718 /// it stays usable from inside another pool. Larger counts build a Rayon
719 /// pool with [`tenferro_cpu::Pool::builder`], so provider code gets the
720 /// worker stack it needs; the pool's workers are confined to the main
721 /// thread's allowed CPU set snapshot on Linux.
722 ///
723 /// # Errors
724 ///
725 /// Returns [`CpuThreadError::ZeroThreads`] for zero,
726 /// [`CpuThreadError::Affinity`] when the mask snapshot or its application
727 /// fails, [`CpuThreadError::ConstructionThread`] when the construction
728 /// thread cannot start, and [`CpuThreadError::Pool`] when the pool cannot be
729 /// built.
730 ///
731 /// # Examples
732 ///
733 /// ```
734 /// use tensor4all_tensorbackend::{CpuBackend, CpuThreadError};
735 ///
736 /// assert!(matches!(
737 /// CpuBackend::with_threads(0),
738 /// Err(CpuThreadError::ZeroThreads)
739 /// ));
740 /// let backend = CpuBackend::with_threads(2)?;
741 /// assert_eq!(backend.num_threads(), 2);
742 /// # Ok::<(), Box<dyn std::error::Error>>(())
743 /// ```
744 pub fn with_threads(threads: usize) -> Result<Self, CpuThreadError> {
745 Self::with_threads_and_wait_policy(threads, default_cpu_wait_policy())
746 }
747
748 /// Build a backend that may use `threads` workers, whose idle workers wait
749 /// as `wait` says.
750 ///
751 /// [`CpuBackend::with_threads`] is this with [`default_cpu_wait_policy`].
752 /// One worker builds a [`CpuEngine::serial`] engine, which owns no pool and
753 /// so has nothing to wait for; `wait` is then unused.
754 ///
755 /// # Errors
756 ///
757 /// The same errors as [`CpuBackend::with_threads`].
758 ///
759 /// # Examples
760 ///
761 /// ```
762 /// use tensor4all_tensorbackend::{CpuBackend, CpuWaitPolicy};
763 ///
764 /// let backend = CpuBackend::with_threads_and_wait_policy(2, CpuWaitPolicy::Passive)?;
765 /// assert_eq!(backend.num_threads(), 2);
766 /// # Ok::<(), Box<dyn std::error::Error>>(())
767 /// ```
768 pub fn with_threads_and_wait_policy(
769 threads: usize,
770 wait: CpuWaitPolicy,
771 ) -> Result<Self, CpuThreadError> {
772 if threads == 0 {
773 return Err(CpuThreadError::ZeroThreads);
774 }
775 if threads == 1 {
776 return Ok(Self {
777 engine: CpuEngine::serial(),
778 worker_cpus: WorkerCpus::default(),
779 });
780 }
781 let (pool, worker_cpus) = build_pool(Some(threads), wait)?;
782 Ok(Self {
783 engine: CpuEngine::new(pool),
784 worker_cpus,
785 })
786 }
787
788 /// Build a backend whose worker `i` is pinned to `cpus[i]`.
789 ///
790 /// This is opt-in per-worker pinning on Linux; its performance benefit is
791 /// unmeasured. Every listed CPU must be in the main thread's allowed set
792 /// ([`Self::allowed_cpus`]), which lets a caller split that set into
793 /// disjoint lists, one per engine. Each worker gets exactly its one CPU,
794 /// so a provider team created from a pinned worker inherits that one CPU.
795 /// A one-element list still builds a one-worker pool rather than a serial
796 /// engine; inline operations of a one-worker engine still run on the
797 /// caller.
798 ///
799 /// The mask is a snapshot: a cpuset or CPU-hotplug change needs a new
800 /// backend. Only the pool's workers are pinned; lane threads and inline
801 /// operations are not moved.
802 ///
803 /// # Errors
804 ///
805 /// Returns [`CpuThreadError::EmptyCpuList`] for an empty list,
806 /// [`CpuThreadError::DuplicateCpu`] for a repeated CPU,
807 /// [`CpuThreadError::CpuNotAllowed`] for a CPU outside the allowed set,
808 /// [`CpuThreadError::UnsupportedPlatform`] off Linux,
809 /// [`CpuThreadError::Affinity`] when the snapshot or a worker's mask fails,
810 /// [`CpuThreadError::WorkerAffinity`] when a worker cannot be pinned to
811 /// exactly its CPU, and [`CpuThreadError::Pool`] when the pool cannot be
812 /// built.
813 ///
814 /// # Examples
815 ///
816 /// ```
817 /// use tensor4all_tensorbackend::{CpuBackend, CpuThreadError};
818 ///
819 /// assert!(matches!(
820 /// CpuBackend::pinned(&[]),
821 /// Err(CpuThreadError::EmptyCpuList)
822 /// ));
823 /// # #[cfg(target_os = "linux")]
824 /// # {
825 /// let allowed = CpuBackend::allowed_cpus()?;
826 /// let backend = CpuBackend::pinned(&allowed[..1])?;
827 /// assert_eq!(backend.num_threads(), 1);
828 /// # }
829 /// # Ok::<(), Box<dyn std::error::Error>>(())
830 /// ```
831 pub fn pinned(cpus: &[usize]) -> Result<Self, CpuThreadError> {
832 if cpus.is_empty() {
833 return Err(CpuThreadError::EmptyCpuList);
834 }
835 #[cfg(not(target_os = "linux"))]
836 {
837 let _ = cpus;
838 Err(CpuThreadError::UnsupportedPlatform)
839 }
840 #[cfg(target_os = "linux")]
841 {
842 let allowed = affinity::main_thread_mask()?.cpus();
843 let mut seen = std::collections::HashSet::with_capacity(cpus.len());
844 for &cpu in cpus {
845 if !seen.insert(cpu) {
846 return Err(CpuThreadError::DuplicateCpu { cpu });
847 }
848 if allowed.binary_search(&cpu).is_err() {
849 return Err(CpuThreadError::CpuNotAllowed { cpu });
850 }
851 }
852
853 let workers = cpus.to_vec();
854 let (sender, receiver) = std::sync::mpsc::channel();
855 let builder = Pool::builder()
856 .threads(workers.len())
857 .on_start(move |worker| {
858 let outcome = workers
859 .get(worker)
860 .copied()
861 .ok_or_else(|| {
862 std::io::Error::other("worker index outside the pinned CPU list")
863 })
864 .and_then(affinity::pin_current_thread);
865 let _ = sender.send((worker, outcome, current_worker_cpus()));
866 });
867 let pool = with_wait_policy(builder, default_cpu_wait_policy())
868 .build()
869 .map_err(|source| CpuThreadError::Pool {
870 source: Box::new(source),
871 })?;
872
873 // Every worker of a successfully built pool runs the start handler,
874 // which sends exactly one report, and the registry keeps its sender
875 // alive while the pool is alive. The loop therefore receives one
876 // report per worker. A closed channel is not expected, but it is
877 // still a worker affinity-handshake failure rather than a separate
878 // unreachable error case.
879 let mut worker_cpus: WorkerCpus = vec![Vec::new(); cpus.len()];
880 for _ in 0..cpus.len() {
881 let (worker, outcome, own_cpus) =
882 receiver
883 .recv()
884 .map_err(|source| CpuThreadError::WorkerAffinity {
885 worker: 0,
886 source: std::io::Error::other(source),
887 })?;
888 if let Some(slot) = worker_cpus.get_mut(worker) {
889 *slot = own_cpus;
890 }
891 if let Err(source) = outcome {
892 return Err(CpuThreadError::WorkerAffinity { worker, source });
893 }
894 }
895 Ok(Self {
896 engine: CpuEngine::new(pool),
897 worker_cpus,
898 })
899 }
900 }
901
902 /// The CPUs the main thread is allowed to run on right now, in ascending
903 /// order.
904 ///
905 /// This is a snapshot, read with `sched_getaffinity(getpid())` on Linux. A
906 /// caller can split it into disjoint lists and pass one list to each
907 /// [`CpuBackend::pinned`] engine. Off Linux this reports
908 /// [`CpuThreadError::UnsupportedPlatform`].
909 ///
910 /// # Errors
911 ///
912 /// Returns [`CpuThreadError::UnsupportedPlatform`] off Linux and
913 /// [`CpuThreadError::Affinity`] when the affinity query fails.
914 ///
915 /// # Examples
916 ///
917 /// ```
918 /// use tensor4all_tensorbackend::CpuBackend;
919 ///
920 /// # #[cfg(target_os = "linux")]
921 /// # {
922 /// let allowed = CpuBackend::allowed_cpus()?;
923 /// assert!(!allowed.is_empty());
924 /// assert!(allowed.windows(2).all(|pair| pair[0] < pair[1]));
925 /// # }
926 /// # Ok::<(), Box<dyn std::error::Error>>(())
927 /// ```
928 pub fn allowed_cpus() -> Result<Vec<usize>, CpuThreadError> {
929 #[cfg(not(target_os = "linux"))]
930 {
931 Err(CpuThreadError::UnsupportedPlatform)
932 }
933 #[cfg(target_os = "linux")]
934 {
935 Ok(affinity::main_thread_mask()?.cpus())
936 }
937 }
938
939 /// The number of workers operations on this backend may use.
940 ///
941 /// # Examples
942 ///
943 /// ```
944 /// use tensor4all_tensorbackend::CpuBackend;
945 ///
946 /// assert_eq!(CpuBackend::with_threads(1)?.num_threads(), 1);
947 /// # Ok::<(), Box<dyn std::error::Error>>(())
948 /// ```
949 pub fn num_threads(&self) -> usize {
950 self.engine.num_threads()
951 }
952
953 /// The CPU set each worker of this backend's pool reported when it started,
954 /// by worker index.
955 ///
956 /// Empty for a serial engine, for a caller-supplied engine
957 /// ([`CpuBackend::from`]) and off Linux. It is a snapshot taken at pool
958 /// build time, so it shows what the workers actually inherited — the main
959 /// thread's allowed set for [`CpuBackend::new`] and
960 /// [`CpuBackend::with_threads`], and its one CPU for
961 /// [`CpuBackend::pinned`].
962 ///
963 /// # Examples
964 ///
965 /// ```
966 /// use tensor4all_tensorbackend::CpuBackend;
967 ///
968 /// let backend = CpuBackend::with_threads(1)?;
969 /// assert!(backend.worker_cpus().is_empty());
970 /// # Ok::<(), Box<dyn std::error::Error>>(())
971 /// ```
972 pub fn worker_cpus(&self) -> Vec<Vec<usize>> {
973 self.worker_cpus.clone()
974 }
975
976 /// Borrow the engine this backend owns.
977 ///
978 /// # Examples
979 ///
980 /// ```
981 /// use tensor4all_tensorbackend::CpuBackend;
982 ///
983 /// let backend = CpuBackend::with_threads(1)?;
984 /// assert_eq!(backend.engine().buffer_pool_len(), 0);
985 /// # Ok::<(), Box<dyn std::error::Error>>(())
986 /// ```
987 pub fn engine(&self) -> &CpuEngine {
988 &self.engine
989 }
990
991 /// Borrow the engine mutably, for one operation or one session.
992 ///
993 /// # Examples
994 ///
995 /// ```
996 /// use tensor4all_tensorbackend::CpuBackend;
997 ///
998 /// let mut backend = CpuBackend::with_threads(1)?;
999 /// backend.engine_mut().reset_buffer_pool();
1000 /// assert_eq!(backend.engine().buffer_pool_len(), 0);
1001 /// # Ok::<(), Box<dyn std::error::Error>>(())
1002 /// ```
1003 pub fn engine_mut(&mut self) -> &mut CpuEngine {
1004 &mut self.engine
1005 }
1006
1007 /// Consume the backend and return the engine it owns.
1008 ///
1009 /// A caller that hands engines to another owner, such as a lane executor,
1010 /// uses this instead of borrowing and cloning: [`CpuBackend`] deliberately
1011 /// does not clone an execution domain.
1012 ///
1013 /// # Examples
1014 ///
1015 /// ```
1016 /// use tensor4all_tensorbackend::CpuBackend;
1017 ///
1018 /// let engine = CpuBackend::with_threads(1)?.into_engine();
1019 /// assert_eq!(engine.num_threads(), 1);
1020 /// # Ok::<(), Box<dyn std::error::Error>>(())
1021 /// ```
1022 pub fn into_engine(self) -> CpuEngine {
1023 self.engine
1024 }
1025}
1026
1027impl Default for CpuBackend {
1028 fn default() -> Self {
1029 Self::new()
1030 }
1031}
1032
1033impl Deref for CpuBackend {
1034 type Target = CpuEngine;
1035
1036 fn deref(&self) -> &Self::Target {
1037 &self.engine
1038 }
1039}
1040
1041impl DerefMut for CpuBackend {
1042 fn deref_mut(&mut self) -> &mut Self::Target {
1043 &mut self.engine
1044 }
1045}
1046
1047impl From<CpuEngine> for CpuBackend {
1048 fn from(engine: CpuEngine) -> Self {
1049 Self {
1050 engine,
1051 worker_cpus: WorkerCpus::default(),
1052 }
1053 }
1054}
1055
1056/// Caller-owned execution domain used by context-aware tensor algorithms.
1057///
1058/// Values are validated against the exact runtime represented by the selected
1059/// context; no implicit host/device transfer is performed by this enum.
1060#[derive(Clone, Debug)]
1061pub enum ExecutionContext {
1062 /// Host execution through one caller-owned CPU context.
1063 Cpu(Arc<CpuExecutionContext>),
1064 /// CUDA execution through one caller-owned CUDA context.
1065 #[cfg(feature = "tenferro-cuda")]
1066 Cuda(Arc<crate::cuda::CudaExecutionContext>),
1067}
1068
1069impl ExecutionContext {
1070 /// Check whether this is the process-global CPU context.
1071 ///
1072 /// Compatibility boundary: legacy CPU-global callers route through the
1073 /// historical host code paths (bitwise-identical numerics) while explicit
1074 /// contexts use the scoped primitives. Without the global-defaults
1075 /// feature there is no global context, so this always reports false.
1076 pub fn is_global_default_cpu(&self) -> bool {
1077 #[cfg(feature = "global-defaults")]
1078 {
1079 match self {
1080 ExecutionContext::Cpu(context) => {
1081 let own = context.eager_runtime().map(|runtime| runtime.id());
1082 let global = defaults::default_eager_ctx().map(|runtime| runtime.id());
1083 matches!((own, global), (Ok(a), Ok(b)) if a == b)
1084 }
1085 #[cfg(feature = "tenferro-cuda")]
1086 ExecutionContext::Cuda(_) => false,
1087 }
1088 }
1089 #[cfg(not(feature = "global-defaults"))]
1090 {
1091 let _ = self;
1092 false
1093 }
1094 }
1095}
1096
1097/// Error returned by explicit CPU context graph or eager-runtime operations.
1098///
1099/// The original tenferro diagnostic is retained as the error source.
1100///
1101/// # Examples
1102///
1103/// ```
1104/// use std::error::Error;
1105/// use std::sync::Arc;
1106/// use tensor4all_tensorbackend::CpuExecutionContextError;
1107///
1108/// let error = CpuExecutionContextError::Initialization {
1109/// component: "graph runtime",
1110/// source: Arc::new(std::io::Error::other("registration failed")),
1111/// };
1112/// assert!(error.source().is_some());
1113/// ```
1114#[derive(Debug, Clone, thiserror::Error)]
1115pub enum CpuExecutionContextError {
1116 /// A context-owned graph or eager runtime could not be initialized.
1117 #[error("failed to initialize {component}: {source}")]
1118 Initialization {
1119 /// Context component being initialized.
1120 component: &'static str,
1121 /// Original tenferro diagnostic.
1122 #[source]
1123 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
1124 },
1125 /// The calling thread is already inside a session of this context.
1126 ///
1127 /// One context owns one engine and lends it to one caller at a time. A
1128 /// session entered from inside another session on the same thread would
1129 /// wait for a lock that thread already holds, so it is reported as this
1130 /// typed error before any lock is taken.
1131 #[error("recursive CPU execution-context session entry on one thread")]
1132 Reentered,
1133 /// Graph compilation or execution failed.
1134 #[error("CPU graph {operation} failed: {source}")]
1135 Graph {
1136 /// Graph operation that failed.
1137 operation: &'static str,
1138 /// Original tenferro diagnostic.
1139 #[source]
1140 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
1141 },
1142}
1143
1144thread_local! {
1145 /// Identities of the execution contexts whose engines this thread holds.
1146 ///
1147 /// Re-entry is per context: holding one context's engine must not block an
1148 /// implicit entry that uses another context's engine.
1149 static ACTIVE_CONTEXTS: RefCell<Vec<u64>> = const { RefCell::new(Vec::new()) };
1150}
1151
1152static NEXT_CONTEXT_ID: AtomicU64 = AtomicU64::new(1);
1153
1154pub(crate) fn next_context_id() -> u64 {
1155 NEXT_CONTEXT_ID.fetch_add(1, Ordering::Relaxed)
1156}
1157
1158pub(crate) struct CanonicalSessionGuard {
1159 context_id: u64,
1160}
1161
1162impl CanonicalSessionGuard {
1163 /// Whether this context's engine is already held on this thread.
1164 pub(crate) fn is_active(context_id: u64) -> bool {
1165 ACTIVE_CONTEXTS.with(|active| active.borrow().contains(&context_id))
1166 }
1167
1168 pub(crate) fn enter(context_id: u64) -> Self {
1169 ACTIVE_CONTEXTS.with(|active| active.borrow_mut().push(context_id));
1170 Self { context_id }
1171 }
1172}
1173
1174impl Drop for CanonicalSessionGuard {
1175 fn drop(&mut self) {
1176 ACTIVE_CONTEXTS.with(|active| {
1177 let mut active = active.borrow_mut();
1178 if active.last() == Some(&self.context_id) {
1179 active.pop();
1180 } else {
1181 active.retain(|context_id| *context_id != self.context_id);
1182 }
1183 });
1184 }
1185}
1186
1187impl CpuExecutionContextError {
1188 fn initialization(
1189 component: &'static str,
1190 source: impl std::error::Error + Send + Sync + 'static,
1191 ) -> Self {
1192 Self::Initialization {
1193 component,
1194 source: Arc::new(source),
1195 }
1196 }
1197
1198 fn graph(
1199 operation: &'static str,
1200 source: impl std::error::Error + Send + Sync + 'static,
1201 ) -> Self {
1202 Self::Graph {
1203 operation,
1204 source: Arc::new(source),
1205 }
1206 }
1207}
1208
1209/// Failure entering the engine of the context that owns an eager runtime.
1210///
1211/// The implicit frontend resolves the engine from the runtime of the operand it
1212/// acts on; the owner may be a CPU or a CUDA execution context, so this error
1213/// keeps the owning context's typed diagnostic instead of flattening both into
1214/// one string.
1215///
1216/// # Examples
1217///
1218/// ```
1219/// use tensor4all_tensorbackend::{
1220/// CpuExecutionContextError, ExecutionContextEntryError,
1221/// };
1222///
1223/// let error = ExecutionContextEntryError::from(CpuExecutionContextError::Reentered);
1224/// assert!(matches!(
1225/// error,
1226/// ExecutionContextEntryError::Cpu(CpuExecutionContextError::Reentered)
1227/// ));
1228/// ```
1229#[cfg(feature = "global-defaults")]
1230#[derive(Debug, Clone, thiserror::Error)]
1231pub enum ExecutionContextEntryError {
1232 /// The owning CPU context rejected the session entry.
1233 #[error(transparent)]
1234 Cpu(#[from] CpuExecutionContextError),
1235 /// The owning CUDA context rejected the session entry.
1236 #[cfg(feature = "tenferro-cuda")]
1237 #[error(transparent)]
1238 Cuda(#[from] crate::cuda::CudaExecutionContextError),
1239}
1240
1241/// The engine a context owns, plus the graph runtime derived from it.
1242///
1243/// The registration the graph runtime holds is derived from the engine, and
1244/// `Runtime::run` needs `&mut engine`, so both live under one lock. The
1245/// [`GraphCompiler`] is not here: it never touches the engine, so it has its own
1246/// lock and a compile never waits for a running graph.
1247struct EngineState {
1248 backend: CpuBackend,
1249 runtime: Option<Runtime>,
1250}
1251
1252impl EngineState {
1253 fn new(backend: CpuBackend) -> Self {
1254 Self {
1255 backend,
1256 runtime: None,
1257 }
1258 }
1259
1260 /// The graph runtime of this context, built from the engine on first use.
1261 fn runtime(&mut self) -> Result<&mut Runtime, CpuExecutionContextError> {
1262 let Self { backend, runtime } = self;
1263 match runtime {
1264 Some(runtime) => Ok(runtime),
1265 None => {
1266 let built = build_graph_runtime(backend)?;
1267 Ok(runtime.insert(built))
1268 }
1269 }
1270 }
1271}
1272
1273/// The runtime engine id `engine` is registered under.
1274///
1275/// An engine built by [`CpuEngine::with_runtime_engine_id`] runs under its own
1276/// id, so every registration and extension module installed on its behalf must
1277/// use that id rather than the canonical [`tenferro_cpu::runtime_engine_id`].
1278fn engine_runtime_id(engine: &CpuEngine) -> tenferro::EngineId {
1279 ExecutionEngine::target(engine).engine_id().clone()
1280}
1281
1282/// The engine state one context owns, behind a handle that can also be reached
1283/// through the eager-runtime owner table.
1284///
1285/// A [`CpuExecutionContext`] holds this in an [`Arc`]; the process-wide owner
1286/// table holds only a [`Weak`](std::sync::Weak), so the entry dangles as soon as the context is
1287/// dropped instead of keeping the engine alive.
1288pub(crate) struct EngineOwner {
1289 id: u64,
1290 /// The engine's registered runtime id, captured once at construction.
1291 ///
1292 /// The eager runtime installs its extension modules under this id, and it is
1293 /// read without taking the engine lock so building the eager runtime can
1294 /// never wait for a running session.
1295 engine_id: tenferro::EngineId,
1296 /// The engine's prepared-plan cache, captured once at construction.
1297 ///
1298 /// A session entered from a Rayon worker runs inline on a fresh serial
1299 /// engine that shares this cache (tensor4all-rs#830). Holding the handle
1300 /// here keeps that entry off the engine lock: the worker never waits for an
1301 /// engine its own pool may be holding.
1302 plans: tenferro_cpu::CpuPlanCache,
1303 engine: Mutex<EngineState>,
1304}
1305
1306impl EngineOwner {
1307 fn new(backend: CpuBackend) -> Self {
1308 Self {
1309 id: next_context_id(),
1310 engine_id: engine_runtime_id(backend.engine()),
1311 plans: backend.engine().plan_cache().clone(),
1312 engine: Mutex::new(EngineState::new(backend)),
1313 }
1314 }
1315
1316 /// Lock the engine state, recovering a lock poisoned by a panic.
1317 ///
1318 /// A panic while the engine was borrowed leaves tenferro's engine marked
1319 /// broken, and the lock poisoned. The engine is kept: it carries the
1320 /// caller's pool, allocation domain, batch policy and runtime engine id, so
1321 /// replacing it with a default engine would silently change the caller's
1322 /// execution policy. Recovering the poison only makes the lock usable again;
1323 /// operations on the broken engine are refused with tenferro's typed
1324 /// `CpuEngineError::Broken`, and the caller builds a new context for a new
1325 /// engine.
1326 fn lock_engine(&self) -> std::sync::MutexGuard<'_, EngineState> {
1327 match self.engine.lock() {
1328 Ok(guard) => guard,
1329 Err(poisoned) => {
1330 let guard = poisoned.into_inner();
1331 self.engine.clear_poison();
1332 guard
1333 }
1334 }
1335 }
1336
1337 /// Check the per-context re-entry guard and install it for one entry.
1338 ///
1339 /// Every fallible entry checks before it locks: a nested entry would wait
1340 /// for an engine lock this thread already holds, so it is reported as
1341 /// [`CpuExecutionContextError::Reentered`] instead.
1342 fn enter_session(&self) -> Result<CanonicalSessionGuard, CpuExecutionContextError> {
1343 if CanonicalSessionGuard::is_active(self.id) {
1344 return Err(CpuExecutionContextError::Reentered);
1345 }
1346 Ok(CanonicalSessionGuard::enter(self.id))
1347 }
1348
1349 /// The single guarded entry that locks this owner's engine.
1350 ///
1351 /// This is the only way to lock the engine from a fallible entry: the
1352 /// re-entry guard is checked and installed here, so no caller can hold the
1353 /// lock without holding the guard too. [`CpuExecutionContext::with_backend`]
1354 /// is the one remaining locking entry, and it installs the same guard
1355 /// without being able to report the rejection.
1356 fn begin_session(&self) -> Result<EngineEntry<'_>, CpuExecutionContextError> {
1357 let guard = self.enter_session()?;
1358 Ok(EngineEntry {
1359 state: self.lock_engine(),
1360 _guard: guard,
1361 })
1362 }
1363
1364 /// Run `f` in one session of this owner's engine.
1365 ///
1366 /// This is the single session-entry implementation shared by the owning
1367 /// context and the eager-runtime route: the same per-context re-entry guard
1368 /// and the same inline-serial rule for Rayon workers apply either way.
1369 fn with_session<R>(
1370 &self,
1371 f: impl FnOnce(&mut CpuEngine) -> R,
1372 ) -> Result<R, CpuExecutionContextError> {
1373 // A worker runs inline on its own serial engine instead of waiting for a
1374 // pool it belongs to (tensor4all-rs#830). The fresh engine shares the
1375 // plan cache, whose entries are immutable prepared plans, so a stage
1376 // entered per item does not re-plan. It does **not** share the context
1377 // engine's buffer pool, extension caches, batch policy or allocation
1378 // domain: an entry from a worker therefore allocates and batches
1379 // independently, and only while it runs.
1380 if rayon::current_thread_index().is_some() {
1381 let _guard = self.enter_session()?;
1382 let mut engine = CpuEngine::serial().with_plan_cache(self.plans.clone());
1383 return Ok(f(&mut engine));
1384 }
1385 let mut entry = self.begin_session()?;
1386 Ok(f(entry.backend.engine_mut()))
1387 }
1388}
1389
1390/// One held engine session: the re-entry guard and the locked engine state.
1391///
1392/// The guard lives exactly as long as the lock it makes safe, so dropping the
1393/// entry releases the engine before the thread is allowed to enter again.
1394struct EngineEntry<'a> {
1395 state: std::sync::MutexGuard<'a, EngineState>,
1396 _guard: CanonicalSessionGuard,
1397}
1398
1399impl Deref for EngineEntry<'_> {
1400 type Target = EngineState;
1401
1402 fn deref(&self) -> &EngineState {
1403 &self.state
1404 }
1405}
1406
1407impl DerefMut for EngineEntry<'_> {
1408 fn deref_mut(&mut self) -> &mut EngineState {
1409 &mut self.state
1410 }
1411}
1412
1413/// Caller-owned CPU execution domain for plain, graph, and eager-AD work.
1414///
1415/// The context owns one [`CpuBackend`] and lends its engine to one caller at a
1416/// time. A session entered from a Rayon worker runs inline on a fresh serial
1417/// engine that shares the context engine's prepared-plan cache instead of
1418/// waiting for the context engine: a worker that waits for a pool install is
1419/// handed more of the enclosing pool's work, and that work may enter a session
1420/// itself (tensor4all-rs#830). Graph preparation caches, the engine's buffer
1421/// pool and the eager runtime are owned by this context and are released when
1422/// it is dropped.
1423///
1424/// # Examples
1425///
1426/// ```
1427/// use tensor4all_tensorbackend::CpuExecutionContext;
1428/// use tensor4all_tensorbackend::CpuBackend;
1429///
1430/// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1431/// let threads = context.with_backend(|backend| backend.num_threads());
1432/// assert_eq!(threads, 1);
1433/// # Ok::<(), Box<dyn std::error::Error>>(())
1434/// ```
1435pub struct CpuExecutionContext {
1436 owner: Arc<EngineOwner>,
1437 compiler: Mutex<GraphCompiler>,
1438 eager: OnceLock<Result<Arc<EagerRuntime>, CpuExecutionContextError>>,
1439}
1440
1441impl std::fmt::Debug for CpuExecutionContext {
1442 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1443 f.debug_struct("CpuExecutionContext")
1444 .field("graph_initialized", &self.engine_initialized())
1445 .field("eager_initialized", &self.eager.get().is_some())
1446 .finish_non_exhaustive()
1447 }
1448}
1449
1450impl CpuExecutionContext {
1451 /// Create an execution context from a caller-selected CPU backend.
1452 ///
1453 /// Runtime construction is lazy, so creating a context cannot fail and does
1454 /// not allocate another executor or consult environment configuration.
1455 pub fn from_backend(backend: CpuBackend) -> Self {
1456 Self {
1457 owner: Arc::new(EngineOwner::new(backend)),
1458 compiler: Mutex::new(GraphCompiler::new()),
1459 eager: OnceLock::new(),
1460 }
1461 }
1462
1463 fn engine_initialized(&self) -> bool {
1464 self.owner
1465 .engine
1466 .try_lock()
1467 .is_ok_and(|state| state.runtime.is_some())
1468 }
1469
1470 /// Lock the compiler, discarding one a panic left behind.
1471 ///
1472 /// A `GraphCompiler` holds planning state that a panic may have left half
1473 /// written; rebuilding it costs nothing a compile would not pay for the next
1474 /// graph anyway. The poison is cleared after the rebuild so later compiles
1475 /// reuse the fresh compiler instead of rebuilding it on every access.
1476 fn lock_compiler(&self) -> std::sync::MutexGuard<'_, GraphCompiler> {
1477 match self.compiler.lock() {
1478 Ok(guard) => guard,
1479 Err(poisoned) => {
1480 let mut guard = poisoned.into_inner();
1481 *guard = GraphCompiler::new();
1482 self.compiler.clear_poison();
1483 guard
1484 }
1485 }
1486 }
1487
1488 /// Run a plain tensor operation with this context's engine.
1489 ///
1490 /// The closure runs while the context-local engine lock is held. A lock
1491 /// poisoned by a panic is recovered, and the engine is kept: it carries the
1492 /// caller's pool, allocation domain, batch policy and runtime engine id, so
1493 /// a broken engine keeps refusing operations with tenferro's typed
1494 /// `CpuEngineError::Broken` instead of being silently replaced by a default
1495 /// one. The caller builds a new context when it needs a new engine. The
1496 /// session entries recover the same way.
1497 ///
1498 /// This borrows the engine for the whole closure, so a session entry from
1499 /// inside `f` on this thread is rejected with
1500 /// [`CpuExecutionContextError::Reentered`] instead of waiting for the engine.
1501 /// Calling `with_backend` from inside a session of this context is the
1502 /// caller's obligation not to do: this entry cannot report an error, so it
1503 /// would wait for the engine the session already holds.
1504 pub fn with_backend<R>(&self, f: impl FnOnce(&mut CpuBackend) -> R) -> R {
1505 // The guard makes a nested session entry of this context a typed error,
1506 // not a wait; another context's engine stays independently enterable.
1507 let _guard = CanonicalSessionGuard::enter(self.owner.id);
1508 f(&mut self.owner.lock_engine().backend)
1509 }
1510
1511 /// Run `f` in one session of this context's engine.
1512 ///
1513 /// This is the engine-borrow entry for callers that need the concrete
1514 /// engine (for example to hand it to an eager AD session). The context lends
1515 /// its engine to one caller at a time for the whole of `f`.
1516 ///
1517 /// # Errors
1518 ///
1519 /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1520 /// already inside a session of this context.
1521 ///
1522 /// # Examples
1523 ///
1524 /// ```
1525 /// use tensor4all_tensorbackend::{CpuBackend, CpuExecutionContext};
1526 ///
1527 /// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1528 /// let threads = context.with_session(|engine| engine.num_threads())?;
1529 /// assert_eq!(threads, 1);
1530 /// # Ok::<(), Box<dyn std::error::Error>>(())
1531 /// ```
1532 pub fn with_session<R>(
1533 &self,
1534 f: impl FnOnce(&mut CpuEngine) -> R,
1535 ) -> Result<R, CpuExecutionContextError> {
1536 self.owner.with_session(f)
1537 }
1538
1539 /// Compile a backend-neutral traced graph using this context's compiler cache.
1540 ///
1541 /// # Errors
1542 ///
1543 /// Returns [`CpuExecutionContextError::Graph`] when graph compilation fails.
1544 pub fn compile_graph(
1545 &self,
1546 graph: &TracedGraph,
1547 ) -> Result<CompiledGraph, CpuExecutionContextError> {
1548 self.lock_compiler()
1549 .compile_traced_graph(graph)
1550 .map_err(|source| CpuExecutionContextError::graph("compilation", source))
1551 }
1552
1553 /// Execute a compiled graph in this context's runtime and prepared-plan cache.
1554 ///
1555 /// `CompiledGraph` is backend-neutral. Backend-prepared executables and
1556 /// workspaces never leave this context-owned runtime.
1557 ///
1558 /// # Errors
1559 ///
1560 /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1561 /// already inside a session of this context, and
1562 /// [`CpuExecutionContextError::Initialization`], [`CpuExecutionContextError::Graph`]
1563 /// when runtime initialization, preparation, or execution fails.
1564 pub fn run_graph(
1565 &self,
1566 graph: &CompiledGraph,
1567 inputs: &[&Tensor],
1568 ) -> Result<Vec<Tensor>, CpuExecutionContextError> {
1569 let reads = inputs
1570 .iter()
1571 .map(|input| TensorRead::from_tensor(input))
1572 .collect::<Vec<_>>();
1573 self.run_graph_reads(graph, &reads)
1574 }
1575
1576 /// Execute a compiled graph on borrowed tensor views.
1577 ///
1578 /// The read form is what the bridge uses: an operand that is already a
1579 /// borrowed view stays borrowed instead of being copied into an owner first.
1580 ///
1581 /// # Errors
1582 ///
1583 /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1584 /// already inside a session of this context, and
1585 /// [`CpuExecutionContextError::Initialization`], [`CpuExecutionContextError::Graph`]
1586 /// when runtime initialization, preparation, or execution fails.
1587 pub(crate) fn run_graph_reads(
1588 &self,
1589 graph: &CompiledGraph,
1590 inputs: &[TensorRead<'_>],
1591 ) -> Result<Vec<Tensor>, CpuExecutionContextError> {
1592 let mut state = self.owner.begin_session()?;
1593 let EngineState { backend, runtime } = &mut *state;
1594 let runtime = match runtime {
1595 Some(runtime) => &*runtime,
1596 None => &*runtime.insert(build_graph_runtime(backend)?),
1597 };
1598 let target = backend.engine().target();
1599 let prepared = runtime
1600 .prepare(graph, inputs, std::slice::from_ref(&target))
1601 .map_err(|source| CpuExecutionContextError::graph("preparation", source))?;
1602 let outputs = {
1603 let mut engines: [&mut dyn ExecutionEngine; 1] = [backend.engine_mut()];
1604 runtime
1605 .run(&mut engines, &prepared, inputs)
1606 .map_err(|source| CpuExecutionContextError::graph("execution", source))?
1607 };
1608 outputs
1609 .into_iter()
1610 .map(|output| {
1611 output
1612 .into_tensor(backend.engine_mut())
1613 .map_err(|source| CpuExecutionContextError::graph("output", source))
1614 })
1615 .collect()
1616 }
1617
1618 /// Return this context's eager reverse-AD runtime.
1619 ///
1620 /// The runtime is engine-independent: it is built once from an
1621 /// [`AdContext`] with the linalg AD rules and the einsum/linalg extension
1622 /// modules installed under this context engine's runtime id, and every call
1623 /// takes the caller's engine ([`EagerRuntime::with_eager_session`]). Repeated
1624 /// calls return the same runtime and therefore the same eager compilation
1625 /// cache. The runtime registers itself with the eager-runtime owner table once,
1626 /// when it is built.
1627 ///
1628 /// # Errors
1629 ///
1630 /// Returns [`CpuExecutionContextError`] when linalg AD-rule or eager-runtime
1631 /// registration fails.
1632 pub fn eager_runtime(&self) -> Result<Arc<EagerRuntime>, CpuExecutionContextError> {
1633 self.eager
1634 .get_or_init(|| {
1635 let runtime = build_eager_runtime(&self.owner.engine_id)?;
1636 #[cfg(feature = "global-defaults")]
1637 register_eager_runtime_owner(
1638 &runtime,
1639 RuntimeOwner::Cpu(Arc::downgrade(&self.owner)),
1640 );
1641 Ok(runtime)
1642 })
1643 .as_ref()
1644 .map(Arc::clone)
1645 .map_err(Clone::clone)
1646 }
1647
1648 /// Return statistics for this context's runtime-owned graph caches.
1649 ///
1650 /// # Errors
1651 ///
1652 /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1653 /// already inside a session of this context, and
1654 /// [`CpuExecutionContextError`] when graph initialization or the cache
1655 /// statistics query fails.
1656 pub fn graph_cache_stats(
1657 &self,
1658 ) -> Result<tenferro::RuntimeCacheStats, CpuExecutionContextError> {
1659 let mut state = self.owner.begin_session()?;
1660 let runtime = state.runtime()?;
1661 runtime
1662 .cache_stats()
1663 .map_err(|source| CpuExecutionContextError::graph("cache statistics", source))
1664 }
1665
1666 /// Return retained-buffer statistics for this context's engine.
1667 ///
1668 /// `buffers` counts the retained allocations and `capacity_bytes` their
1669 /// total element capacity. A context whose pool has just been released by
1670 /// [`Self::reset_graph_buffer_pool`] reports zero.
1671 ///
1672 /// # Errors
1673 ///
1674 /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1675 /// already inside a session of this context.
1676 ///
1677 /// # Examples
1678 ///
1679 /// ```
1680 /// use tensor4all_tensorbackend::CpuExecutionContext;
1681 /// use tenferro::program::{CoreSemanticOp, ProgramInputSpec};
1682 /// use tenferro::{DType, Tensor, TraceContext};
1683 /// use tensor4all_tensorbackend::CpuBackend;
1684 ///
1685 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1686 /// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1687 ///
1688 /// let trace = TraceContext::new();
1689 /// let input = trace.input(ProgramInputSpec::new(DType::F64, [2_usize.into()]))?;
1690 /// let output = trace.add_op(CoreSemanticOp::Neg, &[input])?[0];
1691 /// let graph = trace.finish(&[output])?;
1692 /// let compiled = context.compile_graph(&graph)?;
1693 /// let values = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, -2.0])?;
1694 /// let output = context.run_graph(&compiled, &[&values])?;
1695 /// assert_eq!(output[0].as_slice::<f64>()?, &[-1.0, 2.0]);
1696 ///
1697 /// // Releasing the pool empties it.
1698 /// context.reset_graph_buffer_pool()?;
1699 /// let after = context.graph_buffer_pool_stats()?;
1700 /// assert_eq!(after.buffers, 0);
1701 /// assert_eq!(after.capacity_bytes, 0);
1702 /// # Ok(())
1703 /// # }
1704 /// ```
1705 pub fn graph_buffer_pool_stats(&self) -> Result<BufferPoolStats, CpuExecutionContextError> {
1706 Ok(self
1707 .owner
1708 .begin_session()?
1709 .backend
1710 .engine()
1711 .buffer_pool_stats())
1712 }
1713
1714 /// Release retained buffers owned by this context's engine.
1715 ///
1716 /// The graph runtime, its compiled plans and its prepared-plan cache are
1717 /// kept, so callers can keep running previously compiled graphs afterwards.
1718 ///
1719 /// # Errors
1720 ///
1721 /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1722 /// already inside a session of this context.
1723 ///
1724 /// # Examples
1725 ///
1726 /// ```
1727 /// use tensor4all_tensorbackend::CpuExecutionContext;
1728 /// use tensor4all_tensorbackend::CpuBackend;
1729 ///
1730 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1731 /// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1732 /// context.reset_graph_buffer_pool()?;
1733 ///
1734 /// // The pool retains nothing afterwards, and the runtime stays usable.
1735 /// let stats = context.graph_buffer_pool_stats()?;
1736 /// assert_eq!(stats.buffers, 0);
1737 /// assert_eq!(stats.capacity_bytes, 0);
1738 /// # Ok(())
1739 /// # }
1740 /// ```
1741 pub fn reset_graph_buffer_pool(&self) -> Result<(), CpuExecutionContextError> {
1742 self.owner
1743 .begin_session()?
1744 .backend
1745 .engine_mut()
1746 .reset_buffer_pool();
1747 Ok(())
1748 }
1749
1750 /// Recreate this context's graph runtime and release its prepared caches.
1751 ///
1752 /// # Errors
1753 ///
1754 /// Returns [`CpuExecutionContextError::Reentered`] when this thread is
1755 /// already inside a session of this context.
1756 pub fn reset_graph_runtime(&self) -> Result<(), CpuExecutionContextError> {
1757 {
1758 let mut state = self.owner.begin_session()?;
1759 state.runtime = None;
1760 state.backend.engine_mut().reset_buffer_pool();
1761 }
1762 *self.lock_compiler() = GraphCompiler::new();
1763 Ok(())
1764 }
1765}
1766
1767fn build_graph_runtime(backend: &CpuBackend) -> Result<Runtime, CpuExecutionContextError> {
1768 // Register and extend the runtime under the engine's *own* runtime id: an
1769 // engine built with `CpuEngine::with_runtime_engine_id` would not be found
1770 // by a preparation targeting the canonical CPU id.
1771 let engine_id = engine_runtime_id(backend.engine());
1772 let mut builder = Runtime::builder();
1773 builder
1774 .register_engine(
1775 tenferro_cpu::runtime_engine_registration_with_id(backend.engine(), engine_id.clone())
1776 .map_err(|source| {
1777 CpuExecutionContextError::initialization("graph CPU engine", source)
1778 })?,
1779 )
1780 .map_err(|source| CpuExecutionContextError::initialization("graph CPU engine", source))?;
1781 builder
1782 .install_extension_module(
1783 tenferro_einsum::extension_module::<CpuEngine>(engine_id).map_err(|source| {
1784 CpuExecutionContextError::initialization("einsum extension", source)
1785 })?,
1786 )
1787 .map_err(|source| CpuExecutionContextError::initialization("einsum extension", source))?;
1788 builder
1789 .build()
1790 .map_err(|source| CpuExecutionContextError::initialization("graph runtime", source))
1791}
1792
1793/// Build an AD context with the built-in linalg semantic rules installed.
1794///
1795/// The context is backend-neutral: CPU and CUDA eager runtimes share the same
1796/// rule set and differ only in the execution extension modules installed on the
1797/// runtime.
1798pub(crate) fn build_ad_context(
1799) -> Result<AdContext, tenferro_ad::semantic_extension::SemanticExtensionRegistryError> {
1800 let rules = tenferro_linalg::semantic_ad_rules()?;
1801 let builder = AdContext::builder().with_semantic_extension_rules(rules)?;
1802 Ok(match builder.build() {
1803 Ok(context) => context,
1804 Err(never) => match never {},
1805 })
1806}
1807
1808fn build_eager_runtime(
1809 engine_id: &tenferro::EngineId,
1810) -> Result<Arc<EagerRuntime>, CpuExecutionContextError> {
1811 let ad_context = build_ad_context()
1812 .map_err(|source| CpuExecutionContextError::initialization("linalg AD rules", source))?;
1813 let runtime = EagerRuntime::with_ad_context(&ad_context)
1814 .map_err(|source| CpuExecutionContextError::initialization("eager runtime", source))?;
1815 // [AI Supplied] Install the built-in extension modules before publishing
1816 // the shared context. Lazy first use reconfigures the runtime and advances
1817 // its epoch; doing that after tensors have prepared AD derivatives can
1818 // invalidate those prepared programs under parallel first use.
1819 //
1820 // The modules are installed under the *engine's own* runtime id: an engine
1821 // built with `CpuEngine::with_runtime_engine_id` would otherwise not find
1822 // its einsum/linalg extensions.
1823 let einsum_module = tenferro_einsum::extension_module::<CpuEngine>(engine_id.clone())
1824 .map_err(|source| CpuExecutionContextError::initialization("einsum extension", source))?;
1825 runtime
1826 .install_extension_module(einsum_module)
1827 .map_err(|source| CpuExecutionContextError::initialization("einsum runtime", source))?;
1828 let linalg_module = tenferro_linalg::extension_module::<CpuEngine>(engine_id.clone())
1829 .map_err(|source| CpuExecutionContextError::initialization("linalg extension", source))?;
1830 runtime
1831 .install_extension_module(linalg_module)
1832 .map_err(|source| CpuExecutionContextError::initialization("linalg runtime", source))?;
1833 Ok(runtime)
1834}
1835
1836/// A context that owns an eager runtime, held weakly by the process-wide table.
1837#[cfg(feature = "global-defaults")]
1838pub(crate) enum RuntimeOwner {
1839 Cpu(Weak<EngineOwner>),
1840 #[cfg(feature = "tenferro-cuda")]
1841 Cuda(Weak<crate::cuda::CudaEngineOwner>),
1842}
1843
1844#[cfg(feature = "global-defaults")]
1845impl RuntimeOwner {
1846 fn upgrade(&self) -> Option<LiveRuntimeOwner> {
1847 match self {
1848 Self::Cpu(owner) => owner.upgrade().map(LiveRuntimeOwner::Cpu),
1849 #[cfg(feature = "tenferro-cuda")]
1850 Self::Cuda(owner) => owner.upgrade().map(LiveRuntimeOwner::Cuda),
1851 }
1852 }
1853
1854 fn is_live(&self) -> bool {
1855 match self {
1856 Self::Cpu(owner) => owner.strong_count() > 0,
1857 #[cfg(feature = "tenferro-cuda")]
1858 Self::Cuda(owner) => owner.strong_count() > 0,
1859 }
1860 }
1861}
1862
1863/// An upgraded [`RuntimeOwner`] that keeps its context alive for one session.
1864#[cfg(feature = "global-defaults")]
1865enum LiveRuntimeOwner {
1866 Cpu(Arc<EngineOwner>),
1867 #[cfg(feature = "tenferro-cuda")]
1868 Cuda(Arc<crate::cuda::CudaEngineOwner>),
1869}
1870
1871/// Process-wide map from an eager runtime's id to the context engine that owns it.
1872///
1873/// The entry holds a [`Weak`] handle: a context registers its runtime when it
1874/// builds it, and the entry dangles when the context (the only strong owner) is
1875/// dropped. Lookups prune the dead entry and fall back to the process-global
1876/// context, so no background cleanup is needed.
1877#[cfg(feature = "global-defaults")]
1878static EAGER_RUNTIME_OWNERS: OnceLock<Mutex<HashMap<ContextId, RuntimeOwner>>> = OnceLock::new();
1879
1880#[cfg(feature = "global-defaults")]
1881fn eager_runtime_owners() -> std::sync::MutexGuard<'static, HashMap<ContextId, RuntimeOwner>> {
1882 EAGER_RUNTIME_OWNERS
1883 .get_or_init(|| Mutex::new(HashMap::new()))
1884 .lock()
1885 .unwrap_or_else(|poisoned| poisoned.into_inner())
1886}
1887
1888#[cfg(feature = "global-defaults")]
1889pub(crate) fn register_eager_runtime_owner(runtime: &EagerRuntime, owner: RuntimeOwner) {
1890 let mut owners = eager_runtime_owners();
1891 owners.retain(|_, owner| owner.is_live());
1892 owners.insert(runtime.id(), owner);
1893}
1894
1895/// Run `f` on the engine of the context that owns `runtime`'s eager runtime.
1896///
1897/// This is the transitional implicit route: an implicit (context-free) eager
1898/// operation has no context argument, so it resolves the engine from the
1899/// runtime of the tensor it acts on instead of always taking the
1900/// process-global default context's engine. That keeps a caller-owned context's
1901/// thread policy, device domain, and engine serialization intact for
1902/// context-free readouts and intermediate operations, including on a context
1903/// that is not the global one.
1904///
1905/// The closure receives the owning engine as [`ExecutionEngine`]: the owner may
1906/// be a CPU context ([`CpuEngine`]) or, with the `tenferro-cuda` feature, a CUDA
1907/// context (`tenferro_gpu::cuda::CudaEngine`). The closure runs through the
1908/// owning context's normal session entry, so a nested entry on the same thread
1909/// is reported as [`ExecutionContextEntryError`]'s typed CPU or CUDA re-entry
1910/// variant instead of waiting for the engine. An entry from a Rayon worker runs
1911/// inline on a serial CPU engine that shares the context's prepared-plan cache;
1912/// a CUDA context keeps its single engine and never opens a second one.
1913/// When no live context owns `runtime` (for example after the context was
1914/// dropped), the entry falls back to the process-global CPU context; the global
1915/// context's own runtime resolves to itself.
1916///
1917/// # Errors
1918///
1919/// Returns [`ExecutionContextEntryError`] when the owning context's session
1920/// entry is rejected.
1921///
1922/// # Examples
1923///
1924/// ```
1925/// use tensor4all_tensorbackend::{with_runtime_engine, CpuBackend, CpuExecutionContext};
1926/// use tenferro_cpu::CpuEngine;
1927///
1928/// let context = CpuExecutionContext::from_backend(CpuBackend::with_threads(1)?);
1929/// let runtime = context.eager_runtime()?;
1930/// let threads = with_runtime_engine(&runtime, |engine| {
1931/// engine
1932/// .as_any_mut()
1933/// .downcast_mut::<CpuEngine>()
1934/// .map(|engine| engine.num_threads())
1935/// })?;
1936/// assert_eq!(threads, Some(1));
1937/// # Ok::<(), Box<dyn std::error::Error>>(())
1938/// ```
1939#[cfg(feature = "global-defaults")]
1940pub fn with_runtime_engine<R>(
1941 runtime: &EagerRuntime,
1942 f: impl FnOnce(&mut dyn ExecutionEngine) -> R,
1943) -> Result<R, ExecutionContextEntryError> {
1944 let owner = {
1945 let mut owners = eager_runtime_owners();
1946 match owners.get(&runtime.id()).and_then(RuntimeOwner::upgrade) {
1947 Some(owner) => owner,
1948 None => {
1949 owners.remove(&runtime.id());
1950 LiveRuntimeOwner::Cpu(Arc::clone(&defaults::default_context().owner))
1951 }
1952 }
1953 };
1954 match owner {
1955 LiveRuntimeOwner::Cpu(owner) => owner
1956 .with_session(|engine| f(engine))
1957 .map_err(ExecutionContextEntryError::Cpu),
1958 #[cfg(feature = "tenferro-cuda")]
1959 LiveRuntimeOwner::Cuda(owner) => owner
1960 .with_execution_engine(f)
1961 .map_err(ExecutionContextEntryError::Cuda),
1962 }
1963}
1964
1965#[cfg(feature = "global-defaults")]
1966mod defaults {
1967 use super::*;
1968
1969 static DEFAULT_CONTEXT: OnceLock<Arc<CpuExecutionContext>> = OnceLock::new();
1970
1971 #[cfg(test)]
1972 thread_local! {
1973 static FORCE_EAGER_CONTEXT_FAILURE: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
1974 }
1975
1976 /// Borrow the process-global CPU execution context.
1977 ///
1978 /// New code must take a caller-owned context; this accessor exists for the
1979 /// process-global convenience path and for callers that map the session-entry
1980 /// rejection into their own error type.
1981 pub(crate) fn default_context() -> &'static Arc<CpuExecutionContext> {
1982 DEFAULT_CONTEXT
1983 .get_or_init(|| Arc::new(CpuExecutionContext::from_backend(CpuBackend::new())))
1984 }
1985
1986 /// Error returned when the process-global eager AD runtime cannot be initialized.
1987 ///
1988 /// # Examples
1989 ///
1990 /// ```
1991 /// use std::error::Error;
1992 /// use std::sync::Arc;
1993 /// use tensor4all_tensorbackend::EagerContextError;
1994 ///
1995 /// let error = EagerContextError::Registration {
1996 /// source: Arc::new(std::io::Error::other("registration failed")),
1997 /// };
1998 /// assert!(error.source().is_some());
1999 /// ```
2000 #[derive(Debug, Clone, thiserror::Error)]
2001 pub enum EagerContextError {
2002 /// The tenferro linalg AD extension rule could not be registered.
2003 #[error("failed to register tenferro linalg AD rule: {source}")]
2004 Registration {
2005 /// Original diagnostic returned by tenferro.
2006 #[source]
2007 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
2008 },
2009 }
2010
2011 /// Run a closure against the optional process-global CPU backend.
2012 pub fn with_default_backend<R>(f: impl FnOnce(&mut CpuBackend) -> R) -> R {
2013 default_context().with_backend(f)
2014 }
2015
2016 /// Run `f` in one session of the process-global context's engine.
2017 ///
2018 /// The entry rejection stays inside tenferro's error type: it is reported as
2019 /// a `tenferro_tensor::Error` classified as runtime state, so callers keep
2020 /// one error type for session work.
2021 ///
2022 /// # Errors
2023 ///
2024 /// Returns the tenferro diagnostic when the entry is rejected or when `f`
2025 /// fails.
2026 pub(crate) fn with_default_session<T: Send>(
2027 f: impl FnOnce(&mut CpuEngine) -> Result<T, tenferro_tensor::Error> + Send,
2028 ) -> Result<T, tenferro_tensor::Error> {
2029 default_context()
2030 .with_session(f)
2031 .map_err(|source| {
2032 tenferro_tensor::Error::runtime_state_source("canonical session entry", source)
2033 })
2034 .and_then(std::convert::identity)
2035 }
2036
2037 /// Run `f` against the process-global graph compiler.
2038 ///
2039 /// The compiler lock is released before the caller runs the compiled graph,
2040 /// so a compile never waits for a context engine that is busy running one.
2041 pub(crate) fn with_default_graph_compiler<R>(f: impl FnOnce(&mut GraphCompiler) -> R) -> R {
2042 f(&mut default_context().lock_compiler())
2043 }
2044
2045 pub(crate) fn default_engine_buffer_pool_stats() -> anyhow::Result<BufferPoolStats> {
2046 default_context()
2047 .graph_buffer_pool_stats()
2048 .map_err(anyhow::Error::new)
2049 }
2050
2051 pub(crate) fn reset_default_engine_buffer_pool() -> anyhow::Result<()> {
2052 default_context()
2053 .reset_graph_buffer_pool()
2054 .map_err(anyhow::Error::new)
2055 }
2056
2057 pub(crate) fn reset_default_engine() -> anyhow::Result<()> {
2058 default_context()
2059 .reset_graph_runtime()
2060 .map_err(anyhow::Error::new)
2061 }
2062
2063 /// Return the optional process-global eager context used by convenience APIs.
2064 ///
2065 /// # Errors
2066 ///
2067 /// Returns [`EagerContextError::Registration`] when eager runtime
2068 /// initialization fails.
2069 ///
2070 /// # Examples
2071 ///
2072 /// ```
2073 /// use std::sync::Arc;
2074 /// use tensor4all_tensorbackend::default_eager_ctx;
2075 ///
2076 /// let first = default_eager_ctx().unwrap();
2077 /// let second = default_eager_ctx().unwrap();
2078 /// assert!(Arc::ptr_eq(&first, &second));
2079 /// ```
2080 pub fn default_eager_ctx() -> Result<Arc<EagerRuntime>, EagerContextError> {
2081 #[cfg(test)]
2082 if FORCE_EAGER_CONTEXT_FAILURE.with(std::cell::Cell::get) {
2083 return Err(EagerContextError::Registration {
2084 source: Arc::new(std::io::Error::other(
2085 "forced default eager context registration failure",
2086 )),
2087 });
2088 }
2089 default_context()
2090 .eager_runtime()
2091 .map_err(|source| EagerContextError::Registration {
2092 source: Arc::new(source),
2093 })
2094 }
2095
2096 /// Borrow the process-global CPU execution context.
2097 ///
2098 /// Compatibility entry for CPU-global convenience APIs (e.g. the legacy
2099 /// context-free SRC entry): host tensors constructed through the global
2100 /// default belong to this exact context, so they validate against it.
2101 /// New code must take a caller-owned context instead of consulting this.
2102 ///
2103 /// # Examples
2104 ///
2105 /// ```
2106 /// use tensor4all_tensorbackend::{default_cpu_execution_context, ExecutionContext};
2107 ///
2108 /// let context = ExecutionContext::Cpu(default_cpu_execution_context());
2109 /// assert!(matches!(context, ExecutionContext::Cpu(_)));
2110 /// ```
2111 pub fn default_cpu_execution_context() -> Arc<CpuExecutionContext> {
2112 Arc::clone(default_context())
2113 }
2114
2115 #[cfg(test)]
2116 pub(crate) fn with_forced_eager_context_failure<T>(f: impl FnOnce() -> T) -> T {
2117 let previous = FORCE_EAGER_CONTEXT_FAILURE.with(|failure| failure.replace(true));
2118 let result = f();
2119 FORCE_EAGER_CONTEXT_FAILURE.with(|failure| failure.set(previous));
2120 result
2121 }
2122}
2123
2124#[cfg(feature = "global-defaults")]
2125pub(crate) use defaults::default_context;
2126#[cfg(all(test, feature = "global-defaults"))]
2127pub(crate) use defaults::with_forced_eager_context_failure;
2128#[cfg(feature = "global-defaults")]
2129pub use defaults::{
2130 default_cpu_execution_context, default_eager_ctx, with_default_backend, EagerContextError,
2131};
2132#[cfg(feature = "global-defaults")]
2133pub(crate) use defaults::{
2134 default_engine_buffer_pool_stats, reset_default_engine, reset_default_engine_buffer_pool,
2135 with_default_graph_compiler, with_default_session,
2136};
2137
2138#[cfg(test)]
2139mod tests;