Skip to main content

tenferro_cpu/
engine.rs

1//! The CPU execution engine: a caller-held value that owns one
2//! [`threadteam::Pool`] (or none) and the workspace every operation on it
3//! writes.
4//!
5//! tenferro never builds threads. The caller builds the pool with
6//! [`Pool::builder`](crate::Pool::builder) and hands it to [`CpuEngine::new`]; a
7//! [`CpuEngine::serial`] engine has no pool at all. Every operation takes the
8//! engine through `&mut`, so one computation uses it at a time without run-time
9//! locks. An operation starts on the calling thread with one placement token
10//! minted from the engine's pool; the kernels split that token and enter the pool
11//! only after a provider has chosen more than one task for it.
12
13use std::num::NonZeroUsize;
14use std::sync::Arc;
15
16use tenferro_tensor::{AllocationDomainId, SharedTensorAllocationDomain};
17
18use crate::buffer_pool::{BufferPool, BufferPoolStats};
19use crate::contraction::ExecutionResources;
20use crate::{CacheStats, CpuPlanCache};
21
22/// Typed refusals reported by a [`CpuEngine`] before an operation runs.
23///
24/// They reach callers as the source of a [`crate::Error::RuntimeState`].
25///
26/// # Examples
27///
28/// ```
29/// use tenferro_cpu::CpuEngineError;
30///
31/// assert!(CpuEngineError::Broken.to_string().contains("panic"));
32/// ```
33#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
34#[non_exhaustive]
35pub enum CpuEngineError {
36    /// An earlier operation panicked while it was using the engine.
37    ///
38    /// The engine's workspace may be half updated, so later operations are
39    /// refused; replace the engine with a new one.
40    #[error("the CPU engine is broken by an earlier panic; build a new engine")]
41    Broken,
42}
43
44/// The serial context [`CpuEngine::with_guard`] lends: the engine's workspace
45/// and batch policy, without a placement token.
46///
47/// It is not a provider bundle and cannot fork. A caller that hands serial work
48/// to a provider builds a budget-one bundle from it with
49/// `cpueinsum::Exec::serial_with_workspace`, which takes no token from the
50/// engine's pool.
51///
52/// # Examples
53///
54/// ```
55/// let mut engine = tenferro_cpu::CpuEngine::serial();
56/// let budget = engine.with_guard(|ctx| {
57///     cpueinsum::Exec::serial_with_workspace(ctx.workspace())
58///         .with_batch_policy(ctx.batch_policy())
59///         .budget()
60/// })?;
61/// assert_eq!(budget, 1);
62/// # Ok::<(), tenferro_tensor::Error>(())
63/// ```
64pub struct SerialCtx<'a> {
65    workspace: &'a cpueinsum::ArenaProvider,
66    policy: crate::CpuBatchPolicy,
67}
68
69impl std::fmt::Debug for SerialCtx<'_> {
70    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
71        f.debug_struct("SerialCtx")
72            .field("policy", &self.policy)
73            .finish_non_exhaustive()
74    }
75}
76
77impl<'a> SerialCtx<'a> {
78    /// The engine's scratch provider, the one its tokens' bundles use.
79    ///
80    /// # Examples
81    ///
82    /// ```
83    /// let mut engine = tenferro_cpu::CpuEngine::serial();
84    /// let has_workspace = engine.with_guard(|ctx| {
85    ///     cpueinsum::Exec::serial_with_workspace(ctx.workspace())
86    ///         .workspace()
87    ///         .is_some()
88    /// })?;
89    /// assert!(has_workspace);
90    /// # Ok::<(), tenferro_tensor::Error>(())
91    /// ```
92    pub fn workspace(&self) -> &'a dyn cpueinsum::WorkspaceProvider {
93        self.workspace
94    }
95
96    /// The engine's batch policy (see [`CpuEngine::batch_policy`]).
97    ///
98    /// # Examples
99    ///
100    /// ```
101    /// let mut engine = tenferro_cpu::CpuEngine::serial();
102    /// let policy = engine.with_guard(|ctx| ctx.batch_policy())?;
103    /// assert_eq!(policy, engine.batch_policy());
104    /// # Ok::<(), tenferro_tensor::Error>(())
105    /// ```
106    pub fn batch_policy(&self) -> crate::CpuBatchPolicy {
107        self.policy
108    }
109}
110
111/// A caller-held CPU execution engine.
112///
113/// The engine owns one [`threadteam::Pool`] (or none for [`CpuEngine::serial`], and
114/// for a caller-built pool of one worker, which it releases at construction) and the
115/// workspace operations write: the buffer pool that recycles outputs, the
116/// contraction scratch, and a handle to a [`CpuPlanCache`]. It implements
117/// [`tenferro_tensor::Engine`], so every operation takes it as `&mut`.
118///
119/// # Examples
120///
121/// ```
122/// use tenferro_cpu::CpuEngine;
123/// use tenferro_tensor::{Tensor, TensorElementwise, TensorRead};
124///
125/// let mut engine = CpuEngine::serial();
126/// let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
127/// let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
128/// let c = engine.add_read(TensorRead::from_tensor(&a), TensorRead::from_tensor(&b))?;
129/// assert_eq!(c.as_slice::<f64>()?, &[4.0, 6.0]);
130/// # Ok::<(), tenferro_tensor::Error>(())
131/// ```
132pub struct CpuEngine {
133    pub(crate) resources: ExecutionResources,
134    pub(crate) buffers: BufferPool,
135    pub(crate) plans: CpuPlanCache,
136    pub(crate) batch_policy: crate::CpuBatchPolicy,
137    pub(crate) allocation_domain: Option<Arc<dyn SharedTensorAllocationDomain>>,
138    runtime_clears: u64,
139    pub(crate) broken: bool,
140    /// Runtime engine id this engine runs prepared programs as.
141    pub(crate) runtime_engine_id: tenferro_runtime::EngineId,
142    /// Extension caches the runtime lends to extension executors during a run.
143    ///
144    /// `None` only while the store is lent out, so a take moves it instead of
145    /// building a placeholder store (#2587).
146    pub(crate) extension_caches: Option<tenferro_runtime::ExtensionCacheStore>,
147}
148
149impl std::fmt::Debug for CpuEngine {
150    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
151        f.debug_struct("CpuEngine")
152            .field("num_threads", &self.num_threads())
153            .field("has_team", &self.has_team())
154            .field("batch_policy", &self.batch_policy)
155            .field("allocation_domain", &self.allocation_domain())
156            .field("broken", &self.broken)
157            .finish_non_exhaustive()
158    }
159}
160
161impl CpuEngine {
162    fn with_resources(resources: ExecutionResources) -> Self {
163        Self {
164            resources,
165            buffers: BufferPool::new(),
166            plans: CpuPlanCache::new(),
167            batch_policy: crate::CpuBatchPolicy::default(),
168            allocation_domain: None,
169            runtime_clears: 0,
170            broken: false,
171            runtime_engine_id: crate::runtime_adapter::default_runtime_engine_id(),
172            extension_caches: Some(tenferro_runtime::ExtensionCacheStore::default()),
173        }
174    }
175
176    /// An engine that computes on `pool`, which it takes over.
177    ///
178    /// Every operation may use all workers of the pool; a caller that wants
179    /// fewer threads builds a smaller pool. [`Pool::builder`](crate::Pool::builder)
180    /// reserves the worker stack the providers need by default.
181    ///
182    /// # Examples
183    ///
184    /// ```
185    /// let pool = tenferro_cpu::Pool::builder().threads(4).build()?;
186    /// let engine = tenferro_cpu::CpuEngine::new(pool);
187    /// assert_eq!(engine.num_threads(), 4);
188    /// # Ok::<(), Box<dyn std::error::Error>>(())
189    /// ```
190    pub fn new(pool: crate::Pool) -> Self {
191        Self::with_resources(ExecutionResources::new(Some(pool)))
192    }
193
194    /// An engine with no pool: every operation runs on the calling thread.
195    ///
196    /// A serial engine never starts or enters a pool, so it is safe to use from
197    /// inside any other pool.
198    ///
199    /// # Examples
200    ///
201    /// ```
202    /// let engine = tenferro_cpu::CpuEngine::serial();
203    /// assert_eq!(engine.num_threads(), 1);
204    /// ```
205    pub fn serial() -> Self {
206        Self::with_resources(ExecutionResources::new(None))
207    }
208
209    /// Use `plans` as this engine's plan cache, shared with every other engine
210    /// that holds a clone of it.
211    ///
212    /// # Examples
213    ///
214    /// ```
215    /// use tenferro_cpu::{CpuEngine, CpuPlanCache};
216    ///
217    /// let plans = CpuPlanCache::new();
218    /// let engine = CpuEngine::serial().with_plan_cache(plans.clone());
219    /// assert!(engine.plan_cache().shares_with(&plans));
220    /// ```
221    #[must_use]
222    pub fn with_plan_cache(mut self, plans: CpuPlanCache) -> Self {
223        self.plans = plans;
224        self
225    }
226
227    /// The plan cache this engine reads plans from.
228    ///
229    /// # Examples
230    ///
231    /// ```
232    /// let engine = tenferro_cpu::CpuEngine::serial();
233    /// assert_eq!(engine.plan_cache().stats().entries, 0);
234    /// ```
235    pub fn plan_cache(&self) -> &CpuPlanCache {
236        &self.plans
237    }
238
239    /// Run prepared programs under a runtime engine id other than the default
240    /// [`crate::runtime_engine_id`]; it must match the id this engine kind was
241    /// registered under.
242    ///
243    /// # Examples
244    ///
245    /// ```
246    /// use tenferro_runtime::EngineId;
247    /// let id = EngineId::new("example.cpu.v1")?;
248    /// let engine = tenferro_cpu::CpuEngine::serial().with_runtime_engine_id(id.clone());
249    /// assert_eq!(tenferro_runtime::ExecutionEngine::target(&engine).engine_id(), &id);
250    /// # Ok::<(), Box<dyn std::error::Error>>(())
251    /// ```
252    #[must_use]
253    pub fn with_runtime_engine_id(mut self, id: tenferro_runtime::EngineId) -> Self {
254        self.runtime_engine_id = id;
255        self
256    }
257
258    /// Bind outputs and managed inputs to a shared allocation domain.
259    ///
260    /// # Examples
261    ///
262    /// ```
263    /// let engine = tenferro_cpu::CpuEngine::serial();
264    /// assert!(engine.allocation_domain().is_none());
265    /// ```
266    #[must_use]
267    pub fn with_allocation_domain(mut self, domain: Arc<dyn SharedTensorAllocationDomain>) -> Self {
268        self.allocation_domain = Some(domain);
269        self
270    }
271
272    /// The shared allocation domain this engine is bound to, if any.
273    ///
274    /// # Examples
275    ///
276    /// ```
277    /// assert_eq!(tenferro_cpu::CpuEngine::serial().allocation_domain(), None);
278    /// ```
279    pub fn allocation_domain(&self) -> Option<AllocationDomainId> {
280        self.allocation_domain.as_ref().map(|domain| domain.id())
281    }
282
283    /// The shared allocator this engine is bound to, if any.
284    ///
285    /// # Examples
286    ///
287    /// ```
288    /// assert!(tenferro_cpu::CpuEngine::serial().shared_allocation_domain().is_none());
289    /// ```
290    pub fn shared_allocation_domain(&self) -> Option<&Arc<dyn SharedTensorAllocationDomain>> {
291        self.allocation_domain.as_ref()
292    }
293
294    /// This engine with a different default batch policy.
295    ///
296    /// # Examples
297    ///
298    /// ```
299    /// use tenferro_cpu::{CpuBatchPolicy, CpuBatchStrategy, CpuEngine};
300    ///
301    /// let engine = CpuEngine::serial()
302    ///     .with_batch_policy(CpuBatchPolicy::new(CpuBatchStrategy::Sequential));
303    /// assert_eq!(engine.batch_policy().strategy, CpuBatchStrategy::Sequential);
304    /// ```
305    #[must_use]
306    pub fn with_batch_policy(mut self, policy: crate::CpuBatchPolicy) -> Self {
307        self.batch_policy = policy;
308        self
309    }
310
311    /// Replace the default batch policy, returning the previous one.
312    ///
313    /// For a one-operation override use [`crate::with_batch_policy`].
314    ///
315    /// # Examples
316    ///
317    /// ```
318    /// use tenferro_cpu::{CpuBatchPolicy, CpuBatchStrategy, CpuEngine};
319    ///
320    /// let mut engine = CpuEngine::serial();
321    /// let previous = engine.set_batch_policy(CpuBatchPolicy::new(CpuBatchStrategy::Outer));
322    /// assert_eq!(previous.strategy, CpuBatchStrategy::Auto);
323    /// ```
324    pub fn set_batch_policy(&mut self, policy: crate::CpuBatchPolicy) -> crate::CpuBatchPolicy {
325        std::mem::replace(&mut self.batch_policy, policy)
326    }
327
328    /// The default batch policy of this engine.
329    ///
330    /// # Examples
331    ///
332    /// ```
333    /// use tenferro_cpu::{CpuBatchStrategy, CpuEngine};
334    /// assert_eq!(CpuEngine::serial().batch_policy().strategy, CpuBatchStrategy::Auto);
335    /// ```
336    #[must_use]
337    pub fn batch_policy(&self) -> crate::CpuBatchPolicy {
338        self.batch_policy
339    }
340
341    /// The number of workers operations may use: the pool size, or one.
342    ///
343    /// # Examples
344    ///
345    /// ```
346    /// assert_eq!(tenferro_cpu::CpuEngine::serial().num_threads(), 1);
347    /// ```
348    pub fn num_threads(&self) -> usize {
349        self.thread_budget().get()
350    }
351
352    /// Whether the engine runs its parallel work on a reserved
353    /// `threadteam::Team`.
354    ///
355    /// `true` for an engine built from a pool of two or more workers whose
356    /// members were reserved. `false` for [`CpuEngine::serial`], for a
357    /// one-worker pool, and for a pool whose members could not be reserved
358    /// within about ten seconds of construction (the engine then runs serially
359    /// and [`CpuEngine::num_threads`] reports one). Benchmarks should assert
360    /// this after construction.
361    ///
362    /// # Examples
363    ///
364    /// ```
365    /// assert!(!tenferro_cpu::CpuEngine::serial().has_team());
366    /// let pool = tenferro_cpu::Pool::builder().threads(2).build()?;
367    /// assert!(tenferro_cpu::CpuEngine::new(pool).has_team());
368    /// # Ok::<(), Box<dyn std::error::Error>>(())
369    /// ```
370    pub fn has_team(&self) -> bool {
371        self.resources.has_team()
372    }
373
374    pub(crate) fn thread_budget(&self) -> NonZeroUsize {
375        self.resources.thread_budget()
376    }
377
378    /// The dispatch costs the engine's tokens use: those of its pool, or
379    /// [`threadteam::DispatchCosts::host_default`] for [`CpuEngine::serial`].
380    ///
381    /// Use them with [`CpuEngine::num_threads`] to decide, before taking a token
382    /// with [`CpuEngine::exec`], whether an operation would fork and at what
383    /// width. A token of an engine with a team ([`CpuEngine::has_team`]) charges
384    /// [`threadteam::DispatchCosts::cost`] of [`threadteam::DispatchKind::Team`]
385    /// for a section of two or more participants, so
386    /// [`threadteam::width_for_by`] and [`threadteam::pays_at`] over that cost
387    /// give the answer its `width_for` and `pays` would. A budget of one never
388    /// forks.
389    ///
390    /// # Examples
391    ///
392    /// ```
393    /// use std::num::NonZeroUsize;
394    /// use std::time::Duration;
395    /// use tenferro_cpu::threadteam::{width_for_by, DispatchKind};
396    ///
397    /// let pool = tenferro_cpu::Pool::builder().threads(2).build()?;
398    /// let engine = tenferro_cpu::CpuEngine::new(pool);
399    /// let budget = NonZeroUsize::new(engine.num_threads()).unwrap();
400    /// let cost = |width| engine.dispatch_costs().cost(DispatchKind::Team, width);
401    ///
402    /// // A nanosecond of work stays serial; a second of work uses the budget.
403    /// assert_eq!(width_for_by(Duration::from_nanos(1), 1, budget, cost).get(), 1);
404    /// assert_eq!(
405    ///     width_for_by(Duration::from_secs(1), 1, budget, cost).get(),
406    ///     engine.num_threads(),
407    /// );
408    /// # Ok::<(), Box<dyn std::error::Error>>(())
409    /// ```
410    pub fn dispatch_costs(&self) -> &threadteam::DispatchCosts {
411        self.resources.dispatch_costs()
412    }
413
414    /// Whether an earlier operation broke this engine (see
415    /// [`CpuEngineError::Broken`]).
416    ///
417    /// # Examples
418    ///
419    /// ```
420    /// assert!(!tenferro_cpu::CpuEngine::serial().is_broken());
421    /// ```
422    pub fn is_broken(&self) -> bool {
423        self.broken
424    }
425
426    /// The provider execution bundle for this engine: a placement token over
427    /// every worker of its pool, its workspace and its batch policy.
428    ///
429    /// A downstream kernel that calls a provider (cpueinsum, tcontract,
430    /// tlinalg) directly passes this bundle by `&mut`, so it uses the same
431    /// threads, budget and batch policy as tenferro. The bundle is not `Clone`:
432    /// it is the engine's whole thread budget for that one call, and the provider
433    /// splits it. Getting it applies the same checks as an operation.
434    ///
435    /// # Errors
436    ///
437    /// Returns [`crate::Error::RuntimeState`] with a [`CpuEngineError`] source
438    /// when the engine is broken.
439    ///
440    /// # Examples
441    ///
442    /// ```
443    /// let mut engine = tenferro_cpu::CpuEngine::serial();
444    /// let exec = engine.exec()?;
445    /// assert_eq!(exec.budget(), 1);
446    /// # Ok::<(), tenferro_tensor::Error>(())
447    /// ```
448    pub fn exec(&mut self) -> crate::Result<cpueinsum::Exec<'_>> {
449        self.check_entry("CpuEngine::exec")?;
450        Ok(self.resources.exec(self.batch_policy))
451    }
452
453    /// Run `op` on the engine's provider bundle, keeping the engine's guarantees.
454    ///
455    /// `op` receives the same bundle an operation mints (a placement token over
456    /// every worker of the pool, the workspace and the batch policy; see
457    /// [`CpuEngine::exec`]) and runs on the calling thread. Unlike
458    /// [`CpuEngine::exec`], the call is an engine operation: a panic inside `op`
459    /// marks the engine broken, and contraction scratch retained above
460    /// [`CpuEngine::buffer_pool_limit_bytes`] is released when `op` returns.
461    ///
462    /// The bundle is borrowed for the call only, and the engine stays mutably
463    /// borrowed while `op` runs, so neither can be stored or re-entered from
464    /// inside `op`; the compiler rejects both:
465    ///
466    /// ```compile_fail,E0501
467    /// let mut engine = tenferro_cpu::CpuEngine::serial();
468    /// let engine_ref = &mut engine;
469    /// engine_ref.with_exec(|_| engine_ref.with_exec(|exec| exec.budget()))?;
470    /// # Ok::<(), tenferro_tensor::Error>(())
471    /// ```
472    ///
473    /// ```compile_fail,E0521
474    /// let mut engine = tenferro_cpu::CpuEngine::serial();
475    /// let mut kept = None;
476    /// engine.with_exec(|exec| kept = Some(exec))?;
477    /// # Ok::<(), tenferro_tensor::Error>(())
478    /// ```
479    ///
480    /// # Errors
481    ///
482    /// Returns [`crate::Error::RuntimeState`] with a [`CpuEngineError::Broken`]
483    /// source, without calling `op`, when an earlier operation panicked.
484    ///
485    /// # Panics
486    ///
487    /// Resumes a panic raised by `op` after marking the engine broken.
488    ///
489    /// # Examples
490    ///
491    /// ```
492    /// let mut engine = tenferro_cpu::CpuEngine::serial();
493    /// let budget = engine.with_exec(|exec| exec.budget())?;
494    /// assert_eq!(budget, 1);
495    /// # Ok::<(), tenferro_tensor::Error>(())
496    /// ```
497    ///
498    /// A panic inside `op` breaks the engine, and a broken engine refuses later
499    /// calls with an error:
500    ///
501    /// ```
502    /// use tenferro_cpu::{CpuEngine, CpuEngineError};
503    ///
504    /// let mut engine = CpuEngine::serial();
505    /// let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
506    ///     engine.with_exec(|_| panic!("a failing block"))
507    /// }));
508    /// assert!(unwind.is_err());
509    /// assert!(engine.is_broken());
510    ///
511    /// let refused = engine.with_exec(|exec| exec.budget()).unwrap_err();
512    /// let source = std::error::Error::source(&refused).and_then(|s| s.downcast_ref());
513    /// assert_eq!(source, Some(&CpuEngineError::Broken));
514    /// ```
515    pub fn with_exec<R>(
516        &mut self,
517        op: impl FnOnce(&mut cpueinsum::Exec<'_>) -> R,
518    ) -> crate::Result<R> {
519        self.enter("CpuEngine::with_exec", true, |exec, _, _| Ok(op(exec)))
520    }
521
522    /// Borrow the provider bundle, the buffer pool and the N-ary contraction
523    /// scratch for one downstream call.
524    ///
525    /// # Errors
526    ///
527    /// Returns the entry refusals of [`CpuEngine::exec`], or `op`'s own error.
528    ///
529    /// # Examples
530    ///
531    /// ```
532    /// let mut engine = tenferro_cpu::CpuEngine::serial();
533    /// let len = engine.with_contraction_exec(|_, buffers, workspaces| {
534    ///     workspaces.with_scratch::<f64, _>(4, buffers.max_retained_capacity_bytes(), |scratch| {
535    ///         Ok(scratch.len())
536    ///     })
537    /// })?;
538    /// assert_eq!(len, 4);
539    /// # Ok::<(), tenferro_tensor::Error>(())
540    /// ```
541    #[doc(hidden)]
542    pub fn with_contraction_exec<R>(
543        &mut self,
544        op: impl FnOnce(
545            &mut cpueinsum::Exec<'_>,
546            &mut BufferPool,
547            &crate::ContractionWorkspaces,
548        ) -> crate::Result<R>,
549    ) -> crate::Result<R> {
550        self.enter("CpuEngine::with_contraction_exec", true, op)
551    }
552
553    /// Borrow the provider bundle and buffer pool for a linalg provider call.
554    ///
555    /// The provider chooses its own execution through the bundle; this
556    /// callback itself runs on the calling thread.
557    ///
558    /// # Errors
559    ///
560    /// Returns the entry refusals of [`CpuEngine::exec`], or `op`'s own error.
561    ///
562    /// # Examples
563    ///
564    /// ```
565    /// use tenferro_cpu::{CpuBatchStrategy, CpuEngine};
566    ///
567    /// let mut engine = CpuEngine::serial();
568    /// let strategy = engine.with_linalg_pool(|exec, _| Ok(exec.batch_policy().strategy))?;
569    /// assert_eq!(strategy, CpuBatchStrategy::Auto);
570    /// # Ok::<(), tenferro_tensor::Error>(())
571    /// ```
572    #[doc(hidden)]
573    pub fn with_linalg_pool<R>(
574        &mut self,
575        op: impl FnOnce(&mut cpueinsum::Exec<'_>, &mut BufferPool) -> crate::Result<R>,
576    ) -> crate::Result<R> {
577        self.enter("CpuEngine::with_linalg_pool", false, |exec, buffers, _| {
578            op(exec, buffers)
579        })
580    }
581
582    /// Run `op` on the calling thread with the engine's guarantees, but without
583    /// taking a placement token: the guarded entry of [`CpuEngine::with_exec`]
584    /// at width one.
585    ///
586    /// `op` receives a [`SerialCtx`] with the engine's workspace and batch
587    /// policy. It is not a provider bundle: it carries no token, so nothing in
588    /// it can fork, and getting it reserves none of the pool's workers. Apart
589    /// from the token, the call is the same engine operation as
590    /// [`CpuEngine::with_exec`]: a broken engine is refused, a panic inside `op`
591    /// marks the engine broken (buffers `op` had taken from the buffer pool are
592    /// not returned to it, since a broken engine runs nothing again), and
593    /// contraction scratch retained above [`CpuEngine::buffer_pool_limit_bytes`]
594    /// is released when `op` returns or unwinds.
595    ///
596    /// The context is borrowed for the call only, and the engine stays mutably
597    /// borrowed while `op` runs, so neither can be stored or re-entered from
598    /// inside `op`; the compiler rejects both:
599    ///
600    /// ```compile_fail,E0501
601    /// let mut engine = tenferro_cpu::CpuEngine::serial();
602    /// let engine_ref = &mut engine;
603    /// engine_ref.with_guard(|_| engine_ref.with_guard(|ctx| ctx.batch_policy()))?;
604    /// # Ok::<(), tenferro_tensor::Error>(())
605    /// ```
606    ///
607    /// ```compile_fail,E0521
608    /// let mut engine = tenferro_cpu::CpuEngine::serial();
609    /// let mut kept = None;
610    /// engine.with_guard(|ctx| kept = Some(ctx))?;
611    /// # Ok::<(), tenferro_tensor::Error>(())
612    /// ```
613    ///
614    /// # Errors
615    ///
616    /// Returns [`crate::Error::RuntimeState`] with a [`CpuEngineError::Broken`]
617    /// source, without calling `op`, when an earlier operation panicked.
618    ///
619    /// # Panics
620    ///
621    /// Resumes a panic raised by `op` after marking the engine broken.
622    ///
623    /// # Examples
624    ///
625    /// ```
626    /// use tenferro_cpu::{CpuBatchPolicy, CpuBatchStrategy, CpuEngine};
627    ///
628    /// let mut engine = CpuEngine::new(tenferro_cpu::Pool::builder().threads(2).build()?)
629    ///     .with_batch_policy(CpuBatchPolicy::new(CpuBatchStrategy::Outer));
630    /// let strategy = engine.with_guard(|ctx| ctx.batch_policy().strategy)?;
631    /// assert_eq!(strategy, CpuBatchStrategy::Outer);
632    /// # Ok::<(), Box<dyn std::error::Error>>(())
633    /// ```
634    ///
635    /// A panic inside `op` breaks the engine, and a broken engine refuses later
636    /// calls with an error:
637    ///
638    /// ```
639    /// use tenferro_cpu::{CpuEngine, CpuEngineError};
640    ///
641    /// let mut engine = CpuEngine::serial();
642    /// let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
643    ///     engine.with_guard(|_| panic!("a failing serial phase"))
644    /// }));
645    /// assert!(unwind.is_err());
646    /// assert!(engine.is_broken());
647    ///
648    /// let refused = engine.with_guard(|_| ()).unwrap_err();
649    /// let source = std::error::Error::source(&refused).and_then(|s| s.downcast_ref());
650    /// assert_eq!(source, Some(&CpuEngineError::Broken));
651    /// ```
652    pub fn with_guard<R>(&mut self, op: impl FnOnce(SerialCtx<'_>) -> R) -> crate::Result<R> {
653        self.guarded("CpuEngine::with_guard", true, |resources, policy, _| {
654            Ok(op(SerialCtx {
655                workspace: resources.workspace(),
656                policy,
657            }))
658        })
659    }
660
661    /// The token-minting entry behind the closure-scoped entries: [`Self::guarded`]
662    /// with the bundle and the N-ary scratch lent to `op`.
663    fn enter<R>(
664        &mut self,
665        name: &'static str,
666        bound_retention: bool,
667        op: impl FnOnce(
668            &mut cpueinsum::Exec<'_>,
669            &mut BufferPool,
670            &crate::ContractionWorkspaces,
671        ) -> crate::Result<R>,
672    ) -> crate::Result<R> {
673        self.guarded(name, bound_retention, |resources, policy, buffers| {
674            let mut exec = resources.exec(policy);
675            op(&mut exec, buffers, &resources.nary)
676        })
677    }
678
679    /// The one guarded entry: refuse a broken engine, lend the resources, the
680    /// batch policy and the buffer pool, mark the engine broken if `op`
681    /// unwinds, and with `bound_retention` release workspace retained above the
682    /// buffer limit afterwards.
683    fn guarded<R>(
684        &mut self,
685        name: &'static str,
686        bound_retention: bool,
687        op: impl FnOnce(&ExecutionResources, crate::CpuBatchPolicy, &mut BufferPool) -> crate::Result<R>,
688    ) -> crate::Result<R> {
689        self.check_entry(name)?;
690        let _retention = bound_retention.then(|| {
691            self.resources
692                .retention_guard(self.buffers.max_retained_capacity_bytes())
693        });
694        let (resources, policy) = (&self.resources, self.batch_policy);
695        guard_unwind(&mut self.buffers, &mut self.broken, |buffers| {
696            op(resources, policy, buffers)
697        })
698    }
699
700    /// Refuse an operation that cannot run on this engine now.
701    pub(crate) fn check_entry(&self, op: &'static str) -> crate::Result<()> {
702        if self.broken {
703            return Err(crate::Error::runtime_state_source(
704                op,
705                CpuEngineError::Broken,
706            ));
707        }
708        Ok(())
709    }
710
711    /// Ledger roots of every placement token the engine minted since the last
712    /// call, so a test can read `threadteam::ledger` for the operation it just
713    /// ran (`max_live`, `installs`).
714    ///
715    /// Available with the `ledger` cargo feature, which is test support.
716    #[cfg(feature = "ledger")]
717    #[doc(hidden)]
718    pub fn take_ledger_roots(&self) -> Vec<u64> {
719        self.resources.take_ledger_roots()
720    }
721
722    /// Pool entries (installs from a plain thread or a foreign pool) made by
723    /// the tokens minted since the last `take_ledger_*` call, which it consumes.
724    ///
725    /// Zero means every operation since then stayed on the calling thread.
726    /// Available with the `ledger` cargo feature, which is test support.
727    #[cfg(feature = "ledger")]
728    #[doc(hidden)]
729    pub fn take_ledger_installs(&self) -> usize {
730        self.take_ledger_roots()
731            .into_iter()
732            .map(threadteam::ledger::installs)
733            .sum()
734    }
735
736    /// Largest number of tasks live at once under any token minted since the
737    /// last `take_ledger_*` call, which it consumes. It never exceeds
738    /// [`CpuEngine::num_threads`].
739    ///
740    /// Available with the `ledger` cargo feature, which is test support.
741    #[cfg(feature = "ledger")]
742    #[doc(hidden)]
743    pub fn take_ledger_max_live(&self) -> usize {
744        self.take_ledger_roots()
745            .into_iter()
746            .map(threadteam::ledger::max_live)
747            .max()
748            .unwrap_or(0)
749    }
750
751    /// Number of retained buffers in the engine's buffer pool.
752    ///
753    /// # Examples
754    ///
755    /// ```
756    /// assert_eq!(tenferro_cpu::CpuEngine::serial().buffer_pool_len(), 0);
757    /// ```
758    pub fn buffer_pool_len(&self) -> usize {
759        self.buffers.len()
760    }
761
762    /// Retained-buffer statistics of the engine's buffer pool.
763    ///
764    /// # Examples
765    ///
766    /// ```
767    /// assert_eq!(tenferro_cpu::CpuEngine::serial().buffer_pool_stats().buffers, 0);
768    /// ```
769    pub fn buffer_pool_stats(&self) -> BufferPoolStats {
770        self.buffers.stats()
771    }
772
773    /// The buffer pool as cache statistics (entries and retained bytes).
774    ///
775    /// # Examples
776    ///
777    /// ```
778    /// assert_eq!(tenferro_cpu::CpuEngine::serial().buffer_pool_cache_stats().entries, 0);
779    /// ```
780    pub fn buffer_pool_cache_stats(&self) -> CacheStats {
781        let stats = self.buffers.stats();
782        CacheStats {
783            entries: stats.buffers,
784            retained_bytes: stats.capacity_bytes,
785            ..CacheStats::default()
786        }
787    }
788
789    /// The retention limit of recycled buffers, in retained capacity bytes.
790    ///
791    /// # Examples
792    ///
793    /// ```
794    /// let mut engine = tenferro_cpu::CpuEngine::serial();
795    /// engine.set_buffer_pool_limit_bytes(0);
796    /// assert_eq!(engine.buffer_pool_limit_bytes(), 0);
797    /// ```
798    pub fn buffer_pool_limit_bytes(&self) -> usize {
799        self.buffers.max_retained_capacity_bytes()
800    }
801
802    /// Bound the bytes of recycled buffers and contraction scratch retained
803    /// between operations; zero disables retention.
804    ///
805    /// Destinations larger than the retention limit cannot be recycled, so
806    /// repeated calls allocate again and may pay first-touch costs. The default
807    /// is 100 MiB; raise it when a run keeps producing outputs above that size.
808    ///
809    /// # Examples
810    ///
811    /// ```
812    /// let mut engine = tenferro_cpu::CpuEngine::serial();
813    /// engine.set_buffer_pool_limit_bytes(1 << 20);
814    /// assert_eq!(engine.buffer_pool_limit_bytes(), 1 << 20);
815    /// ```
816    pub fn set_buffer_pool_limit_bytes(&mut self, max_retained_capacity_bytes: usize) {
817        self.buffers
818            .set_max_retained_capacity_bytes(max_retained_capacity_bytes);
819        self.resources.nary.trim_to(max_retained_capacity_bytes);
820        if self.resources.retained_bytes() > max_retained_capacity_bytes {
821            self.resources.trim();
822        }
823    }
824
825    /// Drop every retained buffer and contraction scratch.
826    ///
827    /// # Examples
828    ///
829    /// ```
830    /// let mut engine = tenferro_cpu::CpuEngine::serial();
831    /// engine.reset_buffer_pool();
832    /// assert_eq!(engine.buffer_pool_len(), 0);
833    /// ```
834    pub fn reset_buffer_pool(&mut self) {
835        self.buffers.clear();
836        self.resources.nary.clear_all();
837        self.resources.trim();
838    }
839
840    /// Aggregate statistics of everything this engine retains between
841    /// operations: buffers, contraction scratch, its plan cache and its
842    /// extension cache store.
843    ///
844    /// `entries` and `retained_bytes` cover all of them. `hits`, `misses` and
845    /// `evictions` are the plan cache's alone; the extension store reports its
846    /// own through [`ExtensionCacheStore::stats`](tenferro_runtime::ExtensionCacheStore::stats).
847    /// While a run has the extension store lent out, it is not counted.
848    ///
849    /// # Examples
850    ///
851    /// ```
852    /// assert_eq!(tenferro_cpu::CpuEngine::serial().runtime_cache_stats().entries, 0);
853    /// ```
854    #[doc(hidden)]
855    pub fn runtime_cache_stats(&self) -> CacheStats {
856        let (workspace_entries, nary_bytes) = self.resources.nary.total_stats();
857        let workspace_bytes = nary_bytes.saturating_add(self.resources.retained_bytes());
858        let buffers = self.buffers.cache_stats();
859        let plans = self.plans.stats();
860        let (extension_entries, extension_bytes) =
861            self.extension_caches.as_ref().map_or((0, 0), |store| {
862                let stats = store.stats(tenferro_runtime::ExtensionCacheSelector::All);
863                (stats.entries, stats.retained_bytes)
864            });
865        CacheStats {
866            entries: buffers
867                .entries
868                .saturating_add(plans.entries)
869                .saturating_add(workspace_entries)
870                .saturating_add(extension_entries),
871            retained_bytes: buffers
872                .retained_bytes
873                .saturating_add(plans.retained_bytes)
874                .saturating_add(workspace_bytes)
875                .saturating_add(extension_bytes),
876            hits: plans.hits,
877            misses: plans.misses,
878            evictions: plans.evictions,
879            clears: self.runtime_clears,
880        }
881    }
882
883    /// Drop everything this engine retains between operations: buffers,
884    /// contraction scratch, its extension cache store and its plan cache (and
885    /// therefore the plans of every engine sharing it).
886    ///
887    /// # Examples
888    ///
889    /// ```
890    /// let mut engine = tenferro_cpu::CpuEngine::serial();
891    /// engine.clear_runtime_caches();
892    /// assert_eq!(engine.runtime_cache_stats().clears, 1);
893    /// ```
894    #[doc(hidden)]
895    pub fn clear_runtime_caches(&mut self) {
896        self.reset_buffer_pool();
897        self.plans.clear();
898        if let Some(store) = self.extension_caches.as_mut() {
899            store.clear();
900        }
901        self.runtime_clears = self.runtime_clears.saturating_add(1);
902    }
903}
904
905/// Run one operation's use of the buffer pool, recovering the pool's
906/// accounting and marking the engine broken if it unwinds.
907///
908/// Buffers checked out during the operation are counted as in flight. A normal
909/// exit forgets that accounting (the buffers left as outputs); an unwind
910/// replenishes the pool with replacements for the buffers it lost and marks the
911/// engine broken, since other workspace may be half updated.
912pub(crate) fn guard_unwind<R>(
913    buffers: &mut BufferPool,
914    broken: &mut bool,
915    op: impl FnOnce(&mut BufferPool) -> R,
916) -> R {
917    match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| op(&mut *buffers))) {
918        Ok(value) => value,
919        Err(payload) => {
920            // The engine is never used again after a panic, so the buffers the
921            // operation had checked out need no accounting repair.
922            *broken = true;
923            std::panic::resume_unwind(payload)
924        }
925    }
926}
927
928impl CpuEngine {
929    /// Split borrows for the operation helpers: the provider bundle minted for
930    /// this operation (it borrows only the pool resources), the buffer pool, the
931    /// plan cache and the broken flag.
932    pub(crate) fn parts(
933        &mut self,
934    ) -> (
935        cpueinsum::Exec<'_>,
936        &mut BufferPool,
937        &CpuPlanCache,
938        &mut bool,
939    ) {
940        (
941            self.resources.exec(self.batch_policy),
942            &mut self.buffers,
943            &self.plans,
944            &mut self.broken,
945        )
946    }
947}
948
949#[cfg(test)]
950mod tests;