tenferro_cpu/engine.rs
1//! The CPU execution engine: a caller-held value that owns one
2//! [`threadteam::Pool`] (or none) and the workspace every operation on it
3//! writes.
4//!
5//! tenferro never builds threads. The caller builds the pool with
6//! [`Pool::builder`](crate::Pool::builder) and hands it to [`CpuEngine::new`]; a
7//! [`CpuEngine::serial`] engine has no pool at all. Every operation takes the
8//! engine through `&mut`, so one computation uses it at a time without run-time
9//! locks. An operation starts on the calling thread with one placement token
10//! minted from the engine's pool; the kernels split that token and enter the pool
11//! only after a provider has chosen more than one task for it.
12
13use std::num::NonZeroUsize;
14use std::sync::Arc;
15
16use tenferro_tensor::{AllocationDomainId, SharedTensorAllocationDomain};
17
18use crate::buffer_pool::{BufferPool, BufferPoolStats};
19use crate::contraction::ExecutionResources;
20use crate::{CacheStats, CpuPlanCache};
21
22/// Typed refusals reported by a [`CpuEngine`] before an operation runs.
23///
24/// They reach callers as the source of a [`crate::Error::RuntimeState`].
25///
26/// # Examples
27///
28/// ```
29/// use tenferro_cpu::CpuEngineError;
30///
31/// assert!(CpuEngineError::Broken.to_string().contains("panic"));
32/// ```
33#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
34#[non_exhaustive]
35pub enum CpuEngineError {
36 /// An earlier operation panicked while it was using the engine.
37 ///
38 /// The engine's workspace may be half updated, so later operations are
39 /// refused; replace the engine with a new one.
40 #[error("the CPU engine is broken by an earlier panic; build a new engine")]
41 Broken,
42}
43
44/// The serial context [`CpuEngine::with_guard`] lends: the engine's workspace
45/// and batch policy, without a placement token.
46///
47/// It is not a provider bundle and cannot fork. A caller that hands serial work
48/// to a provider builds a budget-one bundle from it with
49/// `cpueinsum::Exec::serial_with_workspace`, which takes no token from the
50/// engine's pool.
51///
52/// # Examples
53///
54/// ```
55/// let mut engine = tenferro_cpu::CpuEngine::serial();
56/// let budget = engine.with_guard(|ctx| {
57/// cpueinsum::Exec::serial_with_workspace(ctx.workspace())
58/// .with_batch_policy(ctx.batch_policy())
59/// .budget()
60/// })?;
61/// assert_eq!(budget, 1);
62/// # Ok::<(), tenferro_tensor::Error>(())
63/// ```
64pub struct SerialCtx<'a> {
65 workspace: &'a cpueinsum::ArenaProvider,
66 policy: crate::CpuBatchPolicy,
67}
68
69impl std::fmt::Debug for SerialCtx<'_> {
70 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
71 f.debug_struct("SerialCtx")
72 .field("policy", &self.policy)
73 .finish_non_exhaustive()
74 }
75}
76
77impl<'a> SerialCtx<'a> {
78 /// The engine's scratch provider, the one its tokens' bundles use.
79 ///
80 /// # Examples
81 ///
82 /// ```
83 /// let mut engine = tenferro_cpu::CpuEngine::serial();
84 /// let has_workspace = engine.with_guard(|ctx| {
85 /// cpueinsum::Exec::serial_with_workspace(ctx.workspace())
86 /// .workspace()
87 /// .is_some()
88 /// })?;
89 /// assert!(has_workspace);
90 /// # Ok::<(), tenferro_tensor::Error>(())
91 /// ```
92 pub fn workspace(&self) -> &'a dyn cpueinsum::WorkspaceProvider {
93 self.workspace
94 }
95
96 /// The engine's batch policy (see [`CpuEngine::batch_policy`]).
97 ///
98 /// # Examples
99 ///
100 /// ```
101 /// let mut engine = tenferro_cpu::CpuEngine::serial();
102 /// let policy = engine.with_guard(|ctx| ctx.batch_policy())?;
103 /// assert_eq!(policy, engine.batch_policy());
104 /// # Ok::<(), tenferro_tensor::Error>(())
105 /// ```
106 pub fn batch_policy(&self) -> crate::CpuBatchPolicy {
107 self.policy
108 }
109}
110
111/// A caller-held CPU execution engine.
112///
113/// The engine owns one [`threadteam::Pool`] (or none for [`CpuEngine::serial`], and
114/// for a caller-built pool of one worker, which it releases at construction) and the
115/// workspace operations write: the buffer pool that recycles outputs, the
116/// contraction scratch, and a handle to a [`CpuPlanCache`]. It implements
117/// [`tenferro_tensor::Engine`], so every operation takes it as `&mut`.
118///
119/// # Examples
120///
121/// ```
122/// use tenferro_cpu::CpuEngine;
123/// use tenferro_tensor::{Tensor, TensorElementwise, TensorRead};
124///
125/// let mut engine = CpuEngine::serial();
126/// let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
127/// let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
128/// let c = engine.add_read(TensorRead::from_tensor(&a), TensorRead::from_tensor(&b))?;
129/// assert_eq!(c.as_slice::<f64>()?, &[4.0, 6.0]);
130/// # Ok::<(), tenferro_tensor::Error>(())
131/// ```
132pub struct CpuEngine {
133 pub(crate) resources: ExecutionResources,
134 pub(crate) buffers: BufferPool,
135 pub(crate) plans: CpuPlanCache,
136 pub(crate) batch_policy: crate::CpuBatchPolicy,
137 pub(crate) allocation_domain: Option<Arc<dyn SharedTensorAllocationDomain>>,
138 runtime_clears: u64,
139 pub(crate) broken: bool,
140 /// Runtime engine id this engine runs prepared programs as.
141 pub(crate) runtime_engine_id: tenferro_runtime::EngineId,
142 /// Extension caches the runtime lends to extension executors during a run.
143 ///
144 /// `None` only while the store is lent out, so a take moves it instead of
145 /// building a placeholder store (#2587).
146 pub(crate) extension_caches: Option<tenferro_runtime::ExtensionCacheStore>,
147}
148
149impl std::fmt::Debug for CpuEngine {
150 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
151 f.debug_struct("CpuEngine")
152 .field("num_threads", &self.num_threads())
153 .field("has_team", &self.has_team())
154 .field("batch_policy", &self.batch_policy)
155 .field("allocation_domain", &self.allocation_domain())
156 .field("broken", &self.broken)
157 .finish_non_exhaustive()
158 }
159}
160
161impl CpuEngine {
162 fn with_resources(resources: ExecutionResources) -> Self {
163 Self {
164 resources,
165 buffers: BufferPool::new(),
166 plans: CpuPlanCache::new(),
167 batch_policy: crate::CpuBatchPolicy::default(),
168 allocation_domain: None,
169 runtime_clears: 0,
170 broken: false,
171 runtime_engine_id: crate::runtime_adapter::default_runtime_engine_id(),
172 extension_caches: Some(tenferro_runtime::ExtensionCacheStore::default()),
173 }
174 }
175
176 /// An engine that computes on `pool`, which it takes over.
177 ///
178 /// Every operation may use all workers of the pool; a caller that wants
179 /// fewer threads builds a smaller pool. [`Pool::builder`](crate::Pool::builder)
180 /// reserves the worker stack the providers need by default.
181 ///
182 /// # Examples
183 ///
184 /// ```
185 /// let pool = tenferro_cpu::Pool::builder().threads(4).build()?;
186 /// let engine = tenferro_cpu::CpuEngine::new(pool);
187 /// assert_eq!(engine.num_threads(), 4);
188 /// # Ok::<(), Box<dyn std::error::Error>>(())
189 /// ```
190 pub fn new(pool: crate::Pool) -> Self {
191 Self::with_resources(ExecutionResources::new(Some(pool)))
192 }
193
194 /// An engine with no pool: every operation runs on the calling thread.
195 ///
196 /// A serial engine never starts or enters a pool, so it is safe to use from
197 /// inside any other pool.
198 ///
199 /// # Examples
200 ///
201 /// ```
202 /// let engine = tenferro_cpu::CpuEngine::serial();
203 /// assert_eq!(engine.num_threads(), 1);
204 /// ```
205 pub fn serial() -> Self {
206 Self::with_resources(ExecutionResources::new(None))
207 }
208
209 /// Use `plans` as this engine's plan cache, shared with every other engine
210 /// that holds a clone of it.
211 ///
212 /// # Examples
213 ///
214 /// ```
215 /// use tenferro_cpu::{CpuEngine, CpuPlanCache};
216 ///
217 /// let plans = CpuPlanCache::new();
218 /// let engine = CpuEngine::serial().with_plan_cache(plans.clone());
219 /// assert!(engine.plan_cache().shares_with(&plans));
220 /// ```
221 #[must_use]
222 pub fn with_plan_cache(mut self, plans: CpuPlanCache) -> Self {
223 self.plans = plans;
224 self
225 }
226
227 /// The plan cache this engine reads plans from.
228 ///
229 /// # Examples
230 ///
231 /// ```
232 /// let engine = tenferro_cpu::CpuEngine::serial();
233 /// assert_eq!(engine.plan_cache().stats().entries, 0);
234 /// ```
235 pub fn plan_cache(&self) -> &CpuPlanCache {
236 &self.plans
237 }
238
239 /// Run prepared programs under a runtime engine id other than the default
240 /// [`crate::runtime_engine_id`]; it must match the id this engine kind was
241 /// registered under.
242 ///
243 /// # Examples
244 ///
245 /// ```
246 /// use tenferro_runtime::EngineId;
247 /// let id = EngineId::new("example.cpu.v1")?;
248 /// let engine = tenferro_cpu::CpuEngine::serial().with_runtime_engine_id(id.clone());
249 /// assert_eq!(tenferro_runtime::ExecutionEngine::target(&engine).engine_id(), &id);
250 /// # Ok::<(), Box<dyn std::error::Error>>(())
251 /// ```
252 #[must_use]
253 pub fn with_runtime_engine_id(mut self, id: tenferro_runtime::EngineId) -> Self {
254 self.runtime_engine_id = id;
255 self
256 }
257
258 /// Bind outputs and managed inputs to a shared allocation domain.
259 ///
260 /// # Examples
261 ///
262 /// ```
263 /// let engine = tenferro_cpu::CpuEngine::serial();
264 /// assert!(engine.allocation_domain().is_none());
265 /// ```
266 #[must_use]
267 pub fn with_allocation_domain(mut self, domain: Arc<dyn SharedTensorAllocationDomain>) -> Self {
268 self.allocation_domain = Some(domain);
269 self
270 }
271
272 /// The shared allocation domain this engine is bound to, if any.
273 ///
274 /// # Examples
275 ///
276 /// ```
277 /// assert_eq!(tenferro_cpu::CpuEngine::serial().allocation_domain(), None);
278 /// ```
279 pub fn allocation_domain(&self) -> Option<AllocationDomainId> {
280 self.allocation_domain.as_ref().map(|domain| domain.id())
281 }
282
283 /// The shared allocator this engine is bound to, if any.
284 ///
285 /// # Examples
286 ///
287 /// ```
288 /// assert!(tenferro_cpu::CpuEngine::serial().shared_allocation_domain().is_none());
289 /// ```
290 pub fn shared_allocation_domain(&self) -> Option<&Arc<dyn SharedTensorAllocationDomain>> {
291 self.allocation_domain.as_ref()
292 }
293
294 /// This engine with a different default batch policy.
295 ///
296 /// # Examples
297 ///
298 /// ```
299 /// use tenferro_cpu::{CpuBatchPolicy, CpuBatchStrategy, CpuEngine};
300 ///
301 /// let engine = CpuEngine::serial()
302 /// .with_batch_policy(CpuBatchPolicy::new(CpuBatchStrategy::Sequential));
303 /// assert_eq!(engine.batch_policy().strategy, CpuBatchStrategy::Sequential);
304 /// ```
305 #[must_use]
306 pub fn with_batch_policy(mut self, policy: crate::CpuBatchPolicy) -> Self {
307 self.batch_policy = policy;
308 self
309 }
310
311 /// Replace the default batch policy, returning the previous one.
312 ///
313 /// For a one-operation override use [`crate::with_batch_policy`].
314 ///
315 /// # Examples
316 ///
317 /// ```
318 /// use tenferro_cpu::{CpuBatchPolicy, CpuBatchStrategy, CpuEngine};
319 ///
320 /// let mut engine = CpuEngine::serial();
321 /// let previous = engine.set_batch_policy(CpuBatchPolicy::new(CpuBatchStrategy::Outer));
322 /// assert_eq!(previous.strategy, CpuBatchStrategy::Auto);
323 /// ```
324 pub fn set_batch_policy(&mut self, policy: crate::CpuBatchPolicy) -> crate::CpuBatchPolicy {
325 std::mem::replace(&mut self.batch_policy, policy)
326 }
327
328 /// The default batch policy of this engine.
329 ///
330 /// # Examples
331 ///
332 /// ```
333 /// use tenferro_cpu::{CpuBatchStrategy, CpuEngine};
334 /// assert_eq!(CpuEngine::serial().batch_policy().strategy, CpuBatchStrategy::Auto);
335 /// ```
336 #[must_use]
337 pub fn batch_policy(&self) -> crate::CpuBatchPolicy {
338 self.batch_policy
339 }
340
341 /// The number of workers operations may use: the pool size, or one.
342 ///
343 /// # Examples
344 ///
345 /// ```
346 /// assert_eq!(tenferro_cpu::CpuEngine::serial().num_threads(), 1);
347 /// ```
348 pub fn num_threads(&self) -> usize {
349 self.thread_budget().get()
350 }
351
352 /// Whether the engine runs its parallel work on a reserved
353 /// `threadteam::Team`.
354 ///
355 /// `true` for an engine built from a pool of two or more workers whose
356 /// members were reserved. `false` for [`CpuEngine::serial`], for a
357 /// one-worker pool, and for a pool whose members could not be reserved
358 /// within about ten seconds of construction (the engine then runs serially
359 /// and [`CpuEngine::num_threads`] reports one). Benchmarks should assert
360 /// this after construction.
361 ///
362 /// # Examples
363 ///
364 /// ```
365 /// assert!(!tenferro_cpu::CpuEngine::serial().has_team());
366 /// let pool = tenferro_cpu::Pool::builder().threads(2).build()?;
367 /// assert!(tenferro_cpu::CpuEngine::new(pool).has_team());
368 /// # Ok::<(), Box<dyn std::error::Error>>(())
369 /// ```
370 pub fn has_team(&self) -> bool {
371 self.resources.has_team()
372 }
373
374 pub(crate) fn thread_budget(&self) -> NonZeroUsize {
375 self.resources.thread_budget()
376 }
377
378 /// The dispatch costs the engine's tokens use: those of its pool, or
379 /// [`threadteam::DispatchCosts::host_default`] for [`CpuEngine::serial`].
380 ///
381 /// Use them with [`CpuEngine::num_threads`] to decide, before taking a token
382 /// with [`CpuEngine::exec`], whether an operation would fork and at what
383 /// width. A token of an engine with a team ([`CpuEngine::has_team`]) charges
384 /// [`threadteam::DispatchCosts::cost`] of [`threadteam::DispatchKind::Team`]
385 /// for a section of two or more participants, so
386 /// [`threadteam::width_for_by`] and [`threadteam::pays_at`] over that cost
387 /// give the answer its `width_for` and `pays` would. A budget of one never
388 /// forks.
389 ///
390 /// # Examples
391 ///
392 /// ```
393 /// use std::num::NonZeroUsize;
394 /// use std::time::Duration;
395 /// use tenferro_cpu::threadteam::{width_for_by, DispatchKind};
396 ///
397 /// let pool = tenferro_cpu::Pool::builder().threads(2).build()?;
398 /// let engine = tenferro_cpu::CpuEngine::new(pool);
399 /// let budget = NonZeroUsize::new(engine.num_threads()).unwrap();
400 /// let cost = |width| engine.dispatch_costs().cost(DispatchKind::Team, width);
401 ///
402 /// // A nanosecond of work stays serial; a second of work uses the budget.
403 /// assert_eq!(width_for_by(Duration::from_nanos(1), 1, budget, cost).get(), 1);
404 /// assert_eq!(
405 /// width_for_by(Duration::from_secs(1), 1, budget, cost).get(),
406 /// engine.num_threads(),
407 /// );
408 /// # Ok::<(), Box<dyn std::error::Error>>(())
409 /// ```
410 pub fn dispatch_costs(&self) -> &threadteam::DispatchCosts {
411 self.resources.dispatch_costs()
412 }
413
414 /// Whether an earlier operation broke this engine (see
415 /// [`CpuEngineError::Broken`]).
416 ///
417 /// # Examples
418 ///
419 /// ```
420 /// assert!(!tenferro_cpu::CpuEngine::serial().is_broken());
421 /// ```
422 pub fn is_broken(&self) -> bool {
423 self.broken
424 }
425
426 /// The provider execution bundle for this engine: a placement token over
427 /// every worker of its pool, its workspace and its batch policy.
428 ///
429 /// A downstream kernel that calls a provider (cpueinsum, tcontract,
430 /// tlinalg) directly passes this bundle by `&mut`, so it uses the same
431 /// threads, budget and batch policy as tenferro. The bundle is not `Clone`:
432 /// it is the engine's whole thread budget for that one call, and the provider
433 /// splits it. Getting it applies the same checks as an operation.
434 ///
435 /// # Errors
436 ///
437 /// Returns [`crate::Error::RuntimeState`] with a [`CpuEngineError`] source
438 /// when the engine is broken.
439 ///
440 /// # Examples
441 ///
442 /// ```
443 /// let mut engine = tenferro_cpu::CpuEngine::serial();
444 /// let exec = engine.exec()?;
445 /// assert_eq!(exec.budget(), 1);
446 /// # Ok::<(), tenferro_tensor::Error>(())
447 /// ```
448 pub fn exec(&mut self) -> crate::Result<cpueinsum::Exec<'_>> {
449 self.check_entry("CpuEngine::exec")?;
450 Ok(self.resources.exec(self.batch_policy))
451 }
452
453 /// Run `op` on the engine's provider bundle, keeping the engine's guarantees.
454 ///
455 /// `op` receives the same bundle an operation mints (a placement token over
456 /// every worker of the pool, the workspace and the batch policy; see
457 /// [`CpuEngine::exec`]) and runs on the calling thread. Unlike
458 /// [`CpuEngine::exec`], the call is an engine operation: a panic inside `op`
459 /// marks the engine broken, and contraction scratch retained above
460 /// [`CpuEngine::buffer_pool_limit_bytes`] is released when `op` returns.
461 ///
462 /// The bundle is borrowed for the call only, and the engine stays mutably
463 /// borrowed while `op` runs, so neither can be stored or re-entered from
464 /// inside `op`; the compiler rejects both:
465 ///
466 /// ```compile_fail,E0501
467 /// let mut engine = tenferro_cpu::CpuEngine::serial();
468 /// let engine_ref = &mut engine;
469 /// engine_ref.with_exec(|_| engine_ref.with_exec(|exec| exec.budget()))?;
470 /// # Ok::<(), tenferro_tensor::Error>(())
471 /// ```
472 ///
473 /// ```compile_fail,E0521
474 /// let mut engine = tenferro_cpu::CpuEngine::serial();
475 /// let mut kept = None;
476 /// engine.with_exec(|exec| kept = Some(exec))?;
477 /// # Ok::<(), tenferro_tensor::Error>(())
478 /// ```
479 ///
480 /// # Errors
481 ///
482 /// Returns [`crate::Error::RuntimeState`] with a [`CpuEngineError::Broken`]
483 /// source, without calling `op`, when an earlier operation panicked.
484 ///
485 /// # Panics
486 ///
487 /// Resumes a panic raised by `op` after marking the engine broken.
488 ///
489 /// # Examples
490 ///
491 /// ```
492 /// let mut engine = tenferro_cpu::CpuEngine::serial();
493 /// let budget = engine.with_exec(|exec| exec.budget())?;
494 /// assert_eq!(budget, 1);
495 /// # Ok::<(), tenferro_tensor::Error>(())
496 /// ```
497 ///
498 /// A panic inside `op` breaks the engine, and a broken engine refuses later
499 /// calls with an error:
500 ///
501 /// ```
502 /// use tenferro_cpu::{CpuEngine, CpuEngineError};
503 ///
504 /// let mut engine = CpuEngine::serial();
505 /// let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
506 /// engine.with_exec(|_| panic!("a failing block"))
507 /// }));
508 /// assert!(unwind.is_err());
509 /// assert!(engine.is_broken());
510 ///
511 /// let refused = engine.with_exec(|exec| exec.budget()).unwrap_err();
512 /// let source = std::error::Error::source(&refused).and_then(|s| s.downcast_ref());
513 /// assert_eq!(source, Some(&CpuEngineError::Broken));
514 /// ```
515 pub fn with_exec<R>(
516 &mut self,
517 op: impl FnOnce(&mut cpueinsum::Exec<'_>) -> R,
518 ) -> crate::Result<R> {
519 self.enter("CpuEngine::with_exec", true, |exec, _, _| Ok(op(exec)))
520 }
521
522 /// Borrow the provider bundle, the buffer pool and the N-ary contraction
523 /// scratch for one downstream call.
524 ///
525 /// # Errors
526 ///
527 /// Returns the entry refusals of [`CpuEngine::exec`], or `op`'s own error.
528 ///
529 /// # Examples
530 ///
531 /// ```
532 /// let mut engine = tenferro_cpu::CpuEngine::serial();
533 /// let len = engine.with_contraction_exec(|_, buffers, workspaces| {
534 /// workspaces.with_scratch::<f64, _>(4, buffers.max_retained_capacity_bytes(), |scratch| {
535 /// Ok(scratch.len())
536 /// })
537 /// })?;
538 /// assert_eq!(len, 4);
539 /// # Ok::<(), tenferro_tensor::Error>(())
540 /// ```
541 #[doc(hidden)]
542 pub fn with_contraction_exec<R>(
543 &mut self,
544 op: impl FnOnce(
545 &mut cpueinsum::Exec<'_>,
546 &mut BufferPool,
547 &crate::ContractionWorkspaces,
548 ) -> crate::Result<R>,
549 ) -> crate::Result<R> {
550 self.enter("CpuEngine::with_contraction_exec", true, op)
551 }
552
553 /// Borrow the provider bundle and buffer pool for a linalg provider call.
554 ///
555 /// The provider chooses its own execution through the bundle; this
556 /// callback itself runs on the calling thread.
557 ///
558 /// # Errors
559 ///
560 /// Returns the entry refusals of [`CpuEngine::exec`], or `op`'s own error.
561 ///
562 /// # Examples
563 ///
564 /// ```
565 /// use tenferro_cpu::{CpuBatchStrategy, CpuEngine};
566 ///
567 /// let mut engine = CpuEngine::serial();
568 /// let strategy = engine.with_linalg_pool(|exec, _| Ok(exec.batch_policy().strategy))?;
569 /// assert_eq!(strategy, CpuBatchStrategy::Auto);
570 /// # Ok::<(), tenferro_tensor::Error>(())
571 /// ```
572 #[doc(hidden)]
573 pub fn with_linalg_pool<R>(
574 &mut self,
575 op: impl FnOnce(&mut cpueinsum::Exec<'_>, &mut BufferPool) -> crate::Result<R>,
576 ) -> crate::Result<R> {
577 self.enter("CpuEngine::with_linalg_pool", false, |exec, buffers, _| {
578 op(exec, buffers)
579 })
580 }
581
582 /// Run `op` on the calling thread with the engine's guarantees, but without
583 /// taking a placement token: the guarded entry of [`CpuEngine::with_exec`]
584 /// at width one.
585 ///
586 /// `op` receives a [`SerialCtx`] with the engine's workspace and batch
587 /// policy. It is not a provider bundle: it carries no token, so nothing in
588 /// it can fork, and getting it reserves none of the pool's workers. Apart
589 /// from the token, the call is the same engine operation as
590 /// [`CpuEngine::with_exec`]: a broken engine is refused, a panic inside `op`
591 /// marks the engine broken (buffers `op` had taken from the buffer pool are
592 /// not returned to it, since a broken engine runs nothing again), and
593 /// contraction scratch retained above [`CpuEngine::buffer_pool_limit_bytes`]
594 /// is released when `op` returns or unwinds.
595 ///
596 /// The context is borrowed for the call only, and the engine stays mutably
597 /// borrowed while `op` runs, so neither can be stored or re-entered from
598 /// inside `op`; the compiler rejects both:
599 ///
600 /// ```compile_fail,E0501
601 /// let mut engine = tenferro_cpu::CpuEngine::serial();
602 /// let engine_ref = &mut engine;
603 /// engine_ref.with_guard(|_| engine_ref.with_guard(|ctx| ctx.batch_policy()))?;
604 /// # Ok::<(), tenferro_tensor::Error>(())
605 /// ```
606 ///
607 /// ```compile_fail,E0521
608 /// let mut engine = tenferro_cpu::CpuEngine::serial();
609 /// let mut kept = None;
610 /// engine.with_guard(|ctx| kept = Some(ctx))?;
611 /// # Ok::<(), tenferro_tensor::Error>(())
612 /// ```
613 ///
614 /// # Errors
615 ///
616 /// Returns [`crate::Error::RuntimeState`] with a [`CpuEngineError::Broken`]
617 /// source, without calling `op`, when an earlier operation panicked.
618 ///
619 /// # Panics
620 ///
621 /// Resumes a panic raised by `op` after marking the engine broken.
622 ///
623 /// # Examples
624 ///
625 /// ```
626 /// use tenferro_cpu::{CpuBatchPolicy, CpuBatchStrategy, CpuEngine};
627 ///
628 /// let mut engine = CpuEngine::new(tenferro_cpu::Pool::builder().threads(2).build()?)
629 /// .with_batch_policy(CpuBatchPolicy::new(CpuBatchStrategy::Outer));
630 /// let strategy = engine.with_guard(|ctx| ctx.batch_policy().strategy)?;
631 /// assert_eq!(strategy, CpuBatchStrategy::Outer);
632 /// # Ok::<(), Box<dyn std::error::Error>>(())
633 /// ```
634 ///
635 /// A panic inside `op` breaks the engine, and a broken engine refuses later
636 /// calls with an error:
637 ///
638 /// ```
639 /// use tenferro_cpu::{CpuEngine, CpuEngineError};
640 ///
641 /// let mut engine = CpuEngine::serial();
642 /// let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
643 /// engine.with_guard(|_| panic!("a failing serial phase"))
644 /// }));
645 /// assert!(unwind.is_err());
646 /// assert!(engine.is_broken());
647 ///
648 /// let refused = engine.with_guard(|_| ()).unwrap_err();
649 /// let source = std::error::Error::source(&refused).and_then(|s| s.downcast_ref());
650 /// assert_eq!(source, Some(&CpuEngineError::Broken));
651 /// ```
652 pub fn with_guard<R>(&mut self, op: impl FnOnce(SerialCtx<'_>) -> R) -> crate::Result<R> {
653 self.guarded("CpuEngine::with_guard", true, |resources, policy, _| {
654 Ok(op(SerialCtx {
655 workspace: resources.workspace(),
656 policy,
657 }))
658 })
659 }
660
661 /// The token-minting entry behind the closure-scoped entries: [`Self::guarded`]
662 /// with the bundle and the N-ary scratch lent to `op`.
663 fn enter<R>(
664 &mut self,
665 name: &'static str,
666 bound_retention: bool,
667 op: impl FnOnce(
668 &mut cpueinsum::Exec<'_>,
669 &mut BufferPool,
670 &crate::ContractionWorkspaces,
671 ) -> crate::Result<R>,
672 ) -> crate::Result<R> {
673 self.guarded(name, bound_retention, |resources, policy, buffers| {
674 let mut exec = resources.exec(policy);
675 op(&mut exec, buffers, &resources.nary)
676 })
677 }
678
679 /// The one guarded entry: refuse a broken engine, lend the resources, the
680 /// batch policy and the buffer pool, mark the engine broken if `op`
681 /// unwinds, and with `bound_retention` release workspace retained above the
682 /// buffer limit afterwards.
683 fn guarded<R>(
684 &mut self,
685 name: &'static str,
686 bound_retention: bool,
687 op: impl FnOnce(&ExecutionResources, crate::CpuBatchPolicy, &mut BufferPool) -> crate::Result<R>,
688 ) -> crate::Result<R> {
689 self.check_entry(name)?;
690 let _retention = bound_retention.then(|| {
691 self.resources
692 .retention_guard(self.buffers.max_retained_capacity_bytes())
693 });
694 let (resources, policy) = (&self.resources, self.batch_policy);
695 guard_unwind(&mut self.buffers, &mut self.broken, |buffers| {
696 op(resources, policy, buffers)
697 })
698 }
699
700 /// Refuse an operation that cannot run on this engine now.
701 pub(crate) fn check_entry(&self, op: &'static str) -> crate::Result<()> {
702 if self.broken {
703 return Err(crate::Error::runtime_state_source(
704 op,
705 CpuEngineError::Broken,
706 ));
707 }
708 Ok(())
709 }
710
711 /// Ledger roots of every placement token the engine minted since the last
712 /// call, so a test can read `threadteam::ledger` for the operation it just
713 /// ran (`max_live`, `installs`).
714 ///
715 /// Available with the `ledger` cargo feature, which is test support.
716 #[cfg(feature = "ledger")]
717 #[doc(hidden)]
718 pub fn take_ledger_roots(&self) -> Vec<u64> {
719 self.resources.take_ledger_roots()
720 }
721
722 /// Pool entries (installs from a plain thread or a foreign pool) made by
723 /// the tokens minted since the last `take_ledger_*` call, which it consumes.
724 ///
725 /// Zero means every operation since then stayed on the calling thread.
726 /// Available with the `ledger` cargo feature, which is test support.
727 #[cfg(feature = "ledger")]
728 #[doc(hidden)]
729 pub fn take_ledger_installs(&self) -> usize {
730 self.take_ledger_roots()
731 .into_iter()
732 .map(threadteam::ledger::installs)
733 .sum()
734 }
735
736 /// Largest number of tasks live at once under any token minted since the
737 /// last `take_ledger_*` call, which it consumes. It never exceeds
738 /// [`CpuEngine::num_threads`].
739 ///
740 /// Available with the `ledger` cargo feature, which is test support.
741 #[cfg(feature = "ledger")]
742 #[doc(hidden)]
743 pub fn take_ledger_max_live(&self) -> usize {
744 self.take_ledger_roots()
745 .into_iter()
746 .map(threadteam::ledger::max_live)
747 .max()
748 .unwrap_or(0)
749 }
750
751 /// Number of retained buffers in the engine's buffer pool.
752 ///
753 /// # Examples
754 ///
755 /// ```
756 /// assert_eq!(tenferro_cpu::CpuEngine::serial().buffer_pool_len(), 0);
757 /// ```
758 pub fn buffer_pool_len(&self) -> usize {
759 self.buffers.len()
760 }
761
762 /// Retained-buffer statistics of the engine's buffer pool.
763 ///
764 /// # Examples
765 ///
766 /// ```
767 /// assert_eq!(tenferro_cpu::CpuEngine::serial().buffer_pool_stats().buffers, 0);
768 /// ```
769 pub fn buffer_pool_stats(&self) -> BufferPoolStats {
770 self.buffers.stats()
771 }
772
773 /// The buffer pool as cache statistics (entries and retained bytes).
774 ///
775 /// # Examples
776 ///
777 /// ```
778 /// assert_eq!(tenferro_cpu::CpuEngine::serial().buffer_pool_cache_stats().entries, 0);
779 /// ```
780 pub fn buffer_pool_cache_stats(&self) -> CacheStats {
781 let stats = self.buffers.stats();
782 CacheStats {
783 entries: stats.buffers,
784 retained_bytes: stats.capacity_bytes,
785 ..CacheStats::default()
786 }
787 }
788
789 /// The retention limit of recycled buffers, in retained capacity bytes.
790 ///
791 /// # Examples
792 ///
793 /// ```
794 /// let mut engine = tenferro_cpu::CpuEngine::serial();
795 /// engine.set_buffer_pool_limit_bytes(0);
796 /// assert_eq!(engine.buffer_pool_limit_bytes(), 0);
797 /// ```
798 pub fn buffer_pool_limit_bytes(&self) -> usize {
799 self.buffers.max_retained_capacity_bytes()
800 }
801
802 /// Bound the bytes of recycled buffers and contraction scratch retained
803 /// between operations; zero disables retention.
804 ///
805 /// Destinations larger than the retention limit cannot be recycled, so
806 /// repeated calls allocate again and may pay first-touch costs. The default
807 /// is 100 MiB; raise it when a run keeps producing outputs above that size.
808 ///
809 /// # Examples
810 ///
811 /// ```
812 /// let mut engine = tenferro_cpu::CpuEngine::serial();
813 /// engine.set_buffer_pool_limit_bytes(1 << 20);
814 /// assert_eq!(engine.buffer_pool_limit_bytes(), 1 << 20);
815 /// ```
816 pub fn set_buffer_pool_limit_bytes(&mut self, max_retained_capacity_bytes: usize) {
817 self.buffers
818 .set_max_retained_capacity_bytes(max_retained_capacity_bytes);
819 self.resources.nary.trim_to(max_retained_capacity_bytes);
820 if self.resources.retained_bytes() > max_retained_capacity_bytes {
821 self.resources.trim();
822 }
823 }
824
825 /// Drop every retained buffer and contraction scratch.
826 ///
827 /// # Examples
828 ///
829 /// ```
830 /// let mut engine = tenferro_cpu::CpuEngine::serial();
831 /// engine.reset_buffer_pool();
832 /// assert_eq!(engine.buffer_pool_len(), 0);
833 /// ```
834 pub fn reset_buffer_pool(&mut self) {
835 self.buffers.clear();
836 self.resources.nary.clear_all();
837 self.resources.trim();
838 }
839
840 /// Aggregate statistics of everything this engine retains between
841 /// operations: buffers, contraction scratch, its plan cache and its
842 /// extension cache store.
843 ///
844 /// `entries` and `retained_bytes` cover all of them. `hits`, `misses` and
845 /// `evictions` are the plan cache's alone; the extension store reports its
846 /// own through [`ExtensionCacheStore::stats`](tenferro_runtime::ExtensionCacheStore::stats).
847 /// While a run has the extension store lent out, it is not counted.
848 ///
849 /// # Examples
850 ///
851 /// ```
852 /// assert_eq!(tenferro_cpu::CpuEngine::serial().runtime_cache_stats().entries, 0);
853 /// ```
854 #[doc(hidden)]
855 pub fn runtime_cache_stats(&self) -> CacheStats {
856 let (workspace_entries, nary_bytes) = self.resources.nary.total_stats();
857 let workspace_bytes = nary_bytes.saturating_add(self.resources.retained_bytes());
858 let buffers = self.buffers.cache_stats();
859 let plans = self.plans.stats();
860 let (extension_entries, extension_bytes) =
861 self.extension_caches.as_ref().map_or((0, 0), |store| {
862 let stats = store.stats(tenferro_runtime::ExtensionCacheSelector::All);
863 (stats.entries, stats.retained_bytes)
864 });
865 CacheStats {
866 entries: buffers
867 .entries
868 .saturating_add(plans.entries)
869 .saturating_add(workspace_entries)
870 .saturating_add(extension_entries),
871 retained_bytes: buffers
872 .retained_bytes
873 .saturating_add(plans.retained_bytes)
874 .saturating_add(workspace_bytes)
875 .saturating_add(extension_bytes),
876 hits: plans.hits,
877 misses: plans.misses,
878 evictions: plans.evictions,
879 clears: self.runtime_clears,
880 }
881 }
882
883 /// Drop everything this engine retains between operations: buffers,
884 /// contraction scratch, its extension cache store and its plan cache (and
885 /// therefore the plans of every engine sharing it).
886 ///
887 /// # Examples
888 ///
889 /// ```
890 /// let mut engine = tenferro_cpu::CpuEngine::serial();
891 /// engine.clear_runtime_caches();
892 /// assert_eq!(engine.runtime_cache_stats().clears, 1);
893 /// ```
894 #[doc(hidden)]
895 pub fn clear_runtime_caches(&mut self) {
896 self.reset_buffer_pool();
897 self.plans.clear();
898 if let Some(store) = self.extension_caches.as_mut() {
899 store.clear();
900 }
901 self.runtime_clears = self.runtime_clears.saturating_add(1);
902 }
903}
904
905/// Run one operation's use of the buffer pool, recovering the pool's
906/// accounting and marking the engine broken if it unwinds.
907///
908/// Buffers checked out during the operation are counted as in flight. A normal
909/// exit forgets that accounting (the buffers left as outputs); an unwind
910/// replenishes the pool with replacements for the buffers it lost and marks the
911/// engine broken, since other workspace may be half updated.
912pub(crate) fn guard_unwind<R>(
913 buffers: &mut BufferPool,
914 broken: &mut bool,
915 op: impl FnOnce(&mut BufferPool) -> R,
916) -> R {
917 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| op(&mut *buffers))) {
918 Ok(value) => value,
919 Err(payload) => {
920 // The engine is never used again after a panic, so the buffers the
921 // operation had checked out need no accounting repair.
922 *broken = true;
923 std::panic::resume_unwind(payload)
924 }
925 }
926}
927
928impl CpuEngine {
929 /// Split borrows for the operation helpers: the provider bundle minted for
930 /// this operation (it borrows only the pool resources), the buffer pool, the
931 /// plan cache and the broken flag.
932 pub(crate) fn parts(
933 &mut self,
934 ) -> (
935 cpueinsum::Exec<'_>,
936 &mut BufferPool,
937 &CpuPlanCache,
938 &mut bool,
939 ) {
940 (
941 self.resources.exec(self.batch_policy),
942 &mut self.buffers,
943 &self.plans,
944 &mut self.broken,
945 )
946 }
947}
948
949#[cfg(test)]
950mod tests;