Skip to main content

tenferro_cpu/
exec_session.rs

1use crate::buffer_pool::{BufferPool, PoolScalar};
2use crate::engine::guard_unwind;
3use crate::{CpuEngine, CpuPlanCache, Tensor, TensorRead, TensorValue, TensorWrite};
4use num_complex::{Complex32, Complex64};
5use tenferro_tensor::backend::{ElementwiseFusionPlan, Engine};
6use tenferro_tensor::{
7    ActivationOp, CompareDir, ContractionScalar, DType, DotGeneralConfig, ElementwiseReadOp,
8    GatherConfig, PadConfig, ScatterConfig, SliceConfig, TensorView, TypedTensor,
9};
10use tenferro_tensor::{
11    DotGeneralAccumulation, SessionCachedDot, TensorAnalytic, TensorBuffer, TensorDeviceTransfer,
12    TensorDot, TensorElementwise, TensorFusion, TensorIndexing, TensorReduction, TensorStructural,
13};
14
15use super::backend::elementwise_read_into_fallback_with_pool;
16use super::{
17    activation, analytic, copy_tensor_read_into, elementwise, indexing,
18    materialize_tensor_read_in_domain, reduction, softmax, structural,
19};
20
21/// Native-engine marker for [`CpuEngine`]; private to this crate so no other
22/// crate can create a token that claims to be a CPU engine.
23pub(super) struct CpuEngineMarker;
24
25/// Route an eager `dot_general` request to a slot of the engine's plan cache.
26///
27/// A prepared program keys a contraction plan by its instruction index, because
28/// a compiled program has exactly one such index per operation. An eager call
29/// has none, so this derives one from the request's geometry: the dtype, the two
30/// operand shapes, the four axis lists and the conjugation flags, all of which
31/// the plan key already names.
32///
33/// The slot only *routes*. [`crate::contraction::BinaryCache`] still compares the
34/// full key - including both operand layouts, which this fingerprint deliberately
35/// leaves out because reading them here would cost the very allocation the cache
36/// is meant to avoid - before reusing a plan, so two requests that collide
37/// rebuild instead of sharing a plan that does not apply. A collision therefore
38/// costs a miss, never a wrong answer.
39fn eager_dot_cache_slot(
40    capacity: usize,
41    lhs: &TensorRead<'_>,
42    rhs: &TensorRead<'_>,
43    config: &DotGeneralConfig,
44    lhs_conj: bool,
45    rhs_conj: bool,
46) -> Option<usize> {
47    use std::hash::{Hash, Hasher};
48
49    if capacity == 0 {
50        return None;
51    }
52    let mut hasher = std::collections::hash_map::DefaultHasher::new();
53    lhs.dtype().hash(&mut hasher);
54    lhs.shape().hash(&mut hasher);
55    rhs.shape().hash(&mut hasher);
56    config.lhs_contracting_dims.hash(&mut hasher);
57    config.rhs_contracting_dims.hash(&mut hasher);
58    config.lhs_batch_dims.hash(&mut hasher);
59    config.rhs_batch_dims.hash(&mut hasher);
60    lhs_conj.hash(&mut hasher);
61    rhs_conj.hash(&mut hasher);
62    Some(hasher.finish() as usize % capacity)
63}
64
65fn pooled_zero_tensor<T>(
66    buffers: &mut BufferPool,
67    shape: impl tenferro_tensor::IntoShapeVec,
68) -> crate::Result<TypedTensor<T>>
69where
70    T: PoolScalar + Clone + 'static,
71{
72    let shape = shape.into_shape_vec();
73    let element_count =
74        tenferro_tensor::validate::checked_shape_product("dot_general", "output", &shape)?;
75    TypedTensor::from_vec_col_major(shape, T::pool_acquire_zeroed(buffers, element_count))
76}
77
78fn allocate_dot_output(
79    buffers: &mut BufferPool,
80    dtype: DType,
81    shape: impl tenferro_tensor::IntoShapeVec,
82) -> crate::Result<Tensor> {
83    let shape = shape.into_shape_vec();
84    match dtype {
85        DType::F32 => pooled_zero_tensor(buffers, &shape).map(Tensor::from_typed::<f32>),
86        DType::F64 => pooled_zero_tensor(buffers, &shape).map(Tensor::from_typed::<f64>),
87        DType::C32 => pooled_zero_tensor(buffers, &shape)
88            .map(Tensor::from_typed::<tenferro_tensor::Complex32>),
89        DType::C64 => pooled_zero_tensor(buffers, &shape)
90            .map(Tensor::from_typed::<tenferro_tensor::Complex64>),
91        dtype => Err(crate::Error::unsupported_dtype(
92            "dot_general",
93            dtype,
94            crate::cpu_contraction_unsupported_dtype_message(dtype),
95        )),
96    }
97}
98
99fn flatten_compact_read(
100    input: TensorRead<'_>,
101    element_count: usize,
102) -> crate::Result<TensorRead<'_>> {
103    macro_rules! flatten {
104        ($variant:ident, $view:expr) => {
105            Ok(TensorRead::from_view(TensorView::$variant(
106                $view.reshape_view(&[element_count])?,
107            )))
108        };
109    }
110    /// The typed tensor behind a read adapter's tensor, or the refusal this adapter reports.
111    fn read_refusal(tensor: &Tensor) -> crate::Error {
112        crate::Error::unsupported_dtype(
113            "flatten_compact_read",
114            tensor.dtype(),
115            "an externally defined payload is not a compact runtime read",
116        )
117    }
118
119    match input {
120        TensorRead::Tensor(tensor) => match tensor.dtype() {
121            DType::F32 => flatten!(
122                F32,
123                tensor
124                    .as_typed::<f32>()
125                    .ok_or_else(|| read_refusal(tensor))?
126                    .as_view()
127            ),
128            DType::F64 => flatten!(
129                F64,
130                tensor
131                    .as_typed::<f64>()
132                    .ok_or_else(|| read_refusal(tensor))?
133                    .as_view()
134            ),
135            DType::I32 => flatten!(
136                I32,
137                tensor
138                    .as_typed::<i32>()
139                    .ok_or_else(|| read_refusal(tensor))?
140                    .as_view()
141            ),
142            DType::I64 => flatten!(
143                I64,
144                tensor
145                    .as_typed::<i64>()
146                    .ok_or_else(|| read_refusal(tensor))?
147                    .as_view()
148            ),
149            DType::Bool => flatten!(
150                Bool,
151                tensor
152                    .as_typed::<bool>()
153                    .ok_or_else(|| read_refusal(tensor))?
154                    .as_view()
155            ),
156            DType::C32 => flatten!(
157                C32,
158                tensor
159                    .as_typed::<Complex32>()
160                    .ok_or_else(|| read_refusal(tensor))?
161                    .as_view()
162            ),
163            DType::C64 => flatten!(
164                C64,
165                tensor
166                    .as_typed::<Complex64>()
167                    .ok_or_else(|| read_refusal(tensor))?
168                    .as_view()
169            ),
170            // A caller-owned payload has no compact runtime read.
171            DType::External(_) => Err(read_refusal(tensor)),
172        },
173        TensorRead::View(TensorView::F32(view)) => flatten!(F32, view),
174        TensorRead::View(TensorView::F64(view)) => flatten!(F64, view),
175        TensorRead::View(TensorView::I32(view)) => flatten!(I32, view),
176        TensorRead::View(TensorView::I64(view)) => flatten!(I64, view),
177        TensorRead::View(TensorView::Bool(view)) => flatten!(Bool, view),
178        TensorRead::View(TensorView::C32(view)) => flatten!(C32, view),
179        TensorRead::View(TensorView::C64(view)) => flatten!(C64, view),
180    }
181}
182
183impl CpuEngine {
184    /// Replace this engine's effective batch policy, returning the previous one.
185    pub(crate) fn replace_batch_policy(
186        &mut self,
187        policy: crate::CpuBatchPolicy,
188    ) -> crate::CpuBatchPolicy {
189        self.set_batch_policy(policy)
190    }
191
192    /// Run an operation with the one placement token minted for it: every
193    /// worker of the engine's pool, or the serial token of a serial engine.
194    /// Kernels split the token and take it by value, last.
195    fn run_native_with_exec<R>(
196        &mut self,
197        op: impl FnOnce(threadteam::Exec<'_>, &mut BufferPool) -> crate::Result<R>,
198    ) -> crate::Result<R> {
199        self.check_entry("CpuEngine")?;
200        let (exec, buffers, _, broken) = self.parts();
201        guard_unwind(buffers, broken, |buffers| op(exec.into_token(), buffers))
202    }
203
204    fn run_native_with_plans<R>(
205        &mut self,
206        op: impl FnOnce(threadteam::Exec<'_>, &mut BufferPool, &CpuPlanCache) -> crate::Result<R>,
207    ) -> crate::Result<R> {
208        self.check_entry("CpuEngine")?;
209        let (exec, buffers, plans, broken) = self.parts();
210        guard_unwind(buffers, broken, |buffers| {
211            op(exec.into_token(), buffers, plans)
212        })
213    }
214
215    /// Run a contraction-provider call with the engine's bundle, buffers and
216    /// plans; the provider splits the bundle's token itself.
217    fn run_contraction<R>(
218        &mut self,
219        op: impl FnOnce(&mut cpueinsum::Exec<'_>, &mut BufferPool, &CpuPlanCache) -> crate::Result<R>,
220    ) -> crate::Result<R> {
221        self.check_entry("dot_general")?;
222        let _retention = self
223            .resources
224            .retention_guard(self.buffers.max_retained_capacity_bytes());
225        let mut exec = self.resources.exec(self.batch_policy);
226        let plans = &self.plans;
227        guard_unwind(&mut self.buffers, &mut self.broken, |buffers| {
228            op(&mut exec, buffers, plans)
229        })
230    }
231}
232
233impl TensorDeviceTransfer for CpuEngine {
234    fn download_to_host(&mut self, tensor: TensorRead<'_>) -> crate::Result<Tensor> {
235        self.check_entry("CpuEngine::download_to_host")?;
236        if tensor.backend_family().is_some() {
237            return Err(crate::Error::runtime_state(
238                "CpuEngine::download_to_host",
239                "CPU backend received a backend buffer; download the tensor to host with its owning backend before CPU execution",
240            ));
241        }
242        crate::preset_view("CpuEngine::download_to_host", tensor)?.duplicate()
243    }
244
245    fn upload_host_tensor(&mut self, tensor: TensorRead<'_>) -> crate::Result<Tensor> {
246        self.check_entry("CpuEngine::upload_host_tensor")?;
247        if tensor.backend_family().is_some() {
248            return Err(crate::Error::runtime_state(
249                "CpuEngine::upload_host_tensor",
250                "CPU backend upload_host_tensor expects a host tensor; download backend buffers to host before CPU execution",
251            ));
252        }
253        crate::preset_view("CpuEngine::upload_host_tensor", tensor)?.duplicate()
254    }
255}
256
257/// Delegation for operations whose kernels take the operation's placement
258/// token (last) and allocate their outputs from the session pool.
259macro_rules! delegate_with_pool {
260    ($name:ident($($arg:ident : $ty:ty),*) => $callee:path) => {
261        fn $name(&mut self, $($arg: $ty),*) -> crate::Result<Tensor> {
262            self.run_native_with_exec(|exec, buffers| $callee(buffers, $($arg),*, exec))
263        }
264    };
265}
266
267impl TensorElementwise for CpuEngine {
268    // Elementwise — direct delegation within the current session scope.
269    fn elementwise_read_into(
270        &mut self,
271        op: ElementwiseReadOp,
272        inputs: &[TensorRead<'_>],
273        out: TensorWrite<'_>,
274    ) -> crate::Result<()> {
275        self.run_native_with_exec(|exec, buffers| {
276            // The one-shot fast path or the fallback consumes the token, never both.
277            tenferro_internal_cpu_kernels::elementwise_read_into_with_exec(
278                op,
279                inputs,
280                out,
281                |inputs, out, exec| {
282                    elementwise_read_into_fallback_with_pool(buffers, op, inputs, out, exec)
283                },
284                exec,
285            )
286        })
287    }
288
289    fn add_read(&mut self, lhs: TensorRead<'_>, rhs: TensorRead<'_>) -> crate::Result<Tensor> {
290        self.run_native_with_exec(|exec, buffers| {
291            elementwise::add_read_with_pool(buffers, lhs, rhs, exec)
292        })
293    }
294
295    delegate_with_pool!(sub_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::sub_read_with_pool);
296    delegate_with_pool!(mul_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::mul_read_with_pool);
297    delegate_with_pool!(neg_read(input: TensorRead<'_>) => elementwise::neg_read_with_pool);
298    delegate_with_pool!(conj_read(input: TensorRead<'_>) => elementwise::conj_read_with_pool);
299    delegate_with_pool!(div_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::div_read_with_pool);
300    delegate_with_pool!(rem(lhs: &Tensor, rhs: &Tensor) => elementwise::rem_with_pool);
301    delegate_with_pool!(rem_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::rem_read_with_pool);
302    delegate_with_pool!(abs_read(input: TensorRead<'_>) => elementwise::abs_read_with_pool);
303    delegate_with_pool!(sign_read(input: TensorRead<'_>) => elementwise::sign_read_with_pool);
304    delegate_with_pool!(maximum_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::maximum_read_with_pool);
305    delegate_with_pool!(minimum_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::minimum_read_with_pool);
306    delegate_with_pool!(compare_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>, dir: &CompareDir) => elementwise::compare_read_with_pool);
307    delegate_with_pool!(select_read(pred: TensorRead<'_>, on_true: TensorRead<'_>, on_false: TensorRead<'_>) => elementwise::select_read_with_pool);
308    delegate_with_pool!(clamp_read(input: TensorRead<'_>, lower: TensorRead<'_>, upper: TensorRead<'_>) => elementwise::clamp_read_with_pool);
309}
310
311impl TensorAnalytic for CpuEngine {
312    // Analytic
313    delegate_with_pool!(exp_read(input: TensorRead<'_>) => analytic::exp_read_with_pool);
314    delegate_with_pool!(log_read(input: TensorRead<'_>) => analytic::log_read_with_pool);
315    delegate_with_pool!(sin_read(input: TensorRead<'_>) => analytic::sin_read_with_pool);
316    delegate_with_pool!(cos_read(input: TensorRead<'_>) => analytic::cos_read_with_pool);
317    delegate_with_pool!(tanh_read(input: TensorRead<'_>) => analytic::tanh_read_with_pool);
318    delegate_with_pool!(sqrt_read(input: TensorRead<'_>) => analytic::sqrt_read_with_pool);
319    delegate_with_pool!(rsqrt_read(input: TensorRead<'_>) => analytic::rsqrt_read_with_pool);
320    delegate_with_pool!(pow_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => analytic::pow_read_with_pool);
321    delegate_with_pool!(expm1_read(input: TensorRead<'_>) => analytic::expm1_read_with_pool);
322    delegate_with_pool!(log1p_read(input: TensorRead<'_>) => analytic::log1p_read_with_pool);
323    delegate_with_pool!(erf_read(input: TensorRead<'_>) => analytic::erf_read_with_pool);
324}
325
326impl TensorStructural for CpuEngine {
327    // Structural
328    fn to_contiguous_read(&mut self, input: TensorRead<'_>) -> crate::Result<Tensor> {
329        // Preserve the host-compact entry policy. Managed copies must enter
330        // the configured execution scope for provider mapping and allocation,
331        // just as views enter it for strided materialization.
332        if input.as_tensor().is_none() {
333            // Only a view counts as staging: an owned read is already compact,
334            // and the copy below is this method's contract, not a hidden one.
335            self.resources.count_read_staging();
336        }
337        let domain = self.shared_allocation_domain().cloned();
338        if matches!(input, TensorRead::Tensor(_)) && input.backend_family().is_none() {
339            self.check_entry("CpuEngine::to_contiguous_read")?;
340            // An owned compact read copies on the host and fans out nowhere.
341            return materialize_tensor_read_in_domain(
342                &mut self.buffers,
343                "CpuEngine::to_contiguous_read",
344                input,
345                domain.as_deref(),
346                threadteam::Exec::serial(),
347            );
348        }
349        self.run_native_with_exec(|exec, buffers| {
350            materialize_tensor_read_in_domain(
351                buffers,
352                "CpuEngine::to_contiguous_read",
353                input,
354                domain.as_deref(),
355                exec,
356            )
357        })
358    }
359
360    fn copy_read_into(&mut self, src: TensorRead<'_>, dst: TensorWrite<'_>) -> crate::Result<()> {
361        self.run_native_with_exec(|exec, _| {
362            copy_tensor_read_into("CpuEngine::copy_read_into", src, dst, exec)
363        })
364    }
365
366    fn transpose_read(&mut self, input: TensorRead<'_>, perm: &[usize]) -> crate::Result<Tensor> {
367        self.run_native_with_exec(|exec, buffers| {
368            structural::transpose_read_with_pool(buffers, input, perm, exec)
369        })
370    }
371
372    fn reshape_read(&mut self, input: TensorRead<'_>, shape: &[usize]) -> crate::Result<Tensor> {
373        self.check_entry("CpuEngine::reshape_read")?;
374        // INVARIANT: a compact input keeps the serial host copy and its typed
375        // refusal for an external payload, without the engine entry, a token or
376        // a pool buffer, unless the copy can pay for a fork. The pre-entry gate
377        // (`reshape_may_use_pool`) asks the same predicate as the in-entry gate
378        // with the cheapest dispatch cost any token could charge, so a `false`
379        // there is final; otherwise the token decides (`reshape_uses_pool`).
380        // `run_native_with_exec` (not a fresh run) keeps the input placement
381        // that the copy puts on the output, as the reshape contract requires.
382        if let TensorRead::Tensor(tensor) = &input {
383            if !structural::reshape_may_use_pool(
384                tensor,
385                self.thread_budget(),
386                self.dispatch_costs(),
387            )? {
388                return structural::reshape(tensor, shape);
389            }
390            return self.run_native_with_exec(|exec, buffers| {
391                if let TensorRead::Tensor(tensor) = &input {
392                    if !structural::reshape_uses_pool(tensor, &exec)? {
393                        return structural::reshape(tensor, shape);
394                    }
395                }
396                structural::reshape_read_with_pool(buffers, input, shape, exec)
397            });
398        }
399        self.run_native_with_exec(|exec, buffers| {
400            structural::reshape_read_with_pool(buffers, input, shape, exec)
401        })
402    }
403
404    fn broadcast_in_dim_read(
405        &mut self,
406        input: TensorRead<'_>,
407        shape: &[usize],
408        dims: &[usize],
409    ) -> crate::Result<Tensor> {
410        self.run_native_with_exec(|exec, buffers| {
411            structural::broadcast_in_dim_read_with_pool(buffers, input, shape, dims, exec)
412        })
413    }
414
415    fn tril_read(
416        &mut self,
417        input: tenferro_tensor::TensorRead<'_>,
418        k: i64,
419    ) -> crate::Result<Tensor> {
420        self.run_native_with_exec(|exec, buffers| {
421            structural::tril_read_with_pool(buffers, input, k, exec)
422        })
423    }
424
425    fn triu_read(
426        &mut self,
427        input: tenferro_tensor::TensorRead<'_>,
428        k: i64,
429    ) -> crate::Result<Tensor> {
430        self.run_native_with_exec(|exec, buffers| {
431            structural::triu_read_with_pool(buffers, input, k, exec)
432        })
433    }
434
435    fn extract_diagonal_read(
436        &mut self,
437        input: tenferro_tensor::TensorRead<'_>,
438        axis_a: usize,
439        axis_b: usize,
440    ) -> crate::Result<Tensor> {
441        self.run_native_with_exec(|exec, buffers| {
442            structural::extract_diagonal_read_with_pool(buffers, input, axis_a, axis_b, exec)
443        })
444    }
445
446    fn embed_diagonal_read(
447        &mut self,
448        input: tenferro_tensor::TensorRead<'_>,
449        axis_a: usize,
450        axis_b: usize,
451    ) -> crate::Result<Tensor> {
452        self.run_native_with_exec(|exec, buffers| {
453            structural::embed_diagonal_read_with_pool(buffers, input, axis_a, axis_b, exec)
454        })
455    }
456
457    delegate_with_pool!(cast(input: &Tensor, to: crate::DType) => structural::cast_with_pool);
458    delegate_with_pool!(cast_read(input: tenferro_tensor::TensorRead<'_>, to: crate::DType) => structural::cast_read_with_pool);
459    delegate_with_pool!(extract_diagonal(input: &Tensor, axis_a: usize, axis_b: usize) => structural::extract_diagonal_with_pool);
460    delegate_with_pool!(embed_diagonal(input: &Tensor, axis_a: usize, axis_b: usize) => structural::embed_diagonal_with_pool);
461    delegate_with_pool!(tril(input: &Tensor, k: i64) => structural::tril_with_pool);
462    delegate_with_pool!(triu(input: &Tensor, k: i64) => structural::triu_with_pool);
463}
464
465impl TensorReduction for CpuEngine {
466    // Reduction
467
468    fn reduce_sum_read(&mut self, input: TensorRead<'_>, axes: &[usize]) -> crate::Result<Tensor> {
469        self.run_native_with_exec(|exec, buffers| {
470            reduction::reduce_sum_read(buffers, input, axes, exec)
471        })
472    }
473
474    fn reduce_sum_squares_read(
475        &mut self,
476        input: TensorRead<'_>,
477        axes: &[usize],
478    ) -> crate::Result<Tensor> {
479        self.run_native_with_exec(|exec, buffers| {
480            reduction::reduce_sum_squares_read(buffers, input, axes, exec)
481        })
482    }
483
484    fn reduce_prod_read(&mut self, input: TensorRead<'_>, axes: &[usize]) -> crate::Result<Tensor> {
485        self.run_native_with_exec(|exec, buffers| {
486            reduction::reduce_prod_read(buffers, input, axes, exec)
487        })
488    }
489
490    fn reduce_max_read(&mut self, input: TensorRead<'_>, axes: &[usize]) -> crate::Result<Tensor> {
491        self.run_native_with_exec(|exec, buffers| {
492            reduction::reduce_max_read(buffers, input, axes, exec)
493        })
494    }
495
496    fn reduce_min_read(&mut self, input: TensorRead<'_>, axes: &[usize]) -> crate::Result<Tensor> {
497        self.run_native_with_exec(|exec, buffers| {
498            reduction::reduce_min_read(buffers, input, axes, exec)
499        })
500    }
501}
502
503impl CpuEngine {
504    fn execute_dot_allocated(
505        &mut self,
506        cache_slot: Option<usize>,
507        lhs: TensorRead<'_>,
508        rhs: TensorRead<'_>,
509        config: &DotGeneralConfig,
510        lhs_conj: bool,
511        rhs_conj: bool,
512    ) -> crate::Result<Tensor> {
513        let dtype = lhs.dtype();
514        if rhs.dtype() != dtype {
515            return Err(crate::Error::dtype_mismatch(
516                "dot_general",
517                dtype,
518                rhs.dtype(),
519            ));
520        }
521        let output_shape = tenferro_tensor::backend::dot_general_output_shape(
522            lhs.shape(),
523            rhs.shape(),
524            config,
525            "dot_general",
526        )?;
527        let accumulation = DotGeneralAccumulation {
528            lhs_conj,
529            rhs_conj,
530            alpha: ContractionScalar::one(dtype)?,
531            beta: ContractionScalar::zero(dtype)?,
532        };
533        if matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64) {
534            return self.run_contraction(|exec, buffers, plans| {
535                crate::contraction::dot_fresh(
536                    exec,
537                    buffers,
538                    plans,
539                    cache_slot,
540                    lhs,
541                    rhs,
542                    config,
543                    accumulation,
544                    output_shape,
545                )
546            });
547        }
548        let mut output = allocate_dot_output(&mut self.buffers, dtype, output_shape)?;
549        self.dot_general_read_into_accum_cached(
550            cache_slot,
551            lhs,
552            rhs,
553            config,
554            accumulation,
555            TensorWrite::from_tensor(&mut output),
556        )?;
557        Ok(output)
558    }
559}
560
561/// The engine's GEMM entries.
562///
563/// These run on the engine's pool within its budget: `threadteam` owns every CPU thread a call
564/// starts, and no external library sizes itself from a process-wide thread setting. A caller that
565/// wants independent GEMMs on lanes of one context forks the bundle that
566/// [`CpuEngine::with_exec`] lends and runs a cpueinsum plan per lane. See the "Lanes" section of
567/// `docs/guides/cpu-execution.md`.
568///
569/// # Examples
570///
571/// ```
572/// use tenferro_cpu::CpuEngine;
573/// use tenferro_tensor::{DotGeneralConfig, Tensor, TensorDot, TensorRead};
574///
575/// let a = Tensor::from_vec_col_major(vec![2, 2], vec![1.0_f64, 3.0, 2.0, 4.0])?;
576/// let identity = Tensor::from_vec_col_major(vec![2, 2], vec![1.0_f64, 0.0, 0.0, 1.0])?;
577/// let config = DotGeneralConfig {
578///     lhs_contracting_dims: [1].as_slice().into(),
579///     rhs_contracting_dims: [0].as_slice().into(),
580///     lhs_batch_dims: [].as_slice().into(),
581///     rhs_batch_dims: [].as_slice().into(),
582/// };
583/// let mut engine = CpuEngine::serial();
584/// let product = engine.dot_general_read(
585///     TensorRead::from_tensor(&a),
586///     TensorRead::from_tensor(&identity),
587///     &config,
588/// )?;
589/// assert_eq!(product.as_slice::<f64>()?, &[1.0, 3.0, 2.0, 4.0]);
590/// # Ok::<(), tenferro_tensor::Error>(())
591/// ```
592impl TensorDot for CpuEngine {
593    fn dot_general_read(
594        &mut self,
595        lhs: TensorRead<'_>,
596        rhs: TensorRead<'_>,
597        config: &DotGeneralConfig,
598    ) -> crate::Result<Tensor> {
599        let slot = eager_dot_cache_slot(
600            self.plans.contraction_capacity(),
601            &lhs,
602            &rhs,
603            config,
604            false,
605            false,
606        );
607        self.execute_dot_allocated(slot, lhs, rhs, config, false, false)
608    }
609
610    fn dot_general_with_conj_read(
611        &mut self,
612        lhs: TensorRead<'_>,
613        rhs: TensorRead<'_>,
614        config: &DotGeneralConfig,
615        lhs_conj: bool,
616        rhs_conj: bool,
617    ) -> crate::Result<Tensor> {
618        // The contraction lane dispatches on the read's view and consumes its
619        // strides, so a conjugated operand needs no staging copy here (#2214).
620        let slot = eager_dot_cache_slot(
621            self.plans.contraction_capacity(),
622            &lhs,
623            &rhs,
624            config,
625            lhs_conj,
626            rhs_conj,
627        );
628        self.execute_dot_allocated(slot, lhs, rhs, config, lhs_conj, rhs_conj)
629    }
630
631    fn dot_general_read_into(
632        &mut self,
633        lhs: TensorRead<'_>,
634        rhs: TensorRead<'_>,
635        config: &DotGeneralConfig,
636        out: TensorWrite<'_>,
637    ) -> crate::Result<()> {
638        let accumulation = DotGeneralAccumulation::overwrite(lhs.dtype())?;
639        self.dot_general_read_into_accum(lhs, rhs, config, accumulation, out)
640    }
641
642    fn dot_general_read_into_accum(
643        &mut self,
644        lhs: TensorRead<'_>,
645        rhs: TensorRead<'_>,
646        config: &DotGeneralConfig,
647        accumulation: DotGeneralAccumulation,
648        out: TensorWrite<'_>,
649    ) -> crate::Result<()> {
650        let slot = eager_dot_cache_slot(
651            self.plans.contraction_capacity(),
652            &lhs,
653            &rhs,
654            config,
655            accumulation.lhs_conj,
656            accumulation.rhs_conj,
657        );
658        self.dot_general_read_into_accum_cached(slot, lhs, rhs, config, accumulation, out)
659    }
660
661    fn dot_general_with_conj(
662        &mut self,
663        lhs: &Tensor,
664        rhs: &Tensor,
665        config: &DotGeneralConfig,
666        lhs_conj: bool,
667        rhs_conj: bool,
668    ) -> crate::Result<Tensor> {
669        let lhs = TensorRead::from_tensor(lhs);
670        let rhs = TensorRead::from_tensor(rhs);
671        let slot = eager_dot_cache_slot(
672            self.plans.contraction_capacity(),
673            &lhs,
674            &rhs,
675            config,
676            lhs_conj,
677            rhs_conj,
678        );
679        self.execute_dot_allocated(slot, lhs, rhs, config, lhs_conj, rhs_conj)
680    }
681}
682
683impl SessionCachedDot for CpuEngine {
684    fn dot_general_cached(
685        &mut self,
686        cache_slot: Option<usize>,
687        lhs: &Tensor,
688        rhs: &Tensor,
689        config: &DotGeneralConfig,
690    ) -> crate::Result<Tensor> {
691        self.execute_dot_allocated(
692            cache_slot,
693            TensorRead::from_tensor(lhs),
694            TensorRead::from_tensor(rhs),
695            config,
696            false,
697            false,
698        )
699    }
700
701    fn dot_general_read_cached(
702        &mut self,
703        cache_slot: Option<usize>,
704        lhs: TensorRead<'_>,
705        rhs: TensorRead<'_>,
706        config: &DotGeneralConfig,
707    ) -> crate::Result<Tensor> {
708        // The caller's slot reaches the same plan cache (keyed by layout, config
709        // and conjugation), so a view operand is consumed instead of staged
710        // (#2214). Do not copy CUDA's policy of ignoring the slot.
711        self.execute_dot_allocated(cache_slot, lhs, rhs, config, false, false)
712    }
713
714    fn dot_general_with_conj_cached(
715        &mut self,
716        cache_slot: Option<usize>,
717        lhs: &Tensor,
718        rhs: &Tensor,
719        config: &DotGeneralConfig,
720        lhs_conj: bool,
721        rhs_conj: bool,
722    ) -> crate::Result<Tensor> {
723        self.execute_dot_allocated(
724            cache_slot,
725            TensorRead::from_tensor(lhs),
726            TensorRead::from_tensor(rhs),
727            config,
728            lhs_conj,
729            rhs_conj,
730        )
731    }
732
733    fn dot_general_with_conj_read_cached(
734        &mut self,
735        cache_slot: Option<usize>,
736        lhs: TensorRead<'_>,
737        rhs: TensorRead<'_>,
738        config: &DotGeneralConfig,
739        lhs_conj: bool,
740        rhs_conj: bool,
741    ) -> crate::Result<Tensor> {
742        self.execute_dot_allocated(cache_slot, lhs, rhs, config, lhs_conj, rhs_conj)
743    }
744
745    fn dot_general_read_into_accum_cached(
746        &mut self,
747        cache_slot: Option<usize>,
748        lhs: TensorRead<'_>,
749        rhs: TensorRead<'_>,
750        config: &DotGeneralConfig,
751        accumulation: DotGeneralAccumulation,
752        out: TensorWrite<'_>,
753    ) -> crate::Result<()> {
754        if matches!(
755            lhs.dtype(),
756            DType::F32 | DType::F64 | DType::C32 | DType::C64
757        ) {
758            return self.run_contraction(|exec, _buffers, plans| {
759                crate::contraction::dot_into(
760                    exec,
761                    plans,
762                    cache_slot,
763                    lhs,
764                    rhs,
765                    config,
766                    accumulation,
767                    out,
768                )
769            });
770        }
771        // Only floating and complex contractions have a CPU numerical route;
772        // integer and Bool operands are rejected before the output is touched.
773        Err(crate::Error::unsupported_dtype(
774            "dot_general",
775            lhs.dtype(),
776            crate::cpu_contraction_unsupported_dtype_message(lhs.dtype()),
777        ))
778    }
779}
780
781impl TensorIndexing for CpuEngine {
782    // Indexing
783    fn gather(
784        &mut self,
785        operand: &Tensor,
786        start_indices: &Tensor,
787        config: &GatherConfig,
788    ) -> crate::Result<Tensor> {
789        self.run_native_with_plans(|exec, buffers, cache| {
790            indexing::gather_with_pool(buffers, cache, operand, start_indices, config, exec)
791        })
792    }
793    fn scatter(
794        &mut self,
795        operand: &Tensor,
796        indices: &Tensor,
797        updates: &Tensor,
798        config: &ScatterConfig,
799    ) -> crate::Result<Tensor> {
800        self.run_native_with_plans(|exec, buffers, cache| {
801            indexing::scatter_with_pool(buffers, cache, operand, indices, updates, config, exec)
802        })
803    }
804    fn dynamic_update_slice_read(
805        &mut self,
806        operand: tenferro_tensor::TensorRead<'_>,
807        update: tenferro_tensor::TensorRead<'_>,
808        starts: tenferro_tensor::TensorRead<'_>,
809    ) -> crate::Result<Tensor> {
810        self.run_native_with_plans(|exec, buffers, cache| {
811            indexing::dynamic_update_slice_read_with_pool(
812                buffers, cache, operand, update, starts, exec,
813            )
814        })
815    }
816    fn scatter_read(
817        &mut self,
818        operand: tenferro_tensor::TensorRead<'_>,
819        indices: tenferro_tensor::TensorRead<'_>,
820        updates: tenferro_tensor::TensorRead<'_>,
821        config: &ScatterConfig,
822    ) -> crate::Result<Tensor> {
823        self.run_native_with_plans(|exec, buffers, cache| {
824            indexing::scatter_read_with_pool(
825                buffers, cache, operand, indices, updates, config, exec,
826            )
827        })
828    }
829    fn slice(&mut self, input: &Tensor, config: &SliceConfig) -> crate::Result<Tensor> {
830        self.run_native_with_exec(|exec, buffers| {
831            indexing::try_slice_with_pool(buffers, input, config, exec)
832        })
833    }
834    fn slice_read(
835        &mut self,
836        input: tenferro_tensor::TensorRead<'_>,
837        config: &SliceConfig,
838    ) -> crate::Result<Tensor> {
839        self.run_native_with_exec(|exec, buffers| {
840            indexing::slice_read_with_pool(buffers, input, config, exec)
841        })
842    }
843    fn dynamic_slice_read(
844        &mut self,
845        input: tenferro_tensor::TensorRead<'_>,
846        starts: tenferro_tensor::TensorRead<'_>,
847        slice_sizes: &[usize],
848    ) -> crate::Result<Tensor> {
849        self.run_native_with_plans(|exec, buffers, cache| {
850            indexing::dynamic_slice_read_with_pool(buffers, cache, input, starts, slice_sizes, exec)
851        })
852    }
853    fn dynamic_slice(
854        &mut self,
855        input: &Tensor,
856        starts: &Tensor,
857        slice_sizes: &[usize],
858    ) -> crate::Result<Tensor> {
859        self.run_native_with_plans(|exec, buffers, cache| {
860            indexing::dynamic_slice_with_pool(buffers, cache, input, starts, slice_sizes, exec)
861        })
862    }
863    fn dynamic_update_slice(
864        &mut self,
865        operand: &Tensor,
866        update: &Tensor,
867        starts: &Tensor,
868    ) -> crate::Result<Tensor> {
869        self.run_native_with_plans(|exec, buffers, cache| {
870            indexing::dynamic_update_slice_with_pool(buffers, cache, operand, update, starts, exec)
871        })
872    }
873    fn pad(&mut self, input: &Tensor, config: &PadConfig) -> crate::Result<Tensor> {
874        self.run_native_with_exec(|exec, buffers| {
875            indexing::try_pad_with_pool(buffers, input, config, exec)
876        })
877    }
878    fn pad_read(
879        &mut self,
880        input: tenferro_tensor::TensorRead<'_>,
881        config: &PadConfig,
882    ) -> crate::Result<Tensor> {
883        self.run_native_with_exec(|exec, buffers| {
884            indexing::pad_read_with_pool(buffers, input, config, exec)
885        })
886    }
887    fn concatenate(&mut self, inputs: &[&Tensor], axis: usize) -> crate::Result<Tensor> {
888        self.run_native_with_exec(|exec, buffers| {
889            indexing::try_concatenate_with_pool(buffers, inputs, axis, exec)
890        })
891    }
892    fn concatenate_read(
893        &mut self,
894        inputs: &[tenferro_tensor::TensorRead<'_>],
895        axis: usize,
896    ) -> crate::Result<Tensor> {
897        self.run_native_with_exec(|exec, buffers| {
898            indexing::concatenate_read_with_pool(buffers, inputs, axis, exec)
899        })
900    }
901    fn reverse(&mut self, input: &Tensor, axes: &[usize]) -> crate::Result<Tensor> {
902        self.run_native_with_exec(|exec, buffers| {
903            indexing::reverse_with_pool(buffers, input, axes, exec)
904        })
905    }
906    fn reverse_read(
907        &mut self,
908        input: tenferro_tensor::TensorRead<'_>,
909        axes: &[usize],
910    ) -> crate::Result<Tensor> {
911        self.run_native_with_exec(|exec, buffers| {
912            indexing::reverse_read_with_pool(buffers, input, axes, exec)
913        })
914    }
915}
916
917/// Typed view canonicalization runs on the engine.
918impl<T, R> tenferro_tensor::TensorViewCanonicalization<T, R> for CpuEngine
919where
920    T: tenferro_tensor::TensorScalar + PoolScalar,
921    R: tenferro_tensor::TensorRank,
922    R::Shape: Send + Sync,
923    R::Strides: Send + Sync,
924{
925    fn to_contiguous(
926        &mut self,
927        view: &tenferro_tensor::TypedTensorView<'_, T, R>,
928    ) -> crate::Result<TypedTensor<T, R>> {
929        self.run_native_with_exec(|exec, buffers| {
930            crate::structural::typed_materialize_view_with_pool(
931                buffers,
932                view,
933                "CpuEngine::to_contiguous",
934                exec,
935            )
936        })
937    }
938
939    fn copy_into(
940        &mut self,
941        src: &tenferro_tensor::TypedTensorView<'_, T, R>,
942        dst: &mut tenferro_tensor::TypedTensorViewMut<'_, T, R>,
943    ) -> crate::Result<()> {
944        self.run_native_with_exec(|exec, _| {
945            crate::structural::typed_copy_view_into(src, dst, "CpuEngine::copy_into", exec)
946        })
947    }
948}
949
950impl TensorBuffer for CpuEngine {
951    fn reclaim_buffer(&mut self, tensor: Tensor) {
952        crate::backend::reclaim_tensor(&mut self.buffers, tensor);
953    }
954}
955
956impl TensorFusion for CpuEngine {
957    fn execute_elementwise_fusion(
958        &mut self,
959        inputs: &[&Tensor],
960        plan: &ElementwiseFusionPlan,
961    ) -> crate::Result<Option<Vec<Tensor>>> {
962        self.run_native_with_exec(|exec, buffers| {
963            tenferro_cpu_fused::elementwise_fusion_with_pool(buffers, inputs, plan, exec)
964        })
965    }
966
967    fn execute_broadcast_multiply(
968        &mut self,
969        lhs: TensorRead<'_>,
970        lhs_shape: &[usize],
971        lhs_dims: &[usize],
972        rhs: TensorRead<'_>,
973        rhs_shape: &[usize],
974        rhs_dims: &[usize],
975    ) -> crate::Result<Option<Tensor>> {
976        self.run_native_with_exec(|exec, buffers| {
977            elementwise::broadcast_multiply_read_with_pool(
978                buffers, lhs, lhs_shape, lhs_dims, rhs, rhs_shape, rhs_dims, exec,
979            )
980        })
981    }
982
983    fn execute_broadcast_multiply_value(
984        &mut self,
985        lhs: TensorRead<'_>,
986        lhs_shape: &[usize],
987        lhs_dims: &[usize],
988        rhs: TensorRead<'_>,
989        rhs_shape: &[usize],
990        rhs_dims: &[usize],
991    ) -> crate::Result<Option<TensorValue>> {
992        self.run_native_with_exec(|exec, buffers| {
993            elementwise::broadcast_multiply_value_with_pool_and_tag(
994                buffers,
995                lhs,
996                lhs_shape,
997                lhs_dims,
998                rhs,
999                rhs_shape,
1000                rhs_dims,
1001                |_| {},
1002                exec,
1003            )
1004        })
1005    }
1006}
1007
1008impl Engine for CpuEngine {
1009    fn gather_read(
1010        &mut self,
1011        operand: tenferro_tensor::TensorRead<'_>,
1012        start_indices: tenferro_tensor::TensorRead<'_>,
1013        config: &GatherConfig,
1014    ) -> crate::Result<Tensor> {
1015        self.run_native_with_plans(|exec, buffers, cache| {
1016            indexing::gather_read_with_pool(buffers, cache, operand, start_indices, config, exec)
1017        })
1018    }
1019    fn vdot_read(&mut self, lhs: TensorRead<'_>, rhs: TensorRead<'_>) -> crate::Result<Tensor> {
1020        tenferro_tensor::backend::validate_vdot_read(&lhs, &rhs)?;
1021        crate::blas1::validate_cpu_read("Engine::vdot_read", &lhs)?;
1022        crate::blas1::validate_cpu_read("Engine::vdot_read", &rhs)?;
1023        let element_count = tenferro_tensor::validate::checked_shape_product(
1024            "Engine::vdot_read",
1025            "shape",
1026            lhs.shape(),
1027        )?;
1028        let (lhs, rhs, rank) = if lhs.shape().len() > 2
1029            && lhs.is_col_major_contiguous()?
1030            && rhs.is_col_major_contiguous()?
1031        {
1032            (
1033                flatten_compact_read(lhs, element_count)?,
1034                flatten_compact_read(rhs, element_count)?,
1035                1,
1036            )
1037        } else {
1038            let rank = lhs.shape().len();
1039            (lhs, rhs, rank)
1040        };
1041        let axes = (0..rank).collect::<tinyvec::TinyVec<[usize; 4]>>();
1042        let config = DotGeneralConfig {
1043            lhs_contracting_dims: axes.clone(),
1044            rhs_contracting_dims: axes,
1045            lhs_batch_dims: Default::default(),
1046            rhs_batch_dims: Default::default(),
1047        };
1048        self.execute_dot_allocated(None, lhs, rhs, &config, true, false)
1049    }
1050
1051    fn norm_squared_read(&mut self, input: TensorRead<'_>) -> crate::Result<Tensor> {
1052        tenferro_tensor::backend::validate_norm_squared_read(&input)?;
1053        crate::blas1::validate_cpu_read("Engine::norm_squared_read", &input)?;
1054        self.run_native_with_exec(|exec, buffers| {
1055            reduction::norm_squared_read(buffers, input, exec)
1056        })
1057    }
1058
1059    fn fused_activation_read(
1060        &mut self,
1061        op: ActivationOp,
1062        input: TensorRead<'_>,
1063    ) -> crate::Result<Option<Tensor>> {
1064        crate::blas1::validate_cpu_read(op.label(), &input)?;
1065        self.run_native_with_exec(|exec, buffers| {
1066            activation::fused_activation_read_with_pool(buffers, op, input, exec)
1067        })
1068    }
1069
1070    fn softmax_read(
1071        &mut self,
1072        input: TensorRead<'_>,
1073        axis: usize,
1074    ) -> crate::Result<Option<Tensor>> {
1075        crate::blas1::validate_cpu_read("softmax", &input)?;
1076        self.run_native_with_exec(|exec, buffers| {
1077            softmax::softmax_read_with_pool(buffers, input, axis, exec)
1078        })
1079    }
1080
1081    fn axpby_read_into_accum(
1082        &mut self,
1083        alpha: ContractionScalar,
1084        x: TensorRead<'_>,
1085        beta: ContractionScalar,
1086        y: TensorWrite<'_>,
1087    ) -> crate::Result<()> {
1088        tenferro_tensor::backend::validate_axpby_read_into_accum(alpha, &x, beta, &y)?;
1089        self.run_native_with_exec(|exec, buffers| {
1090            crate::blas1::axpby_read_into_accum(buffers, alpha, x, beta, y, exec)
1091        })
1092    }
1093
1094    fn native_engine(&mut self) -> Option<tenferro_tensor::NativeSessionRef<'_>> {
1095        // SAFETY: `CpuEngineMarker` is private to this crate, and this is the
1096        // only place a token carrying it is created; it always points to a
1097        // `CpuEngine`, exclusively borrowed for the token lifetime.
1098        Some(unsafe { tenferro_tensor::NativeSessionRef::new::<CpuEngineMarker, _>(self) })
1099    }
1100}
1101
1102#[cfg(test)]
1103mod tests;