Skip to main content

tenferro_cpu/
lib.rs

1//! CPU execution engine and kernels.
2//!
3//! There is one CPU backend: faer-backed `tlinalg` for linalg and
4//! `cpueinsum`/`tcontract` for contraction, linked unconditionally. The crate has
5//! no backend feature; its only two features, `ledger` and `route-report`, are
6//! off by default, gate host-check hooks that are `#[doc(hidden)]`, and change no
7//! numerical path.
8//!
9//! A [`CpuEngine`] owns one [`Pool`] built by the caller (or none) and the
10//! workspace its operations write. Every operation takes the engine as `&mut`:
11//!
12//! ```rust
13//! use tenferro_cpu::{Pool, CpuEngine};
14//! use tenferro_tensor::{Tensor, TensorElementwise, TensorRead};
15//!
16//! let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
17//! let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
18//!
19//! let mut serial = CpuEngine::serial();
20//! let c = serial.add_read(TensorRead::from_tensor(&a), TensorRead::from_tensor(&b))?;
21//! assert_eq!(c.as_slice::<f64>()?, &[4.0, 6.0]);
22//!
23//! let mut parallel = CpuEngine::new(Pool::builder().threads(2).build()?);
24//! let d = parallel.add_read(TensorRead::from_tensor(&a), TensorRead::from_tensor(&b))?;
25//! assert_eq!(d.as_slice::<f64>()?, &[4.0, 6.0]);
26//! # Ok::<(), Box<dyn std::error::Error>>(())
27//! ```
28#![cfg_attr(docsrs, feature(doc_cfg))]
29/// The Rust scalar type behind a preset variant name a macro received.
30macro_rules! preset_scalar {
31    (F32) => {
32        f32
33    };
34    (F64) => {
35        f64
36    };
37    (I32) => {
38        i32
39    };
40    (I64) => {
41        i64
42    };
43    (Bool) => {
44        bool
45    };
46    (C32) => {
47        num_complex::Complex32
48    };
49    (C64) => {
50        num_complex::Complex64
51    };
52}
53mod activation;
54mod analytic;
55pub mod backend;
56mod batch_policy;
57mod blas1;
58mod borrowed_engine;
59pub(crate) mod buffer_pool {
60    pub use tenferro_cpu_basic::buffer_pool::*;
61}
62mod capability;
63mod dot_runtime;
64pub(crate) use tenferro_cpu_basic::PooledUninitOutput;
65pub(crate) use tenferro_cpu_basic::{erased_raw_strided_ref, erased_raw_strided_uninit_mut};
66pub(crate) use tenferro_internal_cpu_kernels::elementwise;
67mod contraction;
68mod engine;
69mod exec_session;
70mod plan_cache;
71#[cfg(feature = "route-report")]
72#[doc(hidden)]
73pub use contraction::take_last_plan_route;
74#[doc(hidden)]
75pub use contraction::ContractionWorkspaces;
76mod gemm;
77mod indexed_plan_cache;
78mod indexing;
79mod operand;
80mod parallel;
81mod provider;
82mod reduction;
83mod runtime_adapter;
84mod simd_math;
85mod softmax;
86mod structural;
87#[cfg(test)]
88pub(crate) mod test_support;
89
90use num_complex::{Complex32, Complex64};
91#[cfg(test)]
92use strided_kernel::col_major_strides as kernel_col_major_strides;
93#[cfg(test)]
94use strided_kernel::StridedArray;
95
96pub(crate) use tenferro_tensor::*;
97use threadteam::Exec;
98
99/// Stable identity of the CPU backend (there is only faer).
100#[doc(hidden)]
101pub fn cpu_provider_id() -> &'static str {
102    "tenferro.cpu.faer"
103}
104
105pub(crate) fn cpu_contraction_unsupported_dtype_message(dtype: DType) -> String {
106    let remedy = matches!(dtype, DType::I32 | DType::I64)
107        .then_some(format!("; convert {dtype:?} to F64 before contraction"));
108    format!(
109        "CPU contraction providers support F32/F64/C32/C64{}",
110        remedy.unwrap_or_default()
111    )
112}
113
114pub use batch_policy::{with_batch_policy, CpuBatchPolicy, CpuBatchStrategy, CpuBatchThresholds};
115
116pub use borrowed_engine::with_borrowed_cpu_engine;
117pub use buffer_pool::{BufferPool, BufferPoolStats};
118pub use capability::cpu_capabilities;
119#[doc(hidden)]
120pub use dot_runtime::{validate_cpu_host_read, validate_cpu_host_write};
121pub use engine::{CpuEngine, CpuEngineError, SerialCtx};
122#[doc(hidden)]
123pub use gemm::GemmAnalysisCache;
124pub use indexed_plan_cache::IndexedPlanCacheLimits;
125pub use plan_cache::CpuPlanCache;
126#[doc(hidden)]
127pub use provider::{reshape_tensor, with_materialized_tensor_read};
128pub use runtime_adapter::{
129    runtime_engine_id, runtime_engine_registration, runtime_engine_registration_with_id,
130    runtime_hardware_class,
131};
132/// Ordinary CPU entry points that take a caller-provided destination and the
133/// caller's own arithmetic instead of the typed pool.
134pub use tenferro_internal_cpu_kernels::scalar_ops::{
135    scalar_binary_into, scalar_fold, AddOp, BinaryScalarOp, MulOp, SubOp,
136};
137pub use tenferro_internal_cpu_kernels::{same_variant_pair, same_variant_unary};
138/// The placement types of the CPU stack, re-exported so callers need no separate
139/// dependency: build a [`Pool`] with [`Pool::builder`] and hand it to
140/// [`CpuEngine::new`]. The token every fan-out entry takes is
141/// `tenferro_cpu::threadteam::Exec`; it is not re-exported at the root because
142/// [`CpuEngine::exec`] returns the different `tcontract_exec::Exec` bundle.
143pub use threadteam;
144pub use threadteam::{Pool, PoolBuilder};
145
146/// Visit the [`CpuEngine`] behind a type-erased engine, if it is one.
147///
148/// This is a backend-leaf capability bridge. The exact engine marker is checked
149/// before the erased pointer is reconstructed, and the callback cannot return a
150/// borrow of the engine, so the reconstructed reference stays within the
151/// caller's borrow of `engine`.
152///
153/// # Examples
154///
155/// ```
156/// use tenferro_cpu::{with_cpu_engine, CpuEngine};
157/// use tenferro_tensor::Engine;
158///
159/// let mut engine = CpuEngine::serial();
160/// let erased: &mut dyn Engine = &mut engine;
161/// assert_eq!(with_cpu_engine(erased, |cpu| cpu.num_threads()), Some(1));
162/// ```
163#[doc(hidden)]
164pub fn with_cpu_engine<B, R>(engine: &mut B, f: impl FnOnce(&mut CpuEngine) -> R) -> Option<R>
165where
166    B: tenferro_tensor::Engine + ?Sized,
167{
168    let data = engine
169        .native_engine()?
170        .into_marked_ptr::<exec_session::CpuEngineMarker>()?;
171    // SAFETY: only `CpuEngine::native_engine` creates a token with the
172    // crate-private `CpuEngineMarker`, and it points that token at a live
173    // `CpuEngine`. The token borrowed `*engine` exclusively, and this function
174    // keeps holding `engine: &mut B` for the whole visit. `R` is chosen by the
175    // caller before the callback's borrow exists, so the reconstructed
176    // reference cannot escape the original borrow.
177    Some(unsafe { f(data.cast::<CpuEngine>().as_mut()) })
178}
179
180/// The faer build this crate links, re-exported for downstream direct calls.
181///
182/// Direct faer work on tenferro-owned host storage borrows column-major slices
183/// and needs no engine handle, so no faer value crosses a tenferro boundary and
184/// a separately pinned `faer` cannot meet a confusing type mismatch (#1939).
185/// Spelling the call through this path still resolves the exact build tenferro
186/// links, which avoids compiling a second faer into one binary; an application
187/// that must control the faer version itself can add a direct dependency
188/// instead. For work that should use the engine's threads, take the engine's
189/// token (`engine.exec()?`): a multi-thread engine's token carries its
190/// `threadteam::Team` (`token.carries_team()`), and `token.team(|team| ..)` hands
191/// faer `Par::Team(team)`. Keep `Par::Seq` otherwise; `Par::rayon` is never used.
192///
193/// No tenferro function takes a faer type in its signature; this is a
194/// re-export for direct calls, not the shape of the tenferro API. Passing a
195/// faer type through a tenferro signature would make faer's version part of
196/// this crate's public surface.
197///
198/// # Examples
199///
200/// ```rust
201/// use tenferro_cpu::faer::{linalg::matmul::matmul, Accum, Mat, MatRef, Par};
202///
203/// let identity = [1.0_f64, 0.0, 0.0, 1.0];
204/// let a = [1.0_f64, 3.0, 2.0, 4.0];
205/// let mut out = Mat::<f64>::zeros(2, 2);
206/// matmul(
207///     out.as_mut(),
208///     Accum::Replace,
209///     MatRef::from_column_major_slice(&identity, 2, 2),
210///     MatRef::from_column_major_slice(&a, 2, 2),
211///     1.0,
212///     Par::Seq,
213/// );
214/// assert_eq!(*out.as_ref().get(1, 0), 3.0);
215/// ```
216pub use faer;
217
218// Unit tests exercise the pool-aware kernels through the former convenience
219// names without restoring those names to the production crate surface.
220#[cfg(test)]
221pub(crate) use analytic::pow;
222#[cfg(test)]
223macro_rules! test_elementwise_wrapper {
224    ($name:ident($($arg:ident: $ty:ty),*) => $with_pool:ident) => {
225        pub(crate) fn $name($($arg: $ty),*) -> crate::Result<Tensor> {
226            let mut buffers = BufferPool::new();
227            elementwise::$with_pool(&mut buffers, $($arg),*, Exec::serial())
228        }
229    };
230}
231#[cfg(test)]
232test_elementwise_wrapper!(abs(input: &Tensor) => abs_with_pool);
233#[cfg(test)]
234test_elementwise_wrapper!(add(lhs: &Tensor, rhs: &Tensor) => add_with_pool);
235#[cfg(test)]
236test_elementwise_wrapper!(clamp(input: &Tensor, lower: &Tensor, upper: &Tensor) => clamp_with_pool);
237#[cfg(test)]
238test_elementwise_wrapper!(compare(lhs: &Tensor, rhs: &Tensor, dir: &CompareDir) => compare_with_pool);
239#[cfg(test)]
240test_elementwise_wrapper!(conj(input: &Tensor) => conj_with_pool);
241#[cfg(test)]
242test_elementwise_wrapper!(div(lhs: &Tensor, rhs: &Tensor) => div_with_pool);
243#[cfg(test)]
244test_elementwise_wrapper!(maximum(lhs: &Tensor, rhs: &Tensor) => maximum_with_pool);
245#[cfg(test)]
246test_elementwise_wrapper!(minimum(lhs: &Tensor, rhs: &Tensor) => minimum_with_pool);
247#[cfg(test)]
248test_elementwise_wrapper!(mul(lhs: &Tensor, rhs: &Tensor) => mul_with_pool);
249#[cfg(test)]
250test_elementwise_wrapper!(neg(input: &Tensor) => neg_with_pool);
251#[cfg(test)]
252test_elementwise_wrapper!(rem(lhs: &Tensor, rhs: &Tensor) => rem_with_pool);
253#[cfg(test)]
254test_elementwise_wrapper!(select(pred: &Tensor, on_true: &Tensor, on_false: &Tensor) => select_with_pool);
255#[cfg(test)]
256test_elementwise_wrapper!(sign(input: &Tensor) => sign_with_pool);
257#[cfg(test)]
258test_elementwise_wrapper!(sub(lhs: &Tensor, rhs: &Tensor) => sub_with_pool);
259#[cfg(test)]
260pub(crate) use indexing::{dynamic_slice, dynamic_update_slice, gather, pad, scatter};
261#[cfg(test)]
262pub(crate) use reduction::{reduce_max, reduce_min, reduce_prod, reduce_sum, reduce_sum_squares};
263#[cfg(test)]
264pub(crate) use structural::{
265    broadcast_in_dim, embed_diagonal, extract_diagonal, reshape, transpose, tril, triu,
266};
267
268/// Engine-scoped CPU scratch-pool API for operation-family crates.
269///
270/// This module is not an application-facing tensor API. It exists so
271/// operation crates that implement CPU kernels can share a `CpuEngine`'s
272/// allocation pool without exposing the pool as a general public contract.
273#[doc(hidden)]
274pub mod linalg_interop {
275    pub use crate::buffer_pool::{BufferPool, PoolScalar};
276    pub use tenferro_cpu_basic::PooledUninitOutput;
277}
278
279#[derive(Debug, thiserror::Error)]
280pub(crate) enum CpuNumericalError {
281    #[error("{op} received a negative integer exponent for dtype {dtype:?}")]
282    NegativeIntegerExponent { op: &'static str, dtype: DType },
283}
284
285pub(crate) fn cpu_negative_integer_exponent(op: &'static str, dtype: DType) -> crate::Error {
286    crate::Error::extension(
287        op,
288        "cpu",
289        ErrorKind::NumericalFailure,
290        CpuNumericalError::NegativeIntegerExponent { op, dtype },
291    )
292}
293
294pub(crate) use tenferro_cpu_basic::{
295    cpu_backend_buffer_error, cpu_non_host_operand_error, shared_view_read_slice, typed_host_data,
296    typed_view, typed_view_from_view,
297};
298pub(crate) fn materialize_tensor_read_in_domain(
299    buffers: &mut BufferPool,
300    op: &'static str,
301    input: TensorRead<'_>,
302    domain: Option<&dyn SharedTensorAllocationDomain>,
303    exec: Exec<'_>,
304) -> crate::Result<Tensor> {
305    if let Some(domain) = domain {
306        if input.backend_family().is_some() {
307            return materialize_managed_read(op, input, domain);
308        }
309    }
310    materialize_tensor_read(buffers, op, input, exec)
311}
312
313pub(crate) fn materialize_tensor_read(
314    buffers: &mut BufferPool,
315    op: &'static str,
316    input: TensorRead<'_>,
317    exec: Exec<'_>,
318) -> crate::Result<Tensor> {
319    match input {
320        TensorRead::Tensor(tensor) => clone_host_tensor_read(op, tensor),
321        TensorRead::View(view) => materialize_tensor_view(buffers, op, view, exec),
322    }
323}
324
325fn materialize_managed_read(
326    op: &'static str,
327    input: TensorRead<'_>,
328    domain: &dyn SharedTensorAllocationDomain,
329) -> crate::Result<Tensor> {
330    if input.placement().memory_kind != MemoryKind::Managed {
331        return Err(Error::host_access(
332            op,
333            HostAccessError::Unsupported { backend: "backend" },
334        ));
335    }
336    match input.allocation_domain() {
337        Some(actual) if actual == domain.id() => {}
338        Some(actual) => {
339            return Err(Error::host_access(
340                op,
341                HostAccessError::ForeignDomain {
342                    expected: domain.id(),
343                    actual,
344                },
345            ))
346        }
347        None => {
348            return Err(Error::host_access(
349                op,
350                HostAccessError::Unsupported { backend: "backend" },
351            ))
352        }
353    }
354    fn copy<T: TensorScalar>(
355        op: &'static str,
356        input: TypedTensorView<'_, T>,
357        domain: &dyn SharedTensorAllocationDomain,
358    ) -> crate::Result<Tensor> {
359        // INVARIANT: semantic snapshots require independent storage, not a new
360        // writable alias. Both mappings remain scoped to this same-domain copy.
361        input.with_host_read(|source| {
362            let mut output = domain.allocate(T::dtype(), input.shape())?;
363            if output.shape() != input.shape()
364                || output.placement().memory_kind != MemoryKind::Managed
365                || TensorRead::from_tensor(&output).allocation_domain() != Some(domain.id())
366            {
367                return Err(Error::runtime_state(
368                    op,
369                    "shared allocator returned incompatible output",
370                ));
371            }
372            let typed = output.as_typed_mut::<T>().ok_or_else(|| {
373                Error::runtime_state(op, "shared allocator returned the wrong dtype")
374            })?;
375            if let Some(buffer) = typed.backend_buffer_mut() {
376                buffer
377                    .map_write()
378                    .map_err(|error| Error::host_access(op, error))?
379                    .copy_from_slice(source)
380                    .map_err(|error| Error::host_access(op, error))?;
381            } else {
382                typed.with_host_write(|target| target.copy_from_slice(source))?;
383            }
384            Ok(output)
385        })?
386    }
387    // Compact managed descriptors cover retained eager/traced values. Strided
388    // managed canonicalization remains an explicit unsupported boundary.
389    match preset_view(op, input)? {
390        TensorView::F32(view) => copy(op, view, domain),
391        TensorView::F64(view) => copy(op, view, domain),
392        TensorView::I32(view) => copy(op, view, domain),
393        TensorView::I64(view) => copy(op, view, domain),
394        TensorView::Bool(view) => copy(op, view, domain),
395        TensorView::C32(view) => copy(op, view, domain),
396        TensorView::C64(view) => copy(op, view, domain),
397    }
398}
399
400/// Borrow `read` as a preset-scalar view, or refuse a caller-owned payload.
401///
402/// [`TensorRead::tensor_view`] answers `None` for a tensor built with
403/// [`Tensor::external`] (#2486); CPU kernels have no implementation for such a
404/// payload, so `op` reports it as an unsupported dtype.
405pub(crate) fn preset_view<'a>(op: &'static str, read: TensorRead<'a>) -> Result<TensorView<'a>> {
406    let dtype = read.dtype();
407    read.tensor_view().ok_or_else(|| {
408        Error::unsupported_dtype(
409            op,
410            dtype,
411            "an externally defined payload has no preset-scalar view on the CPU backend",
412        )
413    })
414}
415
416pub(crate) fn copy_tensor_read_into(
417    op: &'static str,
418    src: TensorRead<'_>,
419    dst: TensorWrite<'_>,
420    exec: Exec<'_>,
421) -> crate::Result<()> {
422    let src_dtype = src.dtype();
423    let dst_dtype = dst.dtype();
424    macro_rules! copy_source {
425        ($variant:ident, $src:expr) => {{
426            let src = $src;
427            match dst {
428                TensorWrite::Tensor(dst)
429                    if dst.dtype()
430                        == <preset_scalar!($variant) as tenferro_tensor::TensorScalar>::dtype() =>
431                {
432                    let dst = dst
433                        .as_typed_mut::<preset_scalar!($variant)>()
434                        .expect("the dtype guard selects this arm");
435                    let mut dst = dst.as_view_mut();
436                    structural::typed_copy_view_into(&src, &mut dst, op, exec)
437                }
438                TensorWrite::View(TensorViewMut::$variant(mut dst)) => {
439                    structural::typed_copy_view_into(&src, &mut dst, op, exec)
440                }
441                _ => Err(crate::Error::dtype_mismatch(op, src_dtype, dst_dtype)),
442            }
443        }};
444    }
445    /// The typed tensor behind a read adapter's tensor, or the refusal this adapter reports.
446    fn read_refusal(tensor: &Tensor) -> crate::Error {
447        crate::Error::unsupported_dtype(
448            "copy_tensor_read_into",
449            tensor.dtype(),
450            "an externally defined payload is not a runtime read",
451        )
452    }
453
454    match src {
455        TensorRead::Tensor(tensor) => match tensor.dtype() {
456            DType::F32 => copy_source!(
457                F32,
458                tensor
459                    .as_typed::<f32>()
460                    .ok_or_else(|| read_refusal(tensor))?
461                    .as_view()
462            ),
463            DType::F64 => copy_source!(
464                F64,
465                tensor
466                    .as_typed::<f64>()
467                    .ok_or_else(|| read_refusal(tensor))?
468                    .as_view()
469            ),
470            DType::I32 => copy_source!(
471                I32,
472                tensor
473                    .as_typed::<i32>()
474                    .ok_or_else(|| read_refusal(tensor))?
475                    .as_view()
476            ),
477            DType::I64 => copy_source!(
478                I64,
479                tensor
480                    .as_typed::<i64>()
481                    .ok_or_else(|| read_refusal(tensor))?
482                    .as_view()
483            ),
484            DType::Bool => copy_source!(
485                Bool,
486                tensor
487                    .as_typed::<bool>()
488                    .ok_or_else(|| read_refusal(tensor))?
489                    .as_view()
490            ),
491            DType::C32 => copy_source!(
492                C32,
493                tensor
494                    .as_typed::<Complex32>()
495                    .ok_or_else(|| read_refusal(tensor))?
496                    .as_view()
497            ),
498            DType::C64 => copy_source!(
499                C64,
500                tensor
501                    .as_typed::<Complex64>()
502                    .ok_or_else(|| read_refusal(tensor))?
503                    .as_view()
504            ),
505            // A caller-owned payload has no compact runtime read.
506            DType::External(_) => Err(read_refusal(tensor)),
507        },
508        TensorRead::View(TensorView::F32(src)) => copy_source!(F32, src),
509        TensorRead::View(TensorView::F64(src)) => copy_source!(F64, src),
510        TensorRead::View(TensorView::I32(src)) => copy_source!(I32, src),
511        TensorRead::View(TensorView::I64(src)) => copy_source!(I64, src),
512        TensorRead::View(TensorView::Bool(src)) => copy_source!(Bool, src),
513        TensorRead::View(TensorView::C32(src)) => copy_source!(C32, src),
514        TensorRead::View(TensorView::C64(src)) => copy_source!(C64, src),
515        // A caller-owned payload is opaque here, so it cannot be copied into a
516    }
517}
518
519/// Clone one owned host tensor into a fresh allocation.
520///
521/// The accepted inputs are: host placement and a preset scalar
522/// (`validate_cpu_host_view` + `typed_host_data`). `tenferro-ad`'s eager
523/// leaf path states the same acceptance in `cpu_host_owned_read` so it can skip
524/// the session, and `host_leaf_materialization_matches_the_cpu_backend_acceptance`
525/// pins the two against each other. Change both together.
526fn clone_host_tensor_read(op: &'static str, tensor: &Tensor) -> crate::Result<Tensor> {
527    macro_rules! clone_host {
528        ($variant:ident, $tensor:expr) => {{
529            structural::validate_cpu_host_view(op, "source", &$tensor.as_view())?;
530            typed_host_data(op, $tensor)?;
531            $tensor
532                .duplicate()
533                .map(Tensor::from_typed::<preset_scalar!($variant)>)
534        }};
535    }
536
537    match tensor.dtype() {
538        DType::F32 => {
539            let tensor = host_typed::<f32>(op, tensor)?;
540            clone_host!(F32, tensor)
541        }
542        DType::F64 => {
543            let tensor = host_typed::<f64>(op, tensor)?;
544            clone_host!(F64, tensor)
545        }
546        DType::I32 => {
547            let tensor = host_typed::<i32>(op, tensor)?;
548            clone_host!(I32, tensor)
549        }
550        DType::I64 => {
551            let tensor = host_typed::<i64>(op, tensor)?;
552            clone_host!(I64, tensor)
553        }
554        DType::Bool => {
555            let tensor = host_typed::<bool>(op, tensor)?;
556            clone_host!(Bool, tensor)
557        }
558        DType::C32 => {
559            let tensor = host_typed::<Complex32>(op, tensor)?;
560            clone_host!(C32, tensor)
561        }
562        DType::C64 => {
563            let tensor = host_typed::<Complex64>(op, tensor)?;
564            clone_host!(C64, tensor)
565        }
566        // A caller-owned payload is a compact host tensor, so a contiguous copy is
567        // the payload itself, copied into storage this value owns. Sharing the
568        // payload would alias the caller's storage instead of copying it.
569        DType::External(_) => Ok(Tensor::external_with_placement(
570            tensor
571                .external_payload()
572                .ok_or_else(|| host_typed_error(op, tensor))?
573                .duplicate(),
574            tensor.placement().clone(),
575        )),
576    }
577}
578
579/// The typed tensor behind `tensor`, or this module's refusal for a dtype it cannot clone.
580///
581/// Callers reach this from a match on `tensor.dtype()`, so `None` means the tag table and the
582/// runtime dtype disagree rather than a caller mistake.
583fn host_typed<'a, T: TensorScalar>(
584    op: &'static str,
585    tensor: &'a Tensor,
586) -> crate::Result<&'a TypedTensor<T>> {
587    tensor
588        .as_typed::<T>()
589        .ok_or_else(|| host_typed_error(op, tensor))
590}
591
592/// The refusal the accessor reports when the tag and the runtime dtype disagree.
593fn host_typed_error(op: &'static str, tensor: &Tensor) -> crate::Error {
594    crate::Error::unsupported_dtype(
595        op,
596        tensor.dtype(),
597        "the CPU host clone requires a preset scalar",
598    )
599}
600
601fn materialize_tensor_view(
602    buffers: &mut BufferPool,
603    op: &'static str,
604    view: TensorView<'_>,
605    exec: Exec<'_>,
606) -> crate::Result<Tensor> {
607    macro_rules! materialize {
608        ($variant:ident, $view:expr) => {{
609            Ok(Tensor::from_typed::<preset_scalar!($variant)>(
610                structural::typed_materialize_view_with_pool(buffers, &$view, op, exec)?,
611            ))
612        }};
613    }
614
615    match view {
616        TensorView::F32(view) => materialize!(F32, view),
617        TensorView::F64(view) => materialize!(F64, view),
618        TensorView::I32(view) => materialize!(I32, view),
619        TensorView::I64(view) => materialize!(I64, view),
620        TensorView::Bool(view) => materialize!(Bool, view),
621        TensorView::C32(view) => materialize!(C32, view),
622        TensorView::C64(view) => materialize!(C64, view),
623    }
624}
625
626/// Create an output array WITHOUT initializing element values.
627///
628/// # Safety
629/// Caller must write every element before reading. The returned array
630/// contains uninitialized data.
631#[allow(clippy::uninit_vec)]
632#[cfg(test)]
633pub(crate) unsafe fn typed_array_uninit<T>(shape: &[usize]) -> StridedArray<T> {
634    let total: usize = shape.iter().product();
635    let strides = kernel_col_major_strides(shape);
636    let mut data = Vec::with_capacity(total);
637    // SAFETY: test-only helper is used for outputs whose elements are fully overwritten.
638    unsafe { data.set_len(total) };
639    // Invariant: `kernel_col_major_strides(shape)` and `total` describe the
640    // compact column-major array for this validated test output shape.
641    StridedArray::from_parts(data, shape, &strides, 0).expect("column-major output array")
642}
643
644#[cfg(test)]
645pub(crate) fn tensor_from_array<T: Clone + tenferro_tensor::TensorScalar>(
646    array: StridedArray<T>,
647) -> TypedTensor<T> {
648    // Invariant: `StridedArray` owns data whose length matches its validated dimensions.
649    TypedTensor::from_vec_col_major(array.dims().to_vec(), array.into_data())
650        .expect("strided array dimensions match owned data length")
651}
652
653#[cfg(test)]
654mod tests;