tenferro_cpu/
lib.rs

1//! CPU backend, kernels, provider selection, and CPU resource pools.
2//!
3//! # Examples
4//!
5//! ```rust
6//! use tenferro_cpu::CpuBackend;
7//! use tenferro_tensor::{BackendSessionHost, Tensor, TensorRead};
8//!
9//! let mut backend = CpuBackend::new();
10//! let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
11//! let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
12//! let c = backend
13//!     .with_backend_session(|session| {
14//!         session.add_read(TensorRead::from_tensor(&a), TensorRead::from_tensor(&b))
15//!     })??;
16//! assert_eq!(c.as_slice::<f64>().unwrap(), &[4.0, 6.0]);
17//! # Ok::<(), tenferro_tensor::Error>(())
18//! ```
19//!
20//! The deleted one-shot spellings do not compile on the owner or on a session.
21//! Each fixture below fails for that reason and nothing else.
22//!
23//! ```compile_fail
24//! use tenferro_cpu::CpuBackend;
25//! use tenferro_tensor::Tensor;
26//!
27//! let mut backend = CpuBackend::new();
28//! let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0]).unwrap();
29//! let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0]).unwrap();
30//! let _ = backend.add(&a, &b);
31//! ```
32//!
33//! ```compile_fail
34//! use tenferro_cpu::CpuBackend;
35//! use tenferro_tensor::Tensor;
36//!
37//! let mut backend = CpuBackend::new();
38//! let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0]).unwrap();
39//! let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0]).unwrap();
40//! let _ = backend.mul(&a, &b);
41//! ```
42//!
43//! ```compile_fail
44//! use tenferro_cpu::CpuBackend;
45//! use tenferro_tensor::Tensor;
46//!
47//! let mut backend = CpuBackend::new();
48//! let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0]).unwrap();
49//! let _ = backend.exp(&a);
50//! ```
51//!
52//! ```compile_fail
53//! use tenferro_cpu::CpuBackend;
54//! use tenferro_tensor::Tensor;
55//!
56//! let mut backend = CpuBackend::new();
57//! let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0]).unwrap();
58//! let _ = backend.reduce_sum(&a, &[0]);
59//! ```
60//!
61//! ```compile_fail
62//! use tenferro_cpu::CpuBackend;
63//! use tenferro_tensor::Tensor;
64//!
65//! let mut backend = CpuBackend::new();
66//! let a = Tensor::from_vec_col_major(vec![2, 2], vec![1.0_f64, 2.0, 3.0, 4.0]).unwrap();
67//! let _ = backend.transpose(&a, &[1, 0]);
68//! ```
69//!
70//! ```compile_fail
71//! use tenferro_cpu::CpuBackend;
72//! use tenferro_tensor::{DotGeneralConfig, Tensor};
73//!
74//! let mut backend = CpuBackend::new();
75//! let a = Tensor::from_vec_col_major(vec![2, 2], vec![1.0_f64, 2.0, 3.0, 4.0]).unwrap();
76//! let config = DotGeneralConfig {
77//!     lhs_contracting_dims: [1].as_slice().into(),
78//!     rhs_contracting_dims: [0].as_slice().into(),
79//!     lhs_batch_dims: [].as_slice().into(),
80//!     rhs_batch_dims: [].as_slice().into(),
81//! };
82//! let _ = backend.dot_general(&a, &a, &config);
83//! ```
84//!
85//! The owner no longer implements the cache-aware contraction entry, so an
86//! owner-level `BackendCachedDot` bound does not hold either:
87//!
88//! ```compile_fail
89//! use tenferro_cpu::CpuBackend;
90//! use tenferro_tensor::BackendCachedDot;
91//!
92//! fn requires_cached_dot<B: BackendCachedDot>(_backend: &mut B) {}
93//!
94//! let mut backend = CpuBackend::new();
95//! requires_cached_dot(&mut backend);
96//! ```
97#![cfg_attr(docsrs, feature(doc_cfg))]
98// `provider-inject` unit tests deliberately omit the broad default-backend
99// suite below because no fixture has registered its FFI symbols. That makes
100// private helpers referenced only by the broad suite appear unused in this one
101// test build; call-through coverage lives in the registered integration test.
102#![cfg_attr(
103    all(test, feature = "provider-inject"),
104    allow(dead_code, unused_imports)
105)]
106
107/// The Rust scalar type behind a preset variant name a macro received.
108macro_rules! preset_scalar {
109    (F32) => {
110        f32
111    };
112    (F64) => {
113        f64
114    };
115    (I32) => {
116        i32
117    };
118    (I64) => {
119        i64
120    };
121    (Bool) => {
122        bool
123    };
124    (C32) => {
125        num_complex::Complex32
126    };
127    (C64) => {
128        num_complex::Complex64
129    };
130}
131#[cfg(not(any(feature = "cpu-faer", feature = "cpu-blas")))]
132compile_error!("enable at least one CPU backend: cpu-faer or cpu-blas");
133
134#[cfg(all(feature = "provider-inject", not(feature = "cpu-blas")))]
135compile_error!("provider-inject requires cpu-blas");
136
137#[cfg(any(
138    all(feature = "blas-openblas", feature = "blas-accelerate"),
139    all(feature = "blas-openblas", feature = "blas-mkl"),
140    all(feature = "blas-accelerate", feature = "blas-mkl"),
141))]
142compile_error!(
143    "enable at most one explicit BLAS provider feature: blas-openblas, blas-accelerate, or blas-mkl"
144);
145
146#[cfg(all(
147    feature = "provider-inject",
148    any(
149        feature = "blas-openblas",
150        feature = "blas-accelerate",
151        feature = "blas-mkl"
152    )
153))]
154compile_error!("provider-inject cannot be combined with explicit BLAS provider features");
155
156pub mod affinity;
157mod affinity_policy;
158mod analytic;
159mod arbiter;
160pub mod backend;
161mod blas1;
162pub(crate) mod buffer_pool {
163    pub use tenferro_cpu_basic::buffer_pool::*;
164}
165mod capability;
166pub mod context;
167mod domain_executor;
168mod dot_runtime;
169pub(crate) use tenferro_cpu_basic::PooledUninitOutput;
170pub(crate) use tenferro_cpu_basic::{erased_raw_strided_ref, erased_raw_strided_uninit_mut};
171pub(crate) use tenferro_internal_cpu_kernels::elementwise;
172mod batch_policy;
173mod engine;
174mod exec_session;
175mod gemm;
176mod indexed_plan_cache;
177mod indexing;
178#[cfg(feature = "provider-inject")]
179pub mod inject;
180mod placement;
181pub mod provider;
182mod provider_capability;
183mod provider_extensions;
184mod reduction;
185mod resource_domain;
186mod runtime_adapter;
187mod structural;
188mod topology;
189
190use num_complex::{Complex32, Complex64};
191#[cfg(test)]
192use strided_kernel::col_major_strides as kernel_col_major_strides;
193#[cfg(test)]
194use strided_kernel::StridedArray;
195
196use crate::buffer_pool::BufferPool;
197pub(crate) use tenferro_tensor::*;
198
199pub(crate) fn cpu_contraction_unsupported_dtype_message(dtype: DType) -> String {
200    let remedy = matches!(dtype, DType::I32 | DType::I64)
201        .then_some(format!("; convert {dtype:?} to F64 before contraction"));
202    format!(
203        "CPU contraction providers support F32/F64/C32/C64{}",
204        remedy.unwrap_or_default()
205    )
206}
207
208#[cfg(feature = "provider-src")]
209extern crate blas_src as _;
210#[cfg(feature = "provider-inject")]
211extern crate cblas_inject as _;
212#[cfg(feature = "provider-inject")]
213extern crate lapack_inject as _;
214#[cfg(feature = "provider-src")]
215extern crate lapack_src as _;
216
217pub use affinity::{
218    available_parallelism, process_cpu_affinity, process_cpu_affinity_count, CpuAffinityError,
219};
220pub use affinity_policy::{
221    resolve_cpu_affinity, resolve_cpu_affinity_with_override, CpuAffinityInput,
222    CpuAffinityInputError, CpuAffinityPolicy, CpuAffinityResolutionError, CpuAffinitySelection,
223    CpuAffinitySelectionReason,
224};
225pub use backend::execution_scope::{current_cpu_execution, CpuThreadExecution};
226pub use backend::{
227    CpuBackend, CpuBackendError, CpuBackendKind, CpuExecutionInfo, CpuExecutionMode,
228    CpuRuntimeIdentity, ExternalCpuDomainRegistryError,
229};
230pub use batch_policy::{with_batch_policy, CpuBatchPolicy, CpuBatchStrategy, CpuBatchThresholds};
231
232pub use buffer_pool::BufferPoolStats;
233pub use capability::cpu_capabilities;
234pub use context::{CpuContext, CpuContextError, DEFAULT_WORKER_STACK_BYTES};
235pub use domain_executor::{
236    CpuDomainExecutor, CpuDomainExecutorCapabilities, CpuDomainExecutorError, CpuExecutorAffinity,
237    CpuExecutorReentrancy, CpuExecutorShutdown, CpuInnerParallelism, RayonCpuDomainExecutor,
238    ScopedCpuJob, ScopedCpuJobs,
239};
240pub use dot_runtime::{
241    CpuProviderBundle, CpuProviderBundleBuildError, CpuProviderBundleBuilder,
242    CpuProviderBundleInstallError, CpuProviderSlot, GeneralContractionPolicy,
243};
244#[doc(hidden)]
245pub use exec_session::CpuExecSession;
246pub use indexed_plan_cache::IndexedPlanCacheLimits;
247pub use placement::{
248    CpuEngineConstructionError, CpuPlacement, CpuPlacementError, ResolvedCpuPlacement,
249};
250pub use provider::{CpuExecutionContext, ParallelMode};
251pub use provider_capability::{
252    CpuPlacementControl, CpuProviderDomainError, CpuProviderExecutionCapabilities,
253    CpuThreadCountControl,
254};
255pub use resource_domain::{
256    CpuAdmissionMode, CpuDomainOwnership, ExternalCpuDomain, ExternalCpuDomainError,
257};
258pub use runtime_adapter::{
259    runtime_engine_id, runtime_engine_registration, runtime_engine_registration_with_id,
260    runtime_hardware_class,
261};
262/// Ordinary CPU entry points that take a caller-provided destination and the
263/// caller's own arithmetic instead of the typed pool.
264pub use tenferro_internal_cpu_kernels::scalar_ops::{
265    scalar_binary_into, scalar_fold, AddOp, BinaryScalarOp, MulOp, SubOp,
266};
267pub use tenferro_internal_cpu_kernels::{same_variant_pair, same_variant_unary};
268pub use topology::{
269    discover_cpu_topology, CpuId, CpuNode, CpuSet, CpuSetError, CpuTopology, CpuTopologyError,
270    NumaNodeId,
271};
272
273/// Visit a CPU execution session carried by a type-erased backend session.
274///
275/// This is a backend-leaf capability bridge. The exact session marker is checked
276/// before the erased pointer is reconstructed, and the callback cannot return a
277/// borrow of the session, so the borrowed resource lease remains scoped to the
278/// caller's session closure.
279#[doc(hidden)]
280pub fn with_cpu_exec_session<B, R>(
281    session: &mut B,
282    f: impl for<'a> FnOnce(&'a mut CpuExecSession<'a>) -> R,
283) -> Option<R>
284where
285    B: tenferro_tensor::BackendSession + ?Sized,
286{
287    let data = session
288        .native_session()?
289        .into_marked_ptr::<exec_session::CpuExecSessionMarker>()?;
290    // SAFETY: only `CpuExecSession::native_session` creates a token with the
291    // crate-private `CpuExecSessionMarker`, and it points that token at a live
292    // `CpuExecSession`. The token borrowed `*session` exclusively, and this
293    // function keeps holding `session: &mut B` for the whole visit. The
294    // callback is higher-ranked and returns no session borrow, so the
295    // reconstructed reference cannot escape the original session borrow.
296    Some(unsafe { f(data.cast::<CpuExecSession<'static>>().as_mut()) })
297}
298
299/// Invoke a direct faer operation with the parallelism selected by a CPU session.
300///
301/// The `faer::Par` value is scoped to the callback and is derived from the
302/// session's managed thread budget and nesting policy. A non-CPU session, or a
303/// CPU session built without `cpu-faer`, returns a typed unsupported error.
304/// `Par::Seq` remains the portable choice for direct calls outside a session.
305///
306/// # Examples
307///
308/// ```rust
309/// # #[cfg(feature = "cpu-faer")]
310/// # fn example() -> tenferro_tensor::Result<()> {
311/// use tenferro_cpu::{CpuBackend, FaerParallelismExt};
312/// use tenferro_tensor::BackendSessionHost;
313///
314/// let mut backend = CpuBackend::with_threads(2)?;
315/// backend.with_backend_session(|session| {
316///     session.with_faer_parallelism(|parallel| {
317///         let _ = parallel;
318///         Ok(())
319///     })
320/// })??;
321/// # Ok(())
322/// # }
323/// # fn main() {}
324/// ```
325#[cfg(feature = "cpu-faer")]
326#[cfg_attr(docsrs, doc(cfg(feature = "cpu-faer")))]
327pub trait FaerParallelismExt {
328    /// Run a scoped callback with this session's faer parallelism policy.
329    ///
330    /// # Errors
331    ///
332    /// Returns [`tenferro_tensor::Error::Unsupported`] when the session is not
333    /// a CPU/faer execution session, or the callback's own typed error.
334    ///
335    /// # Examples
336    ///
337    /// ```rust
338    /// # #[cfg(feature = "cpu-faer")]
339    /// # fn example(session: &mut dyn tenferro_tensor::BackendSession) -> tenferro_tensor::Result<()> {
340    /// use tenferro_cpu::FaerParallelismExt;
341    /// session.with_faer_parallelism(|parallel| {
342    ///     let _ = parallel;
343    ///     Ok::<_, tenferro_tensor::Error>(())
344    /// })?;
345    /// # Ok(())
346    /// # }
347    /// ```
348    fn with_faer_parallelism(
349        &mut self,
350        callback: impl FnOnce(faer::Par) -> tenferro_tensor::Result<()> + Send,
351    ) -> tenferro_tensor::Result<()>;
352}
353
354#[cfg(feature = "cpu-faer")]
355impl<S> FaerParallelismExt for S
356where
357    S: tenferro_tensor::BackendSession + ?Sized,
358{
359    fn with_faer_parallelism(
360        &mut self,
361        callback: impl FnOnce(faer::Par) -> tenferro_tensor::Result<()> + Send,
362    ) -> tenferro_tensor::Result<()> {
363        with_cpu_exec_session(self, |session| session.with_faer_parallelism(callback))
364            .unwrap_or_else(|| {
365                Err(tenferro_tensor::Error::unsupported(
366                    "with_faer_parallelism",
367                    "selected session is not a CPU/faer execution session",
368                ))
369            })
370    }
371}
372
373// Unit tests exercise the pool-aware kernels through the former convenience
374// names without restoring those names to the production crate surface.
375#[cfg(test)]
376pub(crate) use analytic::pow;
377#[cfg(test)]
378macro_rules! test_elementwise_wrapper {
379    ($name:ident($($arg:ident: $ty:ty),*) => $with_pool:ident) => {
380        pub(crate) fn $name($($arg: $ty),*) -> crate::Result<Tensor> {
381            let mut buffers = BufferPool::new();
382            elementwise::$with_pool(&mut buffers, &strided_kernel::ExecContext::serial(), $($arg),*)
383        }
384    };
385}
386#[cfg(test)]
387test_elementwise_wrapper!(abs(input: &Tensor) => abs_with_pool);
388#[cfg(test)]
389test_elementwise_wrapper!(add(lhs: &Tensor, rhs: &Tensor) => add_with_pool);
390#[cfg(test)]
391test_elementwise_wrapper!(clamp(input: &Tensor, lower: &Tensor, upper: &Tensor) => clamp_with_pool);
392#[cfg(test)]
393test_elementwise_wrapper!(compare(lhs: &Tensor, rhs: &Tensor, dir: &CompareDir) => compare_with_pool);
394#[cfg(test)]
395test_elementwise_wrapper!(conj(input: &Tensor) => conj_with_pool);
396#[cfg(test)]
397test_elementwise_wrapper!(div(lhs: &Tensor, rhs: &Tensor) => div_with_pool);
398#[cfg(test)]
399test_elementwise_wrapper!(maximum(lhs: &Tensor, rhs: &Tensor) => maximum_with_pool);
400#[cfg(test)]
401test_elementwise_wrapper!(minimum(lhs: &Tensor, rhs: &Tensor) => minimum_with_pool);
402#[cfg(test)]
403test_elementwise_wrapper!(mul(lhs: &Tensor, rhs: &Tensor) => mul_with_pool);
404#[cfg(test)]
405test_elementwise_wrapper!(neg(input: &Tensor) => neg_with_pool);
406#[cfg(test)]
407test_elementwise_wrapper!(rem(lhs: &Tensor, rhs: &Tensor) => rem_with_pool);
408#[cfg(test)]
409test_elementwise_wrapper!(select(pred: &Tensor, on_true: &Tensor, on_false: &Tensor) => select_with_pool);
410#[cfg(test)]
411test_elementwise_wrapper!(sign(input: &Tensor) => sign_with_pool);
412#[cfg(test)]
413test_elementwise_wrapper!(sub(lhs: &Tensor, rhs: &Tensor) => sub_with_pool);
414#[cfg(test)]
415pub(crate) use indexing::{dynamic_slice, dynamic_update_slice, gather, pad, scatter};
416#[cfg(test)]
417pub(crate) use reduction::{reduce_max, reduce_min, reduce_prod, reduce_sum, reduce_sum_squares};
418#[cfg(test)]
419pub(crate) use structural::{
420    broadcast_in_dim, embed_diagonal, extract_diagonal, reshape, transpose, tril, triu,
421};
422
423/// Owner-scoped CPU scratch-pool API for operation-family crates.
424///
425/// This module is not an application-facing tensor API. It exists so
426/// operation crates that implement CPU kernels can share `CpuBackend`'s
427/// allocation pool without exposing the pool as a general public contract.
428#[doc(hidden)]
429pub mod linalg_interop {
430    pub use crate::buffer_pool::{BufferPool, PoolScalar};
431    pub use tenferro_cpu_basic::PooledUninitOutput;
432}
433
434#[derive(Debug, thiserror::Error)]
435pub(crate) enum CpuNumericalError {
436    #[error("{op} received a negative integer exponent for dtype {dtype:?}")]
437    NegativeIntegerExponent { op: &'static str, dtype: DType },
438}
439
440pub(crate) fn cpu_negative_integer_exponent(op: &'static str, dtype: DType) -> crate::Error {
441    crate::Error::extension(
442        op,
443        "cpu",
444        ErrorKind::NumericalFailure,
445        CpuNumericalError::NegativeIntegerExponent { op, dtype },
446    )
447}
448
449pub(crate) use tenferro_cpu_basic::{
450    cpu_backend_buffer_error, typed_host_data, typed_view, typed_view_from_view, ConjElem,
451};
452pub(crate) fn materialize_tensor_read_in_domain(
453    buffers: &mut BufferPool,
454    op: &'static str,
455    input: TensorRead<'_>,
456    domain: Option<&dyn SharedTensorAllocationDomain>,
457) -> crate::Result<Tensor> {
458    if let Some(domain) = domain {
459        if input.backend_family().is_some() {
460            return materialize_managed_read(op, input, domain);
461        }
462    }
463    materialize_tensor_read(buffers, op, input)
464}
465
466pub(crate) fn materialize_tensor_read(
467    buffers: &mut BufferPool,
468    op: &'static str,
469    input: TensorRead<'_>,
470) -> crate::Result<Tensor> {
471    match input {
472        TensorRead::Tensor(tensor) => clone_host_tensor_read(op, tensor),
473        TensorRead::View(view) => materialize_tensor_view(buffers, op, view),
474    }
475}
476
477fn materialize_managed_read(
478    op: &'static str,
479    input: TensorRead<'_>,
480    domain: &dyn SharedTensorAllocationDomain,
481) -> crate::Result<Tensor> {
482    if input.placement().memory_kind != MemoryKind::Managed {
483        return Err(Error::host_access(
484            op,
485            HostAccessError::Unsupported { backend: "backend" },
486        ));
487    }
488    match input.allocation_domain() {
489        Some(actual) if actual == domain.id() => {}
490        Some(actual) => {
491            return Err(Error::host_access(
492                op,
493                HostAccessError::ForeignDomain {
494                    expected: domain.id(),
495                    actual,
496                },
497            ))
498        }
499        None => {
500            return Err(Error::host_access(
501                op,
502                HostAccessError::Unsupported { backend: "backend" },
503            ))
504        }
505    }
506    fn copy<T: TensorScalar>(
507        op: &'static str,
508        input: TypedTensorView<'_, T>,
509        domain: &dyn SharedTensorAllocationDomain,
510    ) -> crate::Result<Tensor> {
511        // INVARIANT: semantic snapshots require independent storage, not a new
512        // writable alias. Both mappings remain scoped to this same-domain copy.
513        input.with_host_read(|source| {
514            let mut output = domain.allocate(T::dtype(), input.shape())?;
515            if output.shape() != input.shape()
516                || output.placement().memory_kind != MemoryKind::Managed
517                || TensorRead::from_tensor(&output).allocation_domain() != Some(domain.id())
518            {
519                return Err(Error::runtime_state(
520                    op,
521                    "shared allocator returned incompatible output",
522                ));
523            }
524            let typed = output.as_typed_mut::<T>().ok_or_else(|| {
525                Error::runtime_state(op, "shared allocator returned the wrong dtype")
526            })?;
527            if let Some(buffer) = typed.backend_buffer_mut() {
528                buffer
529                    .map_write()
530                    .map_err(|error| Error::host_access(op, error))?
531                    .copy_from_slice(source)
532                    .map_err(|error| Error::host_access(op, error))?;
533            } else {
534                typed.with_host_write(|target| target.copy_from_slice(source))?;
535            }
536            Ok(output)
537        })?
538    }
539    // Compact managed descriptors cover retained eager/traced values. Strided
540    // managed canonicalization remains an explicit unsupported boundary.
541    match input.tensor_view() {
542        TensorView::F32(view) => copy(op, view, domain),
543        TensorView::F64(view) => copy(op, view, domain),
544        TensorView::I32(view) => copy(op, view, domain),
545        TensorView::I64(view) => copy(op, view, domain),
546        TensorView::Bool(view) => copy(op, view, domain),
547        TensorView::C32(view) => copy(op, view, domain),
548        TensorView::C64(view) => copy(op, view, domain),
549    }
550}
551
552pub(crate) fn copy_tensor_read_into(
553    op: &'static str,
554    src: TensorRead<'_>,
555    dst: TensorWrite<'_>,
556) -> crate::Result<()> {
557    let src_dtype = src.dtype();
558    let dst_dtype = dst.dtype();
559    macro_rules! copy_source {
560        ($variant:ident, $src:expr) => {{
561            let src = $src;
562            match dst {
563                TensorWrite::Tensor(dst)
564                    if dst.dtype()
565                        == <preset_scalar!($variant) as tenferro_tensor::TensorScalar>::dtype() =>
566                {
567                    let dst = dst
568                        .as_typed_mut::<preset_scalar!($variant)>()
569                        .expect("the dtype guard selects this arm");
570                    let mut dst = dst.as_view_mut();
571                    structural::typed_copy_view_into(&src, &mut dst, op)
572                }
573                TensorWrite::View(TensorViewMut::$variant(mut dst)) => {
574                    structural::typed_copy_view_into(&src, &mut dst, op)
575                }
576                _ => Err(crate::Error::dtype_mismatch(op, src_dtype, dst_dtype)),
577            }
578        }};
579    }
580    /// The typed tensor behind a read adapter's tensor, or the refusal this adapter reports.
581    fn read_refusal(tensor: &Tensor) -> crate::Error {
582        crate::Error::unsupported_dtype(
583            "copy_tensor_read_into",
584            tensor.dtype(),
585            "an externally defined payload is not a runtime read",
586        )
587    }
588
589    match src {
590        TensorRead::Tensor(tensor) => match tensor.dtype() {
591            DType::F32 => copy_source!(
592                F32,
593                tensor
594                    .as_typed::<f32>()
595                    .ok_or_else(|| read_refusal(tensor))?
596                    .as_view()
597            ),
598            DType::F64 => copy_source!(
599                F64,
600                tensor
601                    .as_typed::<f64>()
602                    .ok_or_else(|| read_refusal(tensor))?
603                    .as_view()
604            ),
605            DType::I32 => copy_source!(
606                I32,
607                tensor
608                    .as_typed::<i32>()
609                    .ok_or_else(|| read_refusal(tensor))?
610                    .as_view()
611            ),
612            DType::I64 => copy_source!(
613                I64,
614                tensor
615                    .as_typed::<i64>()
616                    .ok_or_else(|| read_refusal(tensor))?
617                    .as_view()
618            ),
619            DType::Bool => copy_source!(
620                Bool,
621                tensor
622                    .as_typed::<bool>()
623                    .ok_or_else(|| read_refusal(tensor))?
624                    .as_view()
625            ),
626            DType::C32 => copy_source!(
627                C32,
628                tensor
629                    .as_typed::<Complex32>()
630                    .ok_or_else(|| read_refusal(tensor))?
631                    .as_view()
632            ),
633            DType::C64 => copy_source!(
634                C64,
635                tensor
636                    .as_typed::<Complex64>()
637                    .ok_or_else(|| read_refusal(tensor))?
638                    .as_view()
639            ),
640            // A caller-owned payload has no compact runtime read.
641            DType::External(_) => Err(read_refusal(tensor)),
642        },
643        TensorRead::View(TensorView::F32(src)) => copy_source!(F32, src),
644        TensorRead::View(TensorView::F64(src)) => copy_source!(F64, src),
645        TensorRead::View(TensorView::I32(src)) => copy_source!(I32, src),
646        TensorRead::View(TensorView::I64(src)) => copy_source!(I64, src),
647        TensorRead::View(TensorView::Bool(src)) => copy_source!(Bool, src),
648        TensorRead::View(TensorView::C32(src)) => copy_source!(C32, src),
649        TensorRead::View(TensorView::C64(src)) => copy_source!(C64, src),
650        // A caller-owned payload is opaque here, so it cannot be copied into a
651    }
652}
653
654/// Clone one owned host tensor into a fresh allocation.
655///
656/// The accepted inputs are: host placement and a preset scalar
657/// (`validate_cpu_host_placement` + `typed_host_data`). `tenferro-ad`'s eager
658/// leaf path states the same acceptance in `cpu_host_owned_read` so it can skip
659/// the session, and `host_leaf_materialization_matches_the_cpu_backend_acceptance`
660/// pins the two against each other. Change both together.
661fn clone_host_tensor_read(op: &'static str, tensor: &Tensor) -> crate::Result<Tensor> {
662    macro_rules! clone_host {
663        ($variant:ident, $tensor:expr) => {{
664            structural::validate_cpu_host_placement(op, "source", $tensor.placement())?;
665            typed_host_data(op, $tensor)?;
666            $tensor
667                .duplicate()
668                .map(Tensor::from_typed::<preset_scalar!($variant)>)
669        }};
670    }
671
672    match tensor.dtype() {
673        DType::F32 => {
674            let tensor = host_typed::<f32>(op, tensor)?;
675            clone_host!(F32, tensor)
676        }
677        DType::F64 => {
678            let tensor = host_typed::<f64>(op, tensor)?;
679            clone_host!(F64, tensor)
680        }
681        DType::I32 => {
682            let tensor = host_typed::<i32>(op, tensor)?;
683            clone_host!(I32, tensor)
684        }
685        DType::I64 => {
686            let tensor = host_typed::<i64>(op, tensor)?;
687            clone_host!(I64, tensor)
688        }
689        DType::Bool => {
690            let tensor = host_typed::<bool>(op, tensor)?;
691            clone_host!(Bool, tensor)
692        }
693        DType::C32 => {
694            let tensor = host_typed::<Complex32>(op, tensor)?;
695            clone_host!(C32, tensor)
696        }
697        DType::C64 => {
698            let tensor = host_typed::<Complex64>(op, tensor)?;
699            clone_host!(C64, tensor)
700        }
701        // A caller-owned payload is a compact host tensor, so a contiguous copy is
702        // the payload itself, copied into storage this value owns. Sharing the
703        // payload would alias the caller's storage instead of copying it.
704        DType::External(_) => Ok(Tensor::external_with_placement(
705            tensor
706                .external_payload()
707                .ok_or_else(|| host_typed_error(op, tensor))?
708                .duplicate(),
709            tensor.placement().clone(),
710        )),
711    }
712}
713
714/// The typed tensor behind `tensor`, or this module's refusal for a dtype it cannot clone.
715///
716/// Callers reach this from a match on `tensor.dtype()`, so `None` means the tag table and the
717/// runtime dtype disagree rather than a caller mistake.
718fn host_typed<'a, T: TensorScalar>(
719    op: &'static str,
720    tensor: &'a Tensor,
721) -> crate::Result<&'a TypedTensor<T>> {
722    tensor
723        .as_typed::<T>()
724        .ok_or_else(|| host_typed_error(op, tensor))
725}
726
727/// The refusal the accessor reports when the tag and the runtime dtype disagree.
728fn host_typed_error(op: &'static str, tensor: &Tensor) -> crate::Error {
729    crate::Error::unsupported_dtype(
730        op,
731        tensor.dtype(),
732        "the CPU host clone requires a preset scalar",
733    )
734}
735
736fn materialize_tensor_view(
737    buffers: &mut BufferPool,
738    op: &'static str,
739    view: TensorView<'_>,
740) -> crate::Result<Tensor> {
741    macro_rules! materialize {
742        ($variant:ident, $view:expr) => {{
743            Ok(Tensor::from_typed::<preset_scalar!($variant)>(
744                structural::typed_materialize_view_with_pool(buffers, &$view, op)?,
745            ))
746        }};
747    }
748
749    match view {
750        TensorView::F32(view) => materialize!(F32, view),
751        TensorView::F64(view) => materialize!(F64, view),
752        TensorView::I32(view) => materialize!(I32, view),
753        TensorView::I64(view) => materialize!(I64, view),
754        TensorView::Bool(view) => materialize!(Bool, view),
755        TensorView::C32(view) => materialize!(C32, view),
756        TensorView::C64(view) => materialize!(C64, view),
757    }
758}
759
760/// Create an output array WITHOUT initializing element values.
761///
762/// # Safety
763/// Caller must write every element before reading. The returned array
764/// contains uninitialized data.
765#[allow(clippy::uninit_vec)]
766#[cfg(test)]
767pub(crate) unsafe fn typed_array_uninit<T>(shape: &[usize]) -> StridedArray<T> {
768    let total: usize = shape.iter().product();
769    let strides = kernel_col_major_strides(shape);
770    let mut data = Vec::with_capacity(total);
771    // SAFETY: test-only helper is used for outputs whose elements are fully overwritten.
772    unsafe { data.set_len(total) };
773    // Invariant: `kernel_col_major_strides(shape)` and `total` describe the
774    // compact column-major array for this validated test output shape.
775    StridedArray::from_parts(data, shape, &strides, 0).expect("column-major output array")
776}
777
778#[cfg(test)]
779pub(crate) fn tensor_from_array<T: Clone + tenferro_tensor::TensorScalar>(
780    array: StridedArray<T>,
781) -> TypedTensor<T> {
782    // Invariant: `StridedArray` owns data whose length matches its validated dimensions.
783    TypedTensor::from_vec_col_major(array.dims().to_vec(), array.into_data())
784        .expect("strided array dimensions match owned data length")
785}
786
787// `provider-inject` owns call-through coverage in the serialized integration
788// fixture, which registers every BLAS symbol before the first operation.  The
789// broad unit suite selects the compiled default backend and therefore must not
790// call an intentionally unregistered injected symbol.
791#[cfg(all(test, not(feature = "provider-inject")))]
792mod tests;
tenferro_cpu/lib.rs

tenferro_cpu/
lib.rs
