1#![cfg_attr(docsrs, feature(doc_cfg))]
29macro_rules! preset_scalar {
31 (F32) => {
32 f32
33 };
34 (F64) => {
35 f64
36 };
37 (I32) => {
38 i32
39 };
40 (I64) => {
41 i64
42 };
43 (Bool) => {
44 bool
45 };
46 (C32) => {
47 num_complex::Complex32
48 };
49 (C64) => {
50 num_complex::Complex64
51 };
52}
53mod activation;
54mod analytic;
55pub mod backend;
56mod batch_policy;
57mod blas1;
58mod borrowed_engine;
59pub(crate) mod buffer_pool {
60 pub use tenferro_cpu_basic::buffer_pool::*;
61}
62mod capability;
63mod dot_runtime;
64pub(crate) use tenferro_cpu_basic::PooledUninitOutput;
65pub(crate) use tenferro_cpu_basic::{erased_raw_strided_ref, erased_raw_strided_uninit_mut};
66pub(crate) use tenferro_internal_cpu_kernels::elementwise;
67mod contraction;
68mod engine;
69mod exec_session;
70mod plan_cache;
71#[cfg(feature = "route-report")]
72#[doc(hidden)]
73pub use contraction::take_last_plan_route;
74#[doc(hidden)]
75pub use contraction::ContractionWorkspaces;
76mod gemm;
77mod indexed_plan_cache;
78mod indexing;
79mod operand;
80mod parallel;
81mod provider;
82mod reduction;
83mod runtime_adapter;
84mod simd_math;
85mod softmax;
86mod structural;
87#[cfg(test)]
88pub(crate) mod test_support;
89
90use num_complex::{Complex32, Complex64};
91#[cfg(test)]
92use strided_kernel::col_major_strides as kernel_col_major_strides;
93#[cfg(test)]
94use strided_kernel::StridedArray;
95
96pub(crate) use tenferro_tensor::*;
97use threadteam::Exec;
98
99#[doc(hidden)]
101pub fn cpu_provider_id() -> &'static str {
102 "tenferro.cpu.faer"
103}
104
105pub(crate) fn cpu_contraction_unsupported_dtype_message(dtype: DType) -> String {
106 let remedy = matches!(dtype, DType::I32 | DType::I64)
107 .then_some(format!("; convert {dtype:?} to F64 before contraction"));
108 format!(
109 "CPU contraction providers support F32/F64/C32/C64{}",
110 remedy.unwrap_or_default()
111 )
112}
113
114pub use batch_policy::{with_batch_policy, CpuBatchPolicy, CpuBatchStrategy, CpuBatchThresholds};
115
116pub use borrowed_engine::with_borrowed_cpu_engine;
117pub use buffer_pool::{BufferPool, BufferPoolStats};
118pub use capability::cpu_capabilities;
119#[doc(hidden)]
120pub use dot_runtime::{validate_cpu_host_read, validate_cpu_host_write};
121pub use engine::{CpuEngine, CpuEngineError, SerialCtx};
122#[doc(hidden)]
123pub use gemm::GemmAnalysisCache;
124pub use indexed_plan_cache::IndexedPlanCacheLimits;
125pub use plan_cache::CpuPlanCache;
126#[doc(hidden)]
127pub use provider::{reshape_tensor, with_materialized_tensor_read};
128pub use runtime_adapter::{
129 runtime_engine_id, runtime_engine_registration, runtime_engine_registration_with_id,
130 runtime_hardware_class,
131};
132pub use tenferro_internal_cpu_kernels::scalar_ops::{
135 scalar_binary_into, scalar_fold, AddOp, BinaryScalarOp, MulOp, SubOp,
136};
137pub use tenferro_internal_cpu_kernels::{same_variant_pair, same_variant_unary};
138pub use threadteam;
144pub use threadteam::{Pool, PoolBuilder};
145
146#[doc(hidden)]
164pub fn with_cpu_engine<B, R>(engine: &mut B, f: impl FnOnce(&mut CpuEngine) -> R) -> Option<R>
165where
166 B: tenferro_tensor::Engine + ?Sized,
167{
168 let data = engine
169 .native_engine()?
170 .into_marked_ptr::<exec_session::CpuEngineMarker>()?;
171 Some(unsafe { f(data.cast::<CpuEngine>().as_mut()) })
178}
179
180pub use faer;
217
218#[cfg(test)]
221pub(crate) use analytic::pow;
222#[cfg(test)]
223macro_rules! test_elementwise_wrapper {
224 ($name:ident($($arg:ident: $ty:ty),*) => $with_pool:ident) => {
225 pub(crate) fn $name($($arg: $ty),*) -> crate::Result<Tensor> {
226 let mut buffers = BufferPool::new();
227 elementwise::$with_pool(&mut buffers, $($arg),*, Exec::serial())
228 }
229 };
230}
231#[cfg(test)]
232test_elementwise_wrapper!(abs(input: &Tensor) => abs_with_pool);
233#[cfg(test)]
234test_elementwise_wrapper!(add(lhs: &Tensor, rhs: &Tensor) => add_with_pool);
235#[cfg(test)]
236test_elementwise_wrapper!(clamp(input: &Tensor, lower: &Tensor, upper: &Tensor) => clamp_with_pool);
237#[cfg(test)]
238test_elementwise_wrapper!(compare(lhs: &Tensor, rhs: &Tensor, dir: &CompareDir) => compare_with_pool);
239#[cfg(test)]
240test_elementwise_wrapper!(conj(input: &Tensor) => conj_with_pool);
241#[cfg(test)]
242test_elementwise_wrapper!(div(lhs: &Tensor, rhs: &Tensor) => div_with_pool);
243#[cfg(test)]
244test_elementwise_wrapper!(maximum(lhs: &Tensor, rhs: &Tensor) => maximum_with_pool);
245#[cfg(test)]
246test_elementwise_wrapper!(minimum(lhs: &Tensor, rhs: &Tensor) => minimum_with_pool);
247#[cfg(test)]
248test_elementwise_wrapper!(mul(lhs: &Tensor, rhs: &Tensor) => mul_with_pool);
249#[cfg(test)]
250test_elementwise_wrapper!(neg(input: &Tensor) => neg_with_pool);
251#[cfg(test)]
252test_elementwise_wrapper!(rem(lhs: &Tensor, rhs: &Tensor) => rem_with_pool);
253#[cfg(test)]
254test_elementwise_wrapper!(select(pred: &Tensor, on_true: &Tensor, on_false: &Tensor) => select_with_pool);
255#[cfg(test)]
256test_elementwise_wrapper!(sign(input: &Tensor) => sign_with_pool);
257#[cfg(test)]
258test_elementwise_wrapper!(sub(lhs: &Tensor, rhs: &Tensor) => sub_with_pool);
259#[cfg(test)]
260pub(crate) use indexing::{dynamic_slice, dynamic_update_slice, gather, pad, scatter};
261#[cfg(test)]
262pub(crate) use reduction::{reduce_max, reduce_min, reduce_prod, reduce_sum, reduce_sum_squares};
263#[cfg(test)]
264pub(crate) use structural::{
265 broadcast_in_dim, embed_diagonal, extract_diagonal, reshape, transpose, tril, triu,
266};
267
268#[doc(hidden)]
274pub mod linalg_interop {
275 pub use crate::buffer_pool::{BufferPool, PoolScalar};
276 pub use tenferro_cpu_basic::PooledUninitOutput;
277}
278
279#[derive(Debug, thiserror::Error)]
280pub(crate) enum CpuNumericalError {
281 #[error("{op} received a negative integer exponent for dtype {dtype:?}")]
282 NegativeIntegerExponent { op: &'static str, dtype: DType },
283}
284
285pub(crate) fn cpu_negative_integer_exponent(op: &'static str, dtype: DType) -> crate::Error {
286 crate::Error::extension(
287 op,
288 "cpu",
289 ErrorKind::NumericalFailure,
290 CpuNumericalError::NegativeIntegerExponent { op, dtype },
291 )
292}
293
294pub(crate) use tenferro_cpu_basic::{
295 cpu_backend_buffer_error, cpu_non_host_operand_error, shared_view_read_slice, typed_host_data,
296 typed_view, typed_view_from_view,
297};
298pub(crate) fn materialize_tensor_read_in_domain(
299 buffers: &mut BufferPool,
300 op: &'static str,
301 input: TensorRead<'_>,
302 domain: Option<&dyn SharedTensorAllocationDomain>,
303 exec: Exec<'_>,
304) -> crate::Result<Tensor> {
305 if let Some(domain) = domain {
306 if input.backend_family().is_some() {
307 return materialize_managed_read(op, input, domain);
308 }
309 }
310 materialize_tensor_read(buffers, op, input, exec)
311}
312
313pub(crate) fn materialize_tensor_read(
314 buffers: &mut BufferPool,
315 op: &'static str,
316 input: TensorRead<'_>,
317 exec: Exec<'_>,
318) -> crate::Result<Tensor> {
319 match input {
320 TensorRead::Tensor(tensor) => clone_host_tensor_read(op, tensor),
321 TensorRead::View(view) => materialize_tensor_view(buffers, op, view, exec),
322 }
323}
324
325fn materialize_managed_read(
326 op: &'static str,
327 input: TensorRead<'_>,
328 domain: &dyn SharedTensorAllocationDomain,
329) -> crate::Result<Tensor> {
330 if input.placement().memory_kind != MemoryKind::Managed {
331 return Err(Error::host_access(
332 op,
333 HostAccessError::Unsupported { backend: "backend" },
334 ));
335 }
336 match input.allocation_domain() {
337 Some(actual) if actual == domain.id() => {}
338 Some(actual) => {
339 return Err(Error::host_access(
340 op,
341 HostAccessError::ForeignDomain {
342 expected: domain.id(),
343 actual,
344 },
345 ))
346 }
347 None => {
348 return Err(Error::host_access(
349 op,
350 HostAccessError::Unsupported { backend: "backend" },
351 ))
352 }
353 }
354 fn copy<T: TensorScalar>(
355 op: &'static str,
356 input: TypedTensorView<'_, T>,
357 domain: &dyn SharedTensorAllocationDomain,
358 ) -> crate::Result<Tensor> {
359 input.with_host_read(|source| {
362 let mut output = domain.allocate(T::dtype(), input.shape())?;
363 if output.shape() != input.shape()
364 || output.placement().memory_kind != MemoryKind::Managed
365 || TensorRead::from_tensor(&output).allocation_domain() != Some(domain.id())
366 {
367 return Err(Error::runtime_state(
368 op,
369 "shared allocator returned incompatible output",
370 ));
371 }
372 let typed = output.as_typed_mut::<T>().ok_or_else(|| {
373 Error::runtime_state(op, "shared allocator returned the wrong dtype")
374 })?;
375 if let Some(buffer) = typed.backend_buffer_mut() {
376 buffer
377 .map_write()
378 .map_err(|error| Error::host_access(op, error))?
379 .copy_from_slice(source)
380 .map_err(|error| Error::host_access(op, error))?;
381 } else {
382 typed.with_host_write(|target| target.copy_from_slice(source))?;
383 }
384 Ok(output)
385 })?
386 }
387 match preset_view(op, input)? {
390 TensorView::F32(view) => copy(op, view, domain),
391 TensorView::F64(view) => copy(op, view, domain),
392 TensorView::I32(view) => copy(op, view, domain),
393 TensorView::I64(view) => copy(op, view, domain),
394 TensorView::Bool(view) => copy(op, view, domain),
395 TensorView::C32(view) => copy(op, view, domain),
396 TensorView::C64(view) => copy(op, view, domain),
397 }
398}
399
400pub(crate) fn preset_view<'a>(op: &'static str, read: TensorRead<'a>) -> Result<TensorView<'a>> {
406 let dtype = read.dtype();
407 read.tensor_view().ok_or_else(|| {
408 Error::unsupported_dtype(
409 op,
410 dtype,
411 "an externally defined payload has no preset-scalar view on the CPU backend",
412 )
413 })
414}
415
416pub(crate) fn copy_tensor_read_into(
417 op: &'static str,
418 src: TensorRead<'_>,
419 dst: TensorWrite<'_>,
420 exec: Exec<'_>,
421) -> crate::Result<()> {
422 let src_dtype = src.dtype();
423 let dst_dtype = dst.dtype();
424 macro_rules! copy_source {
425 ($variant:ident, $src:expr) => {{
426 let src = $src;
427 match dst {
428 TensorWrite::Tensor(dst)
429 if dst.dtype()
430 == <preset_scalar!($variant) as tenferro_tensor::TensorScalar>::dtype() =>
431 {
432 let dst = dst
433 .as_typed_mut::<preset_scalar!($variant)>()
434 .expect("the dtype guard selects this arm");
435 let mut dst = dst.as_view_mut();
436 structural::typed_copy_view_into(&src, &mut dst, op, exec)
437 }
438 TensorWrite::View(TensorViewMut::$variant(mut dst)) => {
439 structural::typed_copy_view_into(&src, &mut dst, op, exec)
440 }
441 _ => Err(crate::Error::dtype_mismatch(op, src_dtype, dst_dtype)),
442 }
443 }};
444 }
445 fn read_refusal(tensor: &Tensor) -> crate::Error {
447 crate::Error::unsupported_dtype(
448 "copy_tensor_read_into",
449 tensor.dtype(),
450 "an externally defined payload is not a runtime read",
451 )
452 }
453
454 match src {
455 TensorRead::Tensor(tensor) => match tensor.dtype() {
456 DType::F32 => copy_source!(
457 F32,
458 tensor
459 .as_typed::<f32>()
460 .ok_or_else(|| read_refusal(tensor))?
461 .as_view()
462 ),
463 DType::F64 => copy_source!(
464 F64,
465 tensor
466 .as_typed::<f64>()
467 .ok_or_else(|| read_refusal(tensor))?
468 .as_view()
469 ),
470 DType::I32 => copy_source!(
471 I32,
472 tensor
473 .as_typed::<i32>()
474 .ok_or_else(|| read_refusal(tensor))?
475 .as_view()
476 ),
477 DType::I64 => copy_source!(
478 I64,
479 tensor
480 .as_typed::<i64>()
481 .ok_or_else(|| read_refusal(tensor))?
482 .as_view()
483 ),
484 DType::Bool => copy_source!(
485 Bool,
486 tensor
487 .as_typed::<bool>()
488 .ok_or_else(|| read_refusal(tensor))?
489 .as_view()
490 ),
491 DType::C32 => copy_source!(
492 C32,
493 tensor
494 .as_typed::<Complex32>()
495 .ok_or_else(|| read_refusal(tensor))?
496 .as_view()
497 ),
498 DType::C64 => copy_source!(
499 C64,
500 tensor
501 .as_typed::<Complex64>()
502 .ok_or_else(|| read_refusal(tensor))?
503 .as_view()
504 ),
505 DType::External(_) => Err(read_refusal(tensor)),
507 },
508 TensorRead::View(TensorView::F32(src)) => copy_source!(F32, src),
509 TensorRead::View(TensorView::F64(src)) => copy_source!(F64, src),
510 TensorRead::View(TensorView::I32(src)) => copy_source!(I32, src),
511 TensorRead::View(TensorView::I64(src)) => copy_source!(I64, src),
512 TensorRead::View(TensorView::Bool(src)) => copy_source!(Bool, src),
513 TensorRead::View(TensorView::C32(src)) => copy_source!(C32, src),
514 TensorRead::View(TensorView::C64(src)) => copy_source!(C64, src),
515 }
517}
518
519fn clone_host_tensor_read(op: &'static str, tensor: &Tensor) -> crate::Result<Tensor> {
527 macro_rules! clone_host {
528 ($variant:ident, $tensor:expr) => {{
529 structural::validate_cpu_host_view(op, "source", &$tensor.as_view())?;
530 typed_host_data(op, $tensor)?;
531 $tensor
532 .duplicate()
533 .map(Tensor::from_typed::<preset_scalar!($variant)>)
534 }};
535 }
536
537 match tensor.dtype() {
538 DType::F32 => {
539 let tensor = host_typed::<f32>(op, tensor)?;
540 clone_host!(F32, tensor)
541 }
542 DType::F64 => {
543 let tensor = host_typed::<f64>(op, tensor)?;
544 clone_host!(F64, tensor)
545 }
546 DType::I32 => {
547 let tensor = host_typed::<i32>(op, tensor)?;
548 clone_host!(I32, tensor)
549 }
550 DType::I64 => {
551 let tensor = host_typed::<i64>(op, tensor)?;
552 clone_host!(I64, tensor)
553 }
554 DType::Bool => {
555 let tensor = host_typed::<bool>(op, tensor)?;
556 clone_host!(Bool, tensor)
557 }
558 DType::C32 => {
559 let tensor = host_typed::<Complex32>(op, tensor)?;
560 clone_host!(C32, tensor)
561 }
562 DType::C64 => {
563 let tensor = host_typed::<Complex64>(op, tensor)?;
564 clone_host!(C64, tensor)
565 }
566 DType::External(_) => Ok(Tensor::external_with_placement(
570 tensor
571 .external_payload()
572 .ok_or_else(|| host_typed_error(op, tensor))?
573 .duplicate(),
574 tensor.placement().clone(),
575 )),
576 }
577}
578
579fn host_typed<'a, T: TensorScalar>(
584 op: &'static str,
585 tensor: &'a Tensor,
586) -> crate::Result<&'a TypedTensor<T>> {
587 tensor
588 .as_typed::<T>()
589 .ok_or_else(|| host_typed_error(op, tensor))
590}
591
592fn host_typed_error(op: &'static str, tensor: &Tensor) -> crate::Error {
594 crate::Error::unsupported_dtype(
595 op,
596 tensor.dtype(),
597 "the CPU host clone requires a preset scalar",
598 )
599}
600
601fn materialize_tensor_view(
602 buffers: &mut BufferPool,
603 op: &'static str,
604 view: TensorView<'_>,
605 exec: Exec<'_>,
606) -> crate::Result<Tensor> {
607 macro_rules! materialize {
608 ($variant:ident, $view:expr) => {{
609 Ok(Tensor::from_typed::<preset_scalar!($variant)>(
610 structural::typed_materialize_view_with_pool(buffers, &$view, op, exec)?,
611 ))
612 }};
613 }
614
615 match view {
616 TensorView::F32(view) => materialize!(F32, view),
617 TensorView::F64(view) => materialize!(F64, view),
618 TensorView::I32(view) => materialize!(I32, view),
619 TensorView::I64(view) => materialize!(I64, view),
620 TensorView::Bool(view) => materialize!(Bool, view),
621 TensorView::C32(view) => materialize!(C32, view),
622 TensorView::C64(view) => materialize!(C64, view),
623 }
624}
625
626#[allow(clippy::uninit_vec)]
632#[cfg(test)]
633pub(crate) unsafe fn typed_array_uninit<T>(shape: &[usize]) -> StridedArray<T> {
634 let total: usize = shape.iter().product();
635 let strides = kernel_col_major_strides(shape);
636 let mut data = Vec::with_capacity(total);
637 unsafe { data.set_len(total) };
639 StridedArray::from_parts(data, shape, &strides, 0).expect("column-major output array")
642}
643
644#[cfg(test)]
645pub(crate) fn tensor_from_array<T: Clone + tenferro_tensor::TensorScalar>(
646 array: StridedArray<T>,
647) -> TypedTensor<T> {
648 TypedTensor::from_vec_col_major(array.dims().to_vec(), array.into_data())
650 .expect("strided array dimensions match owned data length")
651}
652
653#[cfg(test)]
654mod tests;