1use crate::buffer_pool::{BufferPool, PoolScalar};
2use crate::engine::guard_unwind;
3use crate::{CpuEngine, CpuPlanCache, Tensor, TensorRead, TensorValue, TensorWrite};
4use num_complex::{Complex32, Complex64};
5use tenferro_tensor::backend::{ElementwiseFusionPlan, Engine};
6use tenferro_tensor::{
7 ActivationOp, CompareDir, ContractionScalar, DType, DotGeneralConfig, ElementwiseReadOp,
8 GatherConfig, PadConfig, ScatterConfig, SliceConfig, TensorView, TypedTensor,
9};
10use tenferro_tensor::{
11 DotGeneralAccumulation, SessionCachedDot, TensorAnalytic, TensorBuffer, TensorDeviceTransfer,
12 TensorDot, TensorElementwise, TensorFusion, TensorIndexing, TensorReduction, TensorStructural,
13};
14
15use super::backend::elementwise_read_into_fallback_with_pool;
16use super::{
17 activation, analytic, copy_tensor_read_into, elementwise, indexing,
18 materialize_tensor_read_in_domain, reduction, softmax, structural,
19};
20
21pub(super) struct CpuEngineMarker;
24
25fn eager_dot_cache_slot(
40 capacity: usize,
41 lhs: &TensorRead<'_>,
42 rhs: &TensorRead<'_>,
43 config: &DotGeneralConfig,
44 lhs_conj: bool,
45 rhs_conj: bool,
46) -> Option<usize> {
47 use std::hash::{Hash, Hasher};
48
49 if capacity == 0 {
50 return None;
51 }
52 let mut hasher = std::collections::hash_map::DefaultHasher::new();
53 lhs.dtype().hash(&mut hasher);
54 lhs.shape().hash(&mut hasher);
55 rhs.shape().hash(&mut hasher);
56 config.lhs_contracting_dims.hash(&mut hasher);
57 config.rhs_contracting_dims.hash(&mut hasher);
58 config.lhs_batch_dims.hash(&mut hasher);
59 config.rhs_batch_dims.hash(&mut hasher);
60 lhs_conj.hash(&mut hasher);
61 rhs_conj.hash(&mut hasher);
62 Some(hasher.finish() as usize % capacity)
63}
64
65fn pooled_zero_tensor<T>(
66 buffers: &mut BufferPool,
67 shape: impl tenferro_tensor::IntoShapeVec,
68) -> crate::Result<TypedTensor<T>>
69where
70 T: PoolScalar + Clone + 'static,
71{
72 let shape = shape.into_shape_vec();
73 let element_count =
74 tenferro_tensor::validate::checked_shape_product("dot_general", "output", &shape)?;
75 TypedTensor::from_vec_col_major(shape, T::pool_acquire_zeroed(buffers, element_count))
76}
77
78fn allocate_dot_output(
79 buffers: &mut BufferPool,
80 dtype: DType,
81 shape: impl tenferro_tensor::IntoShapeVec,
82) -> crate::Result<Tensor> {
83 let shape = shape.into_shape_vec();
84 match dtype {
85 DType::F32 => pooled_zero_tensor(buffers, &shape).map(Tensor::from_typed::<f32>),
86 DType::F64 => pooled_zero_tensor(buffers, &shape).map(Tensor::from_typed::<f64>),
87 DType::C32 => pooled_zero_tensor(buffers, &shape)
88 .map(Tensor::from_typed::<tenferro_tensor::Complex32>),
89 DType::C64 => pooled_zero_tensor(buffers, &shape)
90 .map(Tensor::from_typed::<tenferro_tensor::Complex64>),
91 dtype => Err(crate::Error::unsupported_dtype(
92 "dot_general",
93 dtype,
94 crate::cpu_contraction_unsupported_dtype_message(dtype),
95 )),
96 }
97}
98
99fn flatten_compact_read(
100 input: TensorRead<'_>,
101 element_count: usize,
102) -> crate::Result<TensorRead<'_>> {
103 macro_rules! flatten {
104 ($variant:ident, $view:expr) => {
105 Ok(TensorRead::from_view(TensorView::$variant(
106 $view.reshape_view(&[element_count])?,
107 )))
108 };
109 }
110 fn read_refusal(tensor: &Tensor) -> crate::Error {
112 crate::Error::unsupported_dtype(
113 "flatten_compact_read",
114 tensor.dtype(),
115 "an externally defined payload is not a compact runtime read",
116 )
117 }
118
119 match input {
120 TensorRead::Tensor(tensor) => match tensor.dtype() {
121 DType::F32 => flatten!(
122 F32,
123 tensor
124 .as_typed::<f32>()
125 .ok_or_else(|| read_refusal(tensor))?
126 .as_view()
127 ),
128 DType::F64 => flatten!(
129 F64,
130 tensor
131 .as_typed::<f64>()
132 .ok_or_else(|| read_refusal(tensor))?
133 .as_view()
134 ),
135 DType::I32 => flatten!(
136 I32,
137 tensor
138 .as_typed::<i32>()
139 .ok_or_else(|| read_refusal(tensor))?
140 .as_view()
141 ),
142 DType::I64 => flatten!(
143 I64,
144 tensor
145 .as_typed::<i64>()
146 .ok_or_else(|| read_refusal(tensor))?
147 .as_view()
148 ),
149 DType::Bool => flatten!(
150 Bool,
151 tensor
152 .as_typed::<bool>()
153 .ok_or_else(|| read_refusal(tensor))?
154 .as_view()
155 ),
156 DType::C32 => flatten!(
157 C32,
158 tensor
159 .as_typed::<Complex32>()
160 .ok_or_else(|| read_refusal(tensor))?
161 .as_view()
162 ),
163 DType::C64 => flatten!(
164 C64,
165 tensor
166 .as_typed::<Complex64>()
167 .ok_or_else(|| read_refusal(tensor))?
168 .as_view()
169 ),
170 DType::External(_) => Err(read_refusal(tensor)),
172 },
173 TensorRead::View(TensorView::F32(view)) => flatten!(F32, view),
174 TensorRead::View(TensorView::F64(view)) => flatten!(F64, view),
175 TensorRead::View(TensorView::I32(view)) => flatten!(I32, view),
176 TensorRead::View(TensorView::I64(view)) => flatten!(I64, view),
177 TensorRead::View(TensorView::Bool(view)) => flatten!(Bool, view),
178 TensorRead::View(TensorView::C32(view)) => flatten!(C32, view),
179 TensorRead::View(TensorView::C64(view)) => flatten!(C64, view),
180 }
181}
182
183impl CpuEngine {
184 pub(crate) fn replace_batch_policy(
186 &mut self,
187 policy: crate::CpuBatchPolicy,
188 ) -> crate::CpuBatchPolicy {
189 self.set_batch_policy(policy)
190 }
191
192 fn run_native_with_exec<R>(
196 &mut self,
197 op: impl FnOnce(threadteam::Exec<'_>, &mut BufferPool) -> crate::Result<R>,
198 ) -> crate::Result<R> {
199 self.check_entry("CpuEngine")?;
200 let (exec, buffers, _, broken) = self.parts();
201 guard_unwind(buffers, broken, |buffers| op(exec.into_token(), buffers))
202 }
203
204 fn run_native_with_plans<R>(
205 &mut self,
206 op: impl FnOnce(threadteam::Exec<'_>, &mut BufferPool, &CpuPlanCache) -> crate::Result<R>,
207 ) -> crate::Result<R> {
208 self.check_entry("CpuEngine")?;
209 let (exec, buffers, plans, broken) = self.parts();
210 guard_unwind(buffers, broken, |buffers| {
211 op(exec.into_token(), buffers, plans)
212 })
213 }
214
215 fn run_contraction<R>(
218 &mut self,
219 op: impl FnOnce(&mut cpueinsum::Exec<'_>, &mut BufferPool, &CpuPlanCache) -> crate::Result<R>,
220 ) -> crate::Result<R> {
221 self.check_entry("dot_general")?;
222 let _retention = self
223 .resources
224 .retention_guard(self.buffers.max_retained_capacity_bytes());
225 let mut exec = self.resources.exec(self.batch_policy);
226 let plans = &self.plans;
227 guard_unwind(&mut self.buffers, &mut self.broken, |buffers| {
228 op(&mut exec, buffers, plans)
229 })
230 }
231}
232
233impl TensorDeviceTransfer for CpuEngine {
234 fn download_to_host(&mut self, tensor: TensorRead<'_>) -> crate::Result<Tensor> {
235 self.check_entry("CpuEngine::download_to_host")?;
236 if tensor.backend_family().is_some() {
237 return Err(crate::Error::runtime_state(
238 "CpuEngine::download_to_host",
239 "CPU backend received a backend buffer; download the tensor to host with its owning backend before CPU execution",
240 ));
241 }
242 crate::preset_view("CpuEngine::download_to_host", tensor)?.duplicate()
243 }
244
245 fn upload_host_tensor(&mut self, tensor: TensorRead<'_>) -> crate::Result<Tensor> {
246 self.check_entry("CpuEngine::upload_host_tensor")?;
247 if tensor.backend_family().is_some() {
248 return Err(crate::Error::runtime_state(
249 "CpuEngine::upload_host_tensor",
250 "CPU backend upload_host_tensor expects a host tensor; download backend buffers to host before CPU execution",
251 ));
252 }
253 crate::preset_view("CpuEngine::upload_host_tensor", tensor)?.duplicate()
254 }
255}
256
257macro_rules! delegate_with_pool {
260 ($name:ident($($arg:ident : $ty:ty),*) => $callee:path) => {
261 fn $name(&mut self, $($arg: $ty),*) -> crate::Result<Tensor> {
262 self.run_native_with_exec(|exec, buffers| $callee(buffers, $($arg),*, exec))
263 }
264 };
265}
266
267impl TensorElementwise for CpuEngine {
268 fn elementwise_read_into(
270 &mut self,
271 op: ElementwiseReadOp,
272 inputs: &[TensorRead<'_>],
273 out: TensorWrite<'_>,
274 ) -> crate::Result<()> {
275 self.run_native_with_exec(|exec, buffers| {
276 tenferro_internal_cpu_kernels::elementwise_read_into_with_exec(
278 op,
279 inputs,
280 out,
281 |inputs, out, exec| {
282 elementwise_read_into_fallback_with_pool(buffers, op, inputs, out, exec)
283 },
284 exec,
285 )
286 })
287 }
288
289 fn add_read(&mut self, lhs: TensorRead<'_>, rhs: TensorRead<'_>) -> crate::Result<Tensor> {
290 self.run_native_with_exec(|exec, buffers| {
291 elementwise::add_read_with_pool(buffers, lhs, rhs, exec)
292 })
293 }
294
295 delegate_with_pool!(sub_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::sub_read_with_pool);
296 delegate_with_pool!(mul_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::mul_read_with_pool);
297 delegate_with_pool!(neg_read(input: TensorRead<'_>) => elementwise::neg_read_with_pool);
298 delegate_with_pool!(conj_read(input: TensorRead<'_>) => elementwise::conj_read_with_pool);
299 delegate_with_pool!(div_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::div_read_with_pool);
300 delegate_with_pool!(rem(lhs: &Tensor, rhs: &Tensor) => elementwise::rem_with_pool);
301 delegate_with_pool!(rem_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::rem_read_with_pool);
302 delegate_with_pool!(abs_read(input: TensorRead<'_>) => elementwise::abs_read_with_pool);
303 delegate_with_pool!(sign_read(input: TensorRead<'_>) => elementwise::sign_read_with_pool);
304 delegate_with_pool!(maximum_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::maximum_read_with_pool);
305 delegate_with_pool!(minimum_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => elementwise::minimum_read_with_pool);
306 delegate_with_pool!(compare_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>, dir: &CompareDir) => elementwise::compare_read_with_pool);
307 delegate_with_pool!(select_read(pred: TensorRead<'_>, on_true: TensorRead<'_>, on_false: TensorRead<'_>) => elementwise::select_read_with_pool);
308 delegate_with_pool!(clamp_read(input: TensorRead<'_>, lower: TensorRead<'_>, upper: TensorRead<'_>) => elementwise::clamp_read_with_pool);
309}
310
311impl TensorAnalytic for CpuEngine {
312 delegate_with_pool!(exp_read(input: TensorRead<'_>) => analytic::exp_read_with_pool);
314 delegate_with_pool!(log_read(input: TensorRead<'_>) => analytic::log_read_with_pool);
315 delegate_with_pool!(sin_read(input: TensorRead<'_>) => analytic::sin_read_with_pool);
316 delegate_with_pool!(cos_read(input: TensorRead<'_>) => analytic::cos_read_with_pool);
317 delegate_with_pool!(tanh_read(input: TensorRead<'_>) => analytic::tanh_read_with_pool);
318 delegate_with_pool!(sqrt_read(input: TensorRead<'_>) => analytic::sqrt_read_with_pool);
319 delegate_with_pool!(rsqrt_read(input: TensorRead<'_>) => analytic::rsqrt_read_with_pool);
320 delegate_with_pool!(pow_read(lhs: TensorRead<'_>, rhs: TensorRead<'_>) => analytic::pow_read_with_pool);
321 delegate_with_pool!(expm1_read(input: TensorRead<'_>) => analytic::expm1_read_with_pool);
322 delegate_with_pool!(log1p_read(input: TensorRead<'_>) => analytic::log1p_read_with_pool);
323 delegate_with_pool!(erf_read(input: TensorRead<'_>) => analytic::erf_read_with_pool);
324}
325
326impl TensorStructural for CpuEngine {
327 fn to_contiguous_read(&mut self, input: TensorRead<'_>) -> crate::Result<Tensor> {
329 if input.as_tensor().is_none() {
333 self.resources.count_read_staging();
336 }
337 let domain = self.shared_allocation_domain().cloned();
338 if matches!(input, TensorRead::Tensor(_)) && input.backend_family().is_none() {
339 self.check_entry("CpuEngine::to_contiguous_read")?;
340 return materialize_tensor_read_in_domain(
342 &mut self.buffers,
343 "CpuEngine::to_contiguous_read",
344 input,
345 domain.as_deref(),
346 threadteam::Exec::serial(),
347 );
348 }
349 self.run_native_with_exec(|exec, buffers| {
350 materialize_tensor_read_in_domain(
351 buffers,
352 "CpuEngine::to_contiguous_read",
353 input,
354 domain.as_deref(),
355 exec,
356 )
357 })
358 }
359
360 fn copy_read_into(&mut self, src: TensorRead<'_>, dst: TensorWrite<'_>) -> crate::Result<()> {
361 self.run_native_with_exec(|exec, _| {
362 copy_tensor_read_into("CpuEngine::copy_read_into", src, dst, exec)
363 })
364 }
365
366 fn transpose_read(&mut self, input: TensorRead<'_>, perm: &[usize]) -> crate::Result<Tensor> {
367 self.run_native_with_exec(|exec, buffers| {
368 structural::transpose_read_with_pool(buffers, input, perm, exec)
369 })
370 }
371
372 fn reshape_read(&mut self, input: TensorRead<'_>, shape: &[usize]) -> crate::Result<Tensor> {
373 self.check_entry("CpuEngine::reshape_read")?;
374 if let TensorRead::Tensor(tensor) = &input {
383 if !structural::reshape_may_use_pool(
384 tensor,
385 self.thread_budget(),
386 self.dispatch_costs(),
387 )? {
388 return structural::reshape(tensor, shape);
389 }
390 return self.run_native_with_exec(|exec, buffers| {
391 if let TensorRead::Tensor(tensor) = &input {
392 if !structural::reshape_uses_pool(tensor, &exec)? {
393 return structural::reshape(tensor, shape);
394 }
395 }
396 structural::reshape_read_with_pool(buffers, input, shape, exec)
397 });
398 }
399 self.run_native_with_exec(|exec, buffers| {
400 structural::reshape_read_with_pool(buffers, input, shape, exec)
401 })
402 }
403
404 fn broadcast_in_dim_read(
405 &mut self,
406 input: TensorRead<'_>,
407 shape: &[usize],
408 dims: &[usize],
409 ) -> crate::Result<Tensor> {
410 self.run_native_with_exec(|exec, buffers| {
411 structural::broadcast_in_dim_read_with_pool(buffers, input, shape, dims, exec)
412 })
413 }
414
415 fn tril_read(
416 &mut self,
417 input: tenferro_tensor::TensorRead<'_>,
418 k: i64,
419 ) -> crate::Result<Tensor> {
420 self.run_native_with_exec(|exec, buffers| {
421 structural::tril_read_with_pool(buffers, input, k, exec)
422 })
423 }
424
425 fn triu_read(
426 &mut self,
427 input: tenferro_tensor::TensorRead<'_>,
428 k: i64,
429 ) -> crate::Result<Tensor> {
430 self.run_native_with_exec(|exec, buffers| {
431 structural::triu_read_with_pool(buffers, input, k, exec)
432 })
433 }
434
435 fn extract_diagonal_read(
436 &mut self,
437 input: tenferro_tensor::TensorRead<'_>,
438 axis_a: usize,
439 axis_b: usize,
440 ) -> crate::Result<Tensor> {
441 self.run_native_with_exec(|exec, buffers| {
442 structural::extract_diagonal_read_with_pool(buffers, input, axis_a, axis_b, exec)
443 })
444 }
445
446 fn embed_diagonal_read(
447 &mut self,
448 input: tenferro_tensor::TensorRead<'_>,
449 axis_a: usize,
450 axis_b: usize,
451 ) -> crate::Result<Tensor> {
452 self.run_native_with_exec(|exec, buffers| {
453 structural::embed_diagonal_read_with_pool(buffers, input, axis_a, axis_b, exec)
454 })
455 }
456
457 delegate_with_pool!(cast(input: &Tensor, to: crate::DType) => structural::cast_with_pool);
458 delegate_with_pool!(cast_read(input: tenferro_tensor::TensorRead<'_>, to: crate::DType) => structural::cast_read_with_pool);
459 delegate_with_pool!(extract_diagonal(input: &Tensor, axis_a: usize, axis_b: usize) => structural::extract_diagonal_with_pool);
460 delegate_with_pool!(embed_diagonal(input: &Tensor, axis_a: usize, axis_b: usize) => structural::embed_diagonal_with_pool);
461 delegate_with_pool!(tril(input: &Tensor, k: i64) => structural::tril_with_pool);
462 delegate_with_pool!(triu(input: &Tensor, k: i64) => structural::triu_with_pool);
463}
464
465impl TensorReduction for CpuEngine {
466 fn reduce_sum_read(&mut self, input: TensorRead<'_>, axes: &[usize]) -> crate::Result<Tensor> {
469 self.run_native_with_exec(|exec, buffers| {
470 reduction::reduce_sum_read(buffers, input, axes, exec)
471 })
472 }
473
474 fn reduce_sum_squares_read(
475 &mut self,
476 input: TensorRead<'_>,
477 axes: &[usize],
478 ) -> crate::Result<Tensor> {
479 self.run_native_with_exec(|exec, buffers| {
480 reduction::reduce_sum_squares_read(buffers, input, axes, exec)
481 })
482 }
483
484 fn reduce_prod_read(&mut self, input: TensorRead<'_>, axes: &[usize]) -> crate::Result<Tensor> {
485 self.run_native_with_exec(|exec, buffers| {
486 reduction::reduce_prod_read(buffers, input, axes, exec)
487 })
488 }
489
490 fn reduce_max_read(&mut self, input: TensorRead<'_>, axes: &[usize]) -> crate::Result<Tensor> {
491 self.run_native_with_exec(|exec, buffers| {
492 reduction::reduce_max_read(buffers, input, axes, exec)
493 })
494 }
495
496 fn reduce_min_read(&mut self, input: TensorRead<'_>, axes: &[usize]) -> crate::Result<Tensor> {
497 self.run_native_with_exec(|exec, buffers| {
498 reduction::reduce_min_read(buffers, input, axes, exec)
499 })
500 }
501}
502
503impl CpuEngine {
504 fn execute_dot_allocated(
505 &mut self,
506 cache_slot: Option<usize>,
507 lhs: TensorRead<'_>,
508 rhs: TensorRead<'_>,
509 config: &DotGeneralConfig,
510 lhs_conj: bool,
511 rhs_conj: bool,
512 ) -> crate::Result<Tensor> {
513 let dtype = lhs.dtype();
514 if rhs.dtype() != dtype {
515 return Err(crate::Error::dtype_mismatch(
516 "dot_general",
517 dtype,
518 rhs.dtype(),
519 ));
520 }
521 let output_shape = tenferro_tensor::backend::dot_general_output_shape(
522 lhs.shape(),
523 rhs.shape(),
524 config,
525 "dot_general",
526 )?;
527 let accumulation = DotGeneralAccumulation {
528 lhs_conj,
529 rhs_conj,
530 alpha: ContractionScalar::one(dtype)?,
531 beta: ContractionScalar::zero(dtype)?,
532 };
533 if matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64) {
534 return self.run_contraction(|exec, buffers, plans| {
535 crate::contraction::dot_fresh(
536 exec,
537 buffers,
538 plans,
539 cache_slot,
540 lhs,
541 rhs,
542 config,
543 accumulation,
544 output_shape,
545 )
546 });
547 }
548 let mut output = allocate_dot_output(&mut self.buffers, dtype, output_shape)?;
549 self.dot_general_read_into_accum_cached(
550 cache_slot,
551 lhs,
552 rhs,
553 config,
554 accumulation,
555 TensorWrite::from_tensor(&mut output),
556 )?;
557 Ok(output)
558 }
559}
560
561impl TensorDot for CpuEngine {
593 fn dot_general_read(
594 &mut self,
595 lhs: TensorRead<'_>,
596 rhs: TensorRead<'_>,
597 config: &DotGeneralConfig,
598 ) -> crate::Result<Tensor> {
599 let slot = eager_dot_cache_slot(
600 self.plans.contraction_capacity(),
601 &lhs,
602 &rhs,
603 config,
604 false,
605 false,
606 );
607 self.execute_dot_allocated(slot, lhs, rhs, config, false, false)
608 }
609
610 fn dot_general_with_conj_read(
611 &mut self,
612 lhs: TensorRead<'_>,
613 rhs: TensorRead<'_>,
614 config: &DotGeneralConfig,
615 lhs_conj: bool,
616 rhs_conj: bool,
617 ) -> crate::Result<Tensor> {
618 let slot = eager_dot_cache_slot(
621 self.plans.contraction_capacity(),
622 &lhs,
623 &rhs,
624 config,
625 lhs_conj,
626 rhs_conj,
627 );
628 self.execute_dot_allocated(slot, lhs, rhs, config, lhs_conj, rhs_conj)
629 }
630
631 fn dot_general_read_into(
632 &mut self,
633 lhs: TensorRead<'_>,
634 rhs: TensorRead<'_>,
635 config: &DotGeneralConfig,
636 out: TensorWrite<'_>,
637 ) -> crate::Result<()> {
638 let accumulation = DotGeneralAccumulation::overwrite(lhs.dtype())?;
639 self.dot_general_read_into_accum(lhs, rhs, config, accumulation, out)
640 }
641
642 fn dot_general_read_into_accum(
643 &mut self,
644 lhs: TensorRead<'_>,
645 rhs: TensorRead<'_>,
646 config: &DotGeneralConfig,
647 accumulation: DotGeneralAccumulation,
648 out: TensorWrite<'_>,
649 ) -> crate::Result<()> {
650 let slot = eager_dot_cache_slot(
651 self.plans.contraction_capacity(),
652 &lhs,
653 &rhs,
654 config,
655 accumulation.lhs_conj,
656 accumulation.rhs_conj,
657 );
658 self.dot_general_read_into_accum_cached(slot, lhs, rhs, config, accumulation, out)
659 }
660
661 fn dot_general_with_conj(
662 &mut self,
663 lhs: &Tensor,
664 rhs: &Tensor,
665 config: &DotGeneralConfig,
666 lhs_conj: bool,
667 rhs_conj: bool,
668 ) -> crate::Result<Tensor> {
669 let lhs = TensorRead::from_tensor(lhs);
670 let rhs = TensorRead::from_tensor(rhs);
671 let slot = eager_dot_cache_slot(
672 self.plans.contraction_capacity(),
673 &lhs,
674 &rhs,
675 config,
676 lhs_conj,
677 rhs_conj,
678 );
679 self.execute_dot_allocated(slot, lhs, rhs, config, lhs_conj, rhs_conj)
680 }
681}
682
683impl SessionCachedDot for CpuEngine {
684 fn dot_general_cached(
685 &mut self,
686 cache_slot: Option<usize>,
687 lhs: &Tensor,
688 rhs: &Tensor,
689 config: &DotGeneralConfig,
690 ) -> crate::Result<Tensor> {
691 self.execute_dot_allocated(
692 cache_slot,
693 TensorRead::from_tensor(lhs),
694 TensorRead::from_tensor(rhs),
695 config,
696 false,
697 false,
698 )
699 }
700
701 fn dot_general_read_cached(
702 &mut self,
703 cache_slot: Option<usize>,
704 lhs: TensorRead<'_>,
705 rhs: TensorRead<'_>,
706 config: &DotGeneralConfig,
707 ) -> crate::Result<Tensor> {
708 self.execute_dot_allocated(cache_slot, lhs, rhs, config, false, false)
712 }
713
714 fn dot_general_with_conj_cached(
715 &mut self,
716 cache_slot: Option<usize>,
717 lhs: &Tensor,
718 rhs: &Tensor,
719 config: &DotGeneralConfig,
720 lhs_conj: bool,
721 rhs_conj: bool,
722 ) -> crate::Result<Tensor> {
723 self.execute_dot_allocated(
724 cache_slot,
725 TensorRead::from_tensor(lhs),
726 TensorRead::from_tensor(rhs),
727 config,
728 lhs_conj,
729 rhs_conj,
730 )
731 }
732
733 fn dot_general_with_conj_read_cached(
734 &mut self,
735 cache_slot: Option<usize>,
736 lhs: TensorRead<'_>,
737 rhs: TensorRead<'_>,
738 config: &DotGeneralConfig,
739 lhs_conj: bool,
740 rhs_conj: bool,
741 ) -> crate::Result<Tensor> {
742 self.execute_dot_allocated(cache_slot, lhs, rhs, config, lhs_conj, rhs_conj)
743 }
744
745 fn dot_general_read_into_accum_cached(
746 &mut self,
747 cache_slot: Option<usize>,
748 lhs: TensorRead<'_>,
749 rhs: TensorRead<'_>,
750 config: &DotGeneralConfig,
751 accumulation: DotGeneralAccumulation,
752 out: TensorWrite<'_>,
753 ) -> crate::Result<()> {
754 if matches!(
755 lhs.dtype(),
756 DType::F32 | DType::F64 | DType::C32 | DType::C64
757 ) {
758 return self.run_contraction(|exec, _buffers, plans| {
759 crate::contraction::dot_into(
760 exec,
761 plans,
762 cache_slot,
763 lhs,
764 rhs,
765 config,
766 accumulation,
767 out,
768 )
769 });
770 }
771 Err(crate::Error::unsupported_dtype(
774 "dot_general",
775 lhs.dtype(),
776 crate::cpu_contraction_unsupported_dtype_message(lhs.dtype()),
777 ))
778 }
779}
780
781impl TensorIndexing for CpuEngine {
782 fn gather(
784 &mut self,
785 operand: &Tensor,
786 start_indices: &Tensor,
787 config: &GatherConfig,
788 ) -> crate::Result<Tensor> {
789 self.run_native_with_plans(|exec, buffers, cache| {
790 indexing::gather_with_pool(buffers, cache, operand, start_indices, config, exec)
791 })
792 }
793 fn scatter(
794 &mut self,
795 operand: &Tensor,
796 indices: &Tensor,
797 updates: &Tensor,
798 config: &ScatterConfig,
799 ) -> crate::Result<Tensor> {
800 self.run_native_with_plans(|exec, buffers, cache| {
801 indexing::scatter_with_pool(buffers, cache, operand, indices, updates, config, exec)
802 })
803 }
804 fn dynamic_update_slice_read(
805 &mut self,
806 operand: tenferro_tensor::TensorRead<'_>,
807 update: tenferro_tensor::TensorRead<'_>,
808 starts: tenferro_tensor::TensorRead<'_>,
809 ) -> crate::Result<Tensor> {
810 self.run_native_with_plans(|exec, buffers, cache| {
811 indexing::dynamic_update_slice_read_with_pool(
812 buffers, cache, operand, update, starts, exec,
813 )
814 })
815 }
816 fn scatter_read(
817 &mut self,
818 operand: tenferro_tensor::TensorRead<'_>,
819 indices: tenferro_tensor::TensorRead<'_>,
820 updates: tenferro_tensor::TensorRead<'_>,
821 config: &ScatterConfig,
822 ) -> crate::Result<Tensor> {
823 self.run_native_with_plans(|exec, buffers, cache| {
824 indexing::scatter_read_with_pool(
825 buffers, cache, operand, indices, updates, config, exec,
826 )
827 })
828 }
829 fn slice(&mut self, input: &Tensor, config: &SliceConfig) -> crate::Result<Tensor> {
830 self.run_native_with_exec(|exec, buffers| {
831 indexing::try_slice_with_pool(buffers, input, config, exec)
832 })
833 }
834 fn slice_read(
835 &mut self,
836 input: tenferro_tensor::TensorRead<'_>,
837 config: &SliceConfig,
838 ) -> crate::Result<Tensor> {
839 self.run_native_with_exec(|exec, buffers| {
840 indexing::slice_read_with_pool(buffers, input, config, exec)
841 })
842 }
843 fn dynamic_slice_read(
844 &mut self,
845 input: tenferro_tensor::TensorRead<'_>,
846 starts: tenferro_tensor::TensorRead<'_>,
847 slice_sizes: &[usize],
848 ) -> crate::Result<Tensor> {
849 self.run_native_with_plans(|exec, buffers, cache| {
850 indexing::dynamic_slice_read_with_pool(buffers, cache, input, starts, slice_sizes, exec)
851 })
852 }
853 fn dynamic_slice(
854 &mut self,
855 input: &Tensor,
856 starts: &Tensor,
857 slice_sizes: &[usize],
858 ) -> crate::Result<Tensor> {
859 self.run_native_with_plans(|exec, buffers, cache| {
860 indexing::dynamic_slice_with_pool(buffers, cache, input, starts, slice_sizes, exec)
861 })
862 }
863 fn dynamic_update_slice(
864 &mut self,
865 operand: &Tensor,
866 update: &Tensor,
867 starts: &Tensor,
868 ) -> crate::Result<Tensor> {
869 self.run_native_with_plans(|exec, buffers, cache| {
870 indexing::dynamic_update_slice_with_pool(buffers, cache, operand, update, starts, exec)
871 })
872 }
873 fn pad(&mut self, input: &Tensor, config: &PadConfig) -> crate::Result<Tensor> {
874 self.run_native_with_exec(|exec, buffers| {
875 indexing::try_pad_with_pool(buffers, input, config, exec)
876 })
877 }
878 fn pad_read(
879 &mut self,
880 input: tenferro_tensor::TensorRead<'_>,
881 config: &PadConfig,
882 ) -> crate::Result<Tensor> {
883 self.run_native_with_exec(|exec, buffers| {
884 indexing::pad_read_with_pool(buffers, input, config, exec)
885 })
886 }
887 fn concatenate(&mut self, inputs: &[&Tensor], axis: usize) -> crate::Result<Tensor> {
888 self.run_native_with_exec(|exec, buffers| {
889 indexing::try_concatenate_with_pool(buffers, inputs, axis, exec)
890 })
891 }
892 fn concatenate_read(
893 &mut self,
894 inputs: &[tenferro_tensor::TensorRead<'_>],
895 axis: usize,
896 ) -> crate::Result<Tensor> {
897 self.run_native_with_exec(|exec, buffers| {
898 indexing::concatenate_read_with_pool(buffers, inputs, axis, exec)
899 })
900 }
901 fn reverse(&mut self, input: &Tensor, axes: &[usize]) -> crate::Result<Tensor> {
902 self.run_native_with_exec(|exec, buffers| {
903 indexing::reverse_with_pool(buffers, input, axes, exec)
904 })
905 }
906 fn reverse_read(
907 &mut self,
908 input: tenferro_tensor::TensorRead<'_>,
909 axes: &[usize],
910 ) -> crate::Result<Tensor> {
911 self.run_native_with_exec(|exec, buffers| {
912 indexing::reverse_read_with_pool(buffers, input, axes, exec)
913 })
914 }
915}
916
917impl<T, R> tenferro_tensor::TensorViewCanonicalization<T, R> for CpuEngine
919where
920 T: tenferro_tensor::TensorScalar + PoolScalar,
921 R: tenferro_tensor::TensorRank,
922 R::Shape: Send + Sync,
923 R::Strides: Send + Sync,
924{
925 fn to_contiguous(
926 &mut self,
927 view: &tenferro_tensor::TypedTensorView<'_, T, R>,
928 ) -> crate::Result<TypedTensor<T, R>> {
929 self.run_native_with_exec(|exec, buffers| {
930 crate::structural::typed_materialize_view_with_pool(
931 buffers,
932 view,
933 "CpuEngine::to_contiguous",
934 exec,
935 )
936 })
937 }
938
939 fn copy_into(
940 &mut self,
941 src: &tenferro_tensor::TypedTensorView<'_, T, R>,
942 dst: &mut tenferro_tensor::TypedTensorViewMut<'_, T, R>,
943 ) -> crate::Result<()> {
944 self.run_native_with_exec(|exec, _| {
945 crate::structural::typed_copy_view_into(src, dst, "CpuEngine::copy_into", exec)
946 })
947 }
948}
949
950impl TensorBuffer for CpuEngine {
951 fn reclaim_buffer(&mut self, tensor: Tensor) {
952 crate::backend::reclaim_tensor(&mut self.buffers, tensor);
953 }
954}
955
956impl TensorFusion for CpuEngine {
957 fn execute_elementwise_fusion(
958 &mut self,
959 inputs: &[&Tensor],
960 plan: &ElementwiseFusionPlan,
961 ) -> crate::Result<Option<Vec<Tensor>>> {
962 self.run_native_with_exec(|exec, buffers| {
963 tenferro_cpu_fused::elementwise_fusion_with_pool(buffers, inputs, plan, exec)
964 })
965 }
966
967 fn execute_broadcast_multiply(
968 &mut self,
969 lhs: TensorRead<'_>,
970 lhs_shape: &[usize],
971 lhs_dims: &[usize],
972 rhs: TensorRead<'_>,
973 rhs_shape: &[usize],
974 rhs_dims: &[usize],
975 ) -> crate::Result<Option<Tensor>> {
976 self.run_native_with_exec(|exec, buffers| {
977 elementwise::broadcast_multiply_read_with_pool(
978 buffers, lhs, lhs_shape, lhs_dims, rhs, rhs_shape, rhs_dims, exec,
979 )
980 })
981 }
982
983 fn execute_broadcast_multiply_value(
984 &mut self,
985 lhs: TensorRead<'_>,
986 lhs_shape: &[usize],
987 lhs_dims: &[usize],
988 rhs: TensorRead<'_>,
989 rhs_shape: &[usize],
990 rhs_dims: &[usize],
991 ) -> crate::Result<Option<TensorValue>> {
992 self.run_native_with_exec(|exec, buffers| {
993 elementwise::broadcast_multiply_value_with_pool_and_tag(
994 buffers,
995 lhs,
996 lhs_shape,
997 lhs_dims,
998 rhs,
999 rhs_shape,
1000 rhs_dims,
1001 |_| {},
1002 exec,
1003 )
1004 })
1005 }
1006}
1007
1008impl Engine for CpuEngine {
1009 fn gather_read(
1010 &mut self,
1011 operand: tenferro_tensor::TensorRead<'_>,
1012 start_indices: tenferro_tensor::TensorRead<'_>,
1013 config: &GatherConfig,
1014 ) -> crate::Result<Tensor> {
1015 self.run_native_with_plans(|exec, buffers, cache| {
1016 indexing::gather_read_with_pool(buffers, cache, operand, start_indices, config, exec)
1017 })
1018 }
1019 fn vdot_read(&mut self, lhs: TensorRead<'_>, rhs: TensorRead<'_>) -> crate::Result<Tensor> {
1020 tenferro_tensor::backend::validate_vdot_read(&lhs, &rhs)?;
1021 crate::blas1::validate_cpu_read("Engine::vdot_read", &lhs)?;
1022 crate::blas1::validate_cpu_read("Engine::vdot_read", &rhs)?;
1023 let element_count = tenferro_tensor::validate::checked_shape_product(
1024 "Engine::vdot_read",
1025 "shape",
1026 lhs.shape(),
1027 )?;
1028 let (lhs, rhs, rank) = if lhs.shape().len() > 2
1029 && lhs.is_col_major_contiguous()?
1030 && rhs.is_col_major_contiguous()?
1031 {
1032 (
1033 flatten_compact_read(lhs, element_count)?,
1034 flatten_compact_read(rhs, element_count)?,
1035 1,
1036 )
1037 } else {
1038 let rank = lhs.shape().len();
1039 (lhs, rhs, rank)
1040 };
1041 let axes = (0..rank).collect::<tinyvec::TinyVec<[usize; 4]>>();
1042 let config = DotGeneralConfig {
1043 lhs_contracting_dims: axes.clone(),
1044 rhs_contracting_dims: axes,
1045 lhs_batch_dims: Default::default(),
1046 rhs_batch_dims: Default::default(),
1047 };
1048 self.execute_dot_allocated(None, lhs, rhs, &config, true, false)
1049 }
1050
1051 fn norm_squared_read(&mut self, input: TensorRead<'_>) -> crate::Result<Tensor> {
1052 tenferro_tensor::backend::validate_norm_squared_read(&input)?;
1053 crate::blas1::validate_cpu_read("Engine::norm_squared_read", &input)?;
1054 self.run_native_with_exec(|exec, buffers| {
1055 reduction::norm_squared_read(buffers, input, exec)
1056 })
1057 }
1058
1059 fn fused_activation_read(
1060 &mut self,
1061 op: ActivationOp,
1062 input: TensorRead<'_>,
1063 ) -> crate::Result<Option<Tensor>> {
1064 crate::blas1::validate_cpu_read(op.label(), &input)?;
1065 self.run_native_with_exec(|exec, buffers| {
1066 activation::fused_activation_read_with_pool(buffers, op, input, exec)
1067 })
1068 }
1069
1070 fn softmax_read(
1071 &mut self,
1072 input: TensorRead<'_>,
1073 axis: usize,
1074 ) -> crate::Result<Option<Tensor>> {
1075 crate::blas1::validate_cpu_read("softmax", &input)?;
1076 self.run_native_with_exec(|exec, buffers| {
1077 softmax::softmax_read_with_pool(buffers, input, axis, exec)
1078 })
1079 }
1080
1081 fn axpby_read_into_accum(
1082 &mut self,
1083 alpha: ContractionScalar,
1084 x: TensorRead<'_>,
1085 beta: ContractionScalar,
1086 y: TensorWrite<'_>,
1087 ) -> crate::Result<()> {
1088 tenferro_tensor::backend::validate_axpby_read_into_accum(alpha, &x, beta, &y)?;
1089 self.run_native_with_exec(|exec, buffers| {
1090 crate::blas1::axpby_read_into_accum(buffers, alpha, x, beta, y, exec)
1091 })
1092 }
1093
1094 fn native_engine(&mut self) -> Option<tenferro_tensor::NativeSessionRef<'_>> {
1095 Some(unsafe { tenferro_tensor::NativeSessionRef::new::<CpuEngineMarker, _>(self) })
1099 }
1100}
1101
1102#[cfg(test)]
1103mod tests;