tenferro_cpu/provider.rs
1//! Object-safe CPU contraction provider contracts.
2//!
3//! Providers synchronously write into engine-owned outputs. Request
4//! constructors are crate-private because only the CPU engine may attest that
5//! tensor metadata and reachable ranges have already been validated.
6
7use core::fmt;
8use std::mem::MaybeUninit;
9use std::num::NonZeroUsize;
10use std::sync::Arc;
11
12use tenferro_tensor::backend::GroupedGemmJob;
13use tenferro_tensor::{
14 DType, DotGeneralAccumulation, Tensor, TensorRead, TensorView, TensorViewMut, TensorWrite,
15};
16
17use crate::arbiter::{with_execution_owner, ResourcePermit};
18use crate::backend::CpuBackendKind;
19use crate::buffer_pool::BufferPool;
20use crate::domain_executor::{indexed_jobs, install_scoped};
21#[cfg(feature = "cpu-blas")]
22use crate::provider_capability::builtin_blas_execution_capabilities;
23#[cfg(not(feature = "cpu-blas"))]
24use crate::provider_capability::serial_capabilities;
25use crate::provider_capability::{engine_worker_capabilities, CpuProviderExecutionCapabilities};
26use crate::resource_domain::CpuResourceDomain;
27use crate::{
28 CpuDomainExecutorError, CpuDomainId, CpuInnerParallelism, CpuPlacementGuarantee, CpuSet,
29};
30
31/// Operand named by a provider capability reason.
32///
33/// # Examples
34///
35/// ```
36/// use tenferro_cpu::provider::CpuOperand;
37/// assert_eq!(CpuOperand::Lhs, CpuOperand::Lhs);
38/// ```
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40pub enum CpuOperand {
41 /// Left input operand.
42 Lhs,
43 /// Right input operand.
44 Rhs,
45 /// Writable output operand.
46 Output,
47}
48
49/// Allocation-free reason that a provider cannot execute a validated request.
50///
51/// An unsupported outcome must be reported before the provider mutates the
52/// output.
53///
54/// # Examples
55///
56/// ```
57/// use tenferro_cpu::provider::CpuProviderUnsupported;
58/// assert_eq!(
59/// CpuProviderUnsupported::RuntimeUnavailable,
60/// CpuProviderUnsupported::RuntimeUnavailable,
61/// );
62/// ```
63#[derive(Clone, Copy, Debug, PartialEq, Eq)]
64#[non_exhaustive]
65pub enum CpuProviderUnsupported {
66 /// The provider does not implement the scalar dtype.
67 DType(DType),
68 /// The provider does not implement the input ranks.
69 Rank {
70 /// Left input rank.
71 lhs: usize,
72 /// Right input rank.
73 rhs: usize,
74 },
75 /// The provider does not implement an operand layout.
76 Layout(CpuOperand),
77 /// The provider cannot implement the requested conjugation.
78 Conjugation,
79 /// The provider cannot implement the requested alpha/beta update.
80 Accumulation,
81 /// The provider cannot implement a strided batch.
82 StridedBatch,
83 /// The provider cannot implement grouped GEMM.
84 Grouped,
85 /// The optional provider runtime is not available in this process.
86 RuntimeUnavailable,
87}
88
89/// Result of attempting a validated request through one provider slot.
90///
91/// # Examples
92///
93/// ```
94/// use tenferro_cpu::provider::{CpuProviderOutcome, CpuProviderUnsupported};
95/// let outcome = CpuProviderOutcome::Unsupported(
96/// CpuProviderUnsupported::RuntimeUnavailable,
97/// );
98/// assert!(matches!(outcome, CpuProviderOutcome::Unsupported(_)));
99/// ```
100#[derive(Clone, Copy, Debug, PartialEq, Eq)]
101#[must_use]
102pub enum CpuProviderOutcome {
103 /// The provider fully executed the request.
104 Executed,
105 /// The provider did not mutate the output and resolution may continue.
106 Unsupported(CpuProviderUnsupported),
107}
108
109/// Parallel scheduling mode selected for one CPU operation.
110///
111/// # Examples
112///
113/// ```
114/// use tenferro_cpu::provider::ParallelMode;
115/// assert_ne!(ParallelMode::Sequential, ParallelMode::Outer);
116/// assert_ne!(ParallelMode::Outer, ParallelMode::Inner);
117/// ```
118#[derive(Clone, Copy, Debug, PartialEq, Eq)]
119pub enum ParallelMode {
120 /// Neither the engine nor the provider may fan out this operation.
121 Sequential,
122 /// The engine owns outer fan-out and delegates Sequential child contexts.
123 /// Providers do not receive an Outer context.
124 Outer,
125 /// One provider kernel may use the selected executor's inner region.
126 Inner,
127}
128
129/// Borrowed execution policy for an already-entered CPU operation.
130///
131/// The context exposes immutable domain facts while keeping the resource lease,
132/// executor object, and the checked executor-entry boundary private. Providers
133/// cannot install or submit work through this value.
134///
135/// # Examples
136///
137/// Providers inspect this value inside a trait method:
138///
139/// ```
140/// use tenferro_cpu::provider::CpuExecutionContext;
141/// # fn inspect(context: &CpuExecutionContext<'_>) {
142/// assert!(context.thread_budget().get() >= 1);
143/// # }
144/// ```
145#[derive(Clone, Copy)]
146pub struct CpuExecutionContext<'a> {
147 domain: &'a CpuResourceDomain,
148 parallel_mode: ParallelMode,
149}
150
151impl fmt::Debug for CpuExecutionContext<'_> {
152 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
153 formatter
154 .debug_struct("CpuExecutionContext")
155 .field("domain_id", &self.domain_id())
156 .field("cpus", &self.cpus())
157 .field("thread_budget", &self.thread_budget())
158 .field("placement_guarantee", &self.placement_guarantee())
159 .field("parallel_mode", &self.parallel_mode())
160 .finish_non_exhaustive()
161 }
162}
163
164impl<'a> CpuExecutionContext<'a> {
165 fn entered(domain: &'a CpuResourceDomain, parallel_mode: ParallelMode) -> Self {
166 Self {
167 domain,
168 parallel_mode,
169 }
170 }
171
172 /// Return the stable identity of the selected CPU resource domain.
173 ///
174 /// # Examples
175 ///
176 /// ```
177 /// use tenferro_cpu::CpuExecutionContext;
178 /// # fn inspect(context: &CpuExecutionContext<'_>) {
179 /// let _domain_id = context.domain_id();
180 /// # }
181 /// ```
182 pub fn domain_id(&self) -> CpuDomainId {
183 self.domain.id()
184 }
185
186 /// Return the selected domain's declared logical CPU set, when present.
187 ///
188 /// # Examples
189 ///
190 /// ```
191 /// use tenferro_cpu::CpuExecutionContext;
192 /// # fn inspect(context: &CpuExecutionContext<'_>) {
193 /// if let Some(cpus) = context.cpus() {
194 /// assert!(!cpus.is_empty());
195 /// }
196 /// # }
197 /// ```
198 pub fn cpus(&self) -> Option<&CpuSet> {
199 self.domain.cpus()
200 }
201
202 /// Return the selected domain's admission contract.
203 ///
204 /// # Examples
205 ///
206 /// ```rust
207 /// use tenferro_cpu::{CpuAdmissionMode, CpuExecutionContext};
208 /// # fn inspect(context: &CpuExecutionContext<'_>) {
209 /// let _mode: CpuAdmissionMode = context.admission_mode();
210 /// # }
211 /// ```
212 pub fn admission_mode(&self) -> crate::CpuAdmissionMode {
213 self.domain.admission_mode()
214 }
215
216 /// Return the non-zero maximum participating-thread budget.
217 ///
218 /// # Examples
219 ///
220 /// ```
221 /// use tenferro_cpu::CpuExecutionContext;
222 /// # fn inspect(context: &CpuExecutionContext<'_>) {
223 /// assert!(context.thread_budget().get() >= 1);
224 /// # }
225 /// ```
226 pub fn thread_budget(&self) -> NonZeroUsize {
227 self.domain.thread_budget()
228 }
229
230 /// Return the strength of the selected domain's placement guarantee, when present.
231 ///
232 /// # Examples
233 ///
234 /// ```
235 /// use tenferro_cpu::CpuExecutionContext;
236 /// # fn inspect(context: &CpuExecutionContext<'_>) {
237 /// let _guarantee = context.placement_guarantee();
238 /// # }
239 /// ```
240 pub fn placement_guarantee(&self) -> Option<CpuPlacementGuarantee> {
241 self.domain.placement_guarantee()
242 }
243
244 /// Return the engine-selected scheduling mode for this entered provider call.
245 ///
246 /// Provider calls observe Sequential or Inner. Outer scheduling creates a
247 /// separate Sequential context inside every submitted child.
248 ///
249 /// # Examples
250 ///
251 /// ```
252 /// use tenferro_cpu::{CpuExecutionContext, ParallelMode};
253 /// # fn inspect(context: &CpuExecutionContext<'_>) {
254 /// assert!(matches!(
255 /// context.parallel_mode(),
256 /// ParallelMode::Sequential | ParallelMode::Inner
257 /// ));
258 /// # }
259 /// ```
260 pub fn parallel_mode(&self) -> ParallelMode {
261 self.parallel_mode
262 }
263
264 /// Materialize a borrowed tensor view for one scoped operation and reclaim
265 /// its temporary host buffer before returning.
266 ///
267 /// Owned tensor inputs are borrowed directly. View inputs are materialized
268 /// from `buffers`, passed to `operation`, and returned to the same pool on
269 /// both success and ordinary error. The receiver is an unforgeable proof
270 /// that the caller is already inside the selected CPU execution domain.
271 ///
272 /// # Errors
273 ///
274 /// Returns [`tenferro_tensor::Error::RuntimeState`] when the view is not
275 /// accessible from CPU host memory, propagates typed view-materialization
276 /// errors, and otherwise returns the error produced by `operation`.
277 ///
278 /// # Examples
279 ///
280 /// ```
281 /// use tenferro_cpu::CpuBackend;
282 /// use tenferro_tensor::{StridedSliceSpec, TensorRead, TensorView, TypedTensor};
283 ///
284 /// let mut backend = CpuBackend::with_threads(1)?;
285 /// let input = TypedTensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
286 /// backend.with_linalg_pool(|context, buffers| {
287 /// let view = input
288 /// .as_view()
289 /// .try_slice(&[StridedSliceSpec::reverse()])?;
290 /// context.with_materialized_tensor_read(
291 /// buffers,
292 /// "example",
293 /// TensorRead::from_view(TensorView::F64(view)),
294 /// |materialized, _| {
295 /// assert_eq!(materialized.as_slice::<f64>().unwrap(), &[2.0, 1.0]);
296 /// Ok(())
297 /// },
298 /// )
299 /// })?;
300 /// # Ok::<(), Box<dyn std::error::Error>>(())
301 /// ```
302 #[doc(hidden)]
303 pub fn with_materialized_tensor_read<R>(
304 &self,
305 buffers: &mut BufferPool,
306 op: &'static str,
307 input: TensorRead<'_>,
308 operation: impl FnOnce(&Tensor, &mut BufferPool) -> tenferro_tensor::Result<R>,
309 ) -> tenferro_tensor::Result<R> {
310 match input {
311 TensorRead::Tensor(tensor) => operation(tensor, buffers),
312 TensorRead::View(view) => {
313 let materialized = self.with_native_parallelism(|| {
314 crate::materialize_tensor_read(buffers, op, TensorRead::View(view))
315 })?;
316 let result = operation(&materialized, buffers);
317 reclaim_tensor(buffers, materialized);
318 result
319 }
320 }
321 }
322
323 /// Reshape a compact tensor while retaining the current execution proof.
324 ///
325 /// This metadata-only helper does not enter an executor or borrow another
326 /// scratch pool.
327 ///
328 /// # Errors
329 ///
330 /// Returns [`tenferro_tensor::Error::Validation`] when the input and output
331 /// shapes have different element counts or the requested layout is invalid.
332 ///
333 /// # Examples
334 ///
335 /// ```
336 /// use tenferro_cpu::CpuBackend;
337 /// use tenferro_tensor::Tensor;
338 ///
339 /// let mut backend = CpuBackend::with_threads(1)?;
340 /// let input = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
341 /// backend.with_linalg_pool(|context, _| {
342 /// let output = context.reshape_tensor(&input, &[2, 1])?;
343 /// assert_eq!(output.shape(), &[2, 1]);
344 /// Ok(())
345 /// })?;
346 /// # Ok::<(), Box<dyn std::error::Error>>(())
347 /// ```
348 #[doc(hidden)]
349 pub fn reshape_tensor(
350 &self,
351 input: &Tensor,
352 shape: &[usize],
353 ) -> tenferro_tensor::Result<Tensor> {
354 crate::structural::reshape(input, shape)
355 }
356
357 /// Return the faer policy selected by this operation context.
358 ///
359 /// This hidden public method is the owner-scoped extension contract used by
360 /// operation-family crates such as `tenferro-linalg`. Keeping the mapping
361 /// here prevents sibling crates from deriving a second CPU threading
362 /// policy.
363 ///
364 /// # Examples
365 ///
366 /// ```
367 /// use tenferro_cpu::CpuExecutionContext;
368 /// # fn inspect(context: &CpuExecutionContext<'_>) {
369 /// let _policy = context.faer_parallelism();
370 /// # }
371 /// ```
372 #[cfg(feature = "cpu-faer")]
373 #[doc(hidden)]
374 pub fn faer_parallelism(self) -> faer::Par {
375 match (
376 self.parallel_mode,
377 self.domain.executor_capabilities().inner_parallelism,
378 ) {
379 (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
380 faer::Par::rayon(self.thread_budget().get())
381 }
382 _ => faer::Par::Seq,
383 }
384 }
385
386 pub(crate) fn strided_exec_context(&self) -> strided_kernel::ExecContext {
387 match (
388 self.parallel_mode,
389 self.domain.executor_capabilities().inner_parallelism,
390 ) {
391 (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
392 // This is only an operation-local thread limit for strided's
393 // replay policy. The Rayon pool itself is the already-entered
394 // CpuContext pool installed by `with_native_parallelism`.
395 match strided_kernel::ExecContext::max_threads(self.thread_budget().get()) {
396 Ok(context) => context,
397 // INVARIANT: CpuExecutionContext stores a NonZeroUsize
398 // thread budget, and this branch passes that positive value.
399 Err(_) => unreachable!("CpuExecutionContext has a non-zero thread budget"),
400 }
401 }
402 _ => strided_kernel::ExecContext::serial(),
403 }
404 }
405
406 pub(crate) fn with_native_parallelism<R>(&self, operation: impl FnOnce() -> R) -> R {
407 let policy = match (
408 self.parallel_mode,
409 self.domain.executor_capabilities().inner_parallelism,
410 ) {
411 (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
412 strided_kernel::ExecutionPolicy::Rayon {
413 max_threads: self.thread_budget(),
414 }
415 }
416 _ => strided_kernel::ExecutionPolicy::Sequential,
417 };
418 strided_kernel::with_execution_policy(policy, operation)
419 }
420}
421
422fn reclaim_tensor(buffers: &mut BufferPool, tensor: Tensor) {
423 match tensor {
424 Tensor::F32(tensor) => crate::backend::reclaim_typed(buffers, tensor),
425 Tensor::F64(tensor) => crate::backend::reclaim_typed(buffers, tensor),
426 Tensor::I32(tensor) => crate::backend::reclaim_typed(buffers, tensor),
427 Tensor::I64(tensor) => crate::backend::reclaim_typed(buffers, tensor),
428 Tensor::Bool(tensor) => crate::backend::reclaim_typed(buffers, tensor),
429 Tensor::C32(tensor) => crate::backend::reclaim_typed(buffers, tensor),
430 Tensor::C64(tensor) => crate::backend::reclaim_typed(buffers, tensor),
431 }
432}
433
434/// Crate-private unentered capability for one CPU operation.
435///
436/// This is the only type that owns the resource permit and may cross the
437/// selected domain executor boundary. A [`CpuExecutionContext`] is constructed
438/// only inside an installed job or an outer child job.
439#[derive(Clone, Copy)]
440pub(crate) struct CpuOperationEntry<'a> {
441 domain: &'a CpuResourceDomain,
442 permit: &'a ResourcePermit,
443}
444
445impl<'a> CpuOperationEntry<'a> {
446 pub(crate) fn new(domain: &'a CpuResourceDomain, permit: &'a ResourcePermit) -> Self {
447 Self { domain, permit }
448 }
449
450 pub(crate) fn domain_id(self) -> CpuDomainId {
451 self.domain.id()
452 }
453
454 pub(crate) fn enter<R: Send>(
455 self,
456 parallel_mode: ParallelMode,
457 operation: impl FnOnce(&CpuExecutionContext<'_>) -> R + Send,
458 ) -> Result<R, CpuDomainExecutorError> {
459 if parallel_mode == ParallelMode::Outer {
460 return Err(CpuDomainExecutorError::Scheduling {
461 message: "CPU executor install requires Sequential or Inner mode, got Outer"
462 .to_owned(),
463 });
464 }
465 let owner = self.permit.owner();
466 with_execution_owner(owner, || {
467 install_scoped(self.domain.executor().as_ref(), || {
468 with_execution_owner(owner, || {
469 let context = CpuExecutionContext::entered(self.domain, parallel_mode);
470 operation(&context)
471 })
472 })
473 })
474 }
475
476 pub(crate) fn enter_or_reuse<R: Send>(
477 self,
478 entered: Option<&CpuExecutionContext<'_>>,
479 parallel_mode: ParallelMode,
480 operation: impl FnOnce(&CpuExecutionContext<'_>) -> R + Send,
481 ) -> Result<R, CpuDomainExecutorError> {
482 let Some(entered) = entered else {
483 return self.enter(parallel_mode, operation);
484 };
485 if parallel_mode == ParallelMode::Outer {
486 return Err(CpuDomainExecutorError::Scheduling {
487 message: "entered CPU session requires Sequential or Inner mode, got Outer"
488 .to_owned(),
489 });
490 }
491 if entered.domain_id() != self.domain.id() {
492 return Err(CpuDomainExecutorError::Scheduling {
493 message: format!(
494 "entered CPU session domain {:?} does not match operation domain {:?}",
495 entered.domain_id(),
496 self.domain.id()
497 ),
498 });
499 }
500 let owner = self.permit.owner();
501 Ok(with_execution_owner(owner, || {
502 let context = CpuExecutionContext::entered(self.domain, parallel_mode);
503 operation(&context)
504 }))
505 }
506
507 pub(crate) fn supports_infallible_session_entry(self) -> bool {
508 self.domain.ownership() == crate::CpuDomainOwnership::Managed
509 }
510
511 pub(crate) fn enter_managed_session<R: Send>(
512 self,
513 operation: impl FnOnce(CpuExecutionContext<'a>) -> R + Send,
514 ) -> R {
515 assert!(
516 self.supports_infallible_session_entry(),
517 "managed session entry requires a Tenferro-managed CPU domain"
518 );
519 let mode = self.preferred_engine_mode();
520 self.enter(mode, |_| {
521 operation(CpuExecutionContext::entered(self.domain, mode))
522 })
523 .unwrap_or_else(|error| {
524 panic!("Tenferro-managed CPU executor violated synchronous install contract: {error}")
525 })
526 }
527
528 pub(crate) fn submit_outer(
529 self,
530 len: usize,
531 operation: impl Fn(usize, &CpuExecutionContext<'_>) -> Result<(), CpuDomainExecutorError> + Sync,
532 ) -> Result<(), CpuDomainExecutorError> {
533 if !self.supports_outer() {
534 return Err(CpuDomainExecutorError::Scheduling {
535 message: format!(
536 "CPU domain {:?} does not support Outer mode",
537 self.domain.id()
538 ),
539 });
540 }
541 let owner = self.permit.owner();
542 let lane_count = len.min(self.domain.thread_budget().get());
543 let jobs = indexed_jobs(lane_count, |lane| {
544 // INVARIANT: valid lanes partition `0..len` by residue modulo the
545 // nonzero `lane_count`, so every logical job runs exactly once
546 // while the executor can schedule at most the domain budget.
547 let mut index = lane;
548 while index < len {
549 with_execution_owner(owner, || {
550 let context =
551 CpuExecutionContext::entered(self.domain, ParallelMode::Sequential);
552 operation(index, &context)
553 })?;
554 let Some(next) = index.checked_add(lane_count) else {
555 break;
556 };
557 index = next;
558 }
559 Ok(())
560 });
561 with_execution_owner(owner, || self.domain.executor().submit(&jobs))?;
562 if let Some(index) = jobs.invalid_index_attempt() {
563 return Err(CpuDomainExecutorError::Scheduling {
564 message: format!(
565 "executor requested scoped CPU lane index {index}, but the submission has {lane_count} lanes for {len} logical jobs"
566 ),
567 });
568 }
569 Ok(())
570 }
571
572 pub(crate) fn preferred_engine_mode(self) -> ParallelMode {
573 if self.domain.thread_budget().get() > 1
574 && self.domain.executor_capabilities().inner_parallelism == CpuInnerParallelism::Rayon
575 {
576 ParallelMode::Inner
577 } else {
578 ParallelMode::Sequential
579 }
580 }
581
582 pub(crate) fn preferred_provider_mode(
583 self,
584 accepts: impl Fn(ParallelMode) -> bool,
585 ) -> Result<ParallelMode, crate::CpuProviderDomainError> {
586 if self.domain.thread_budget().get() == 1 {
587 return if accepts(ParallelMode::Sequential) {
588 Ok(ParallelMode::Sequential)
589 } else {
590 Err(crate::CpuProviderDomainError::ParallelModeNotSupported {
591 mode: ParallelMode::Sequential,
592 })
593 };
594 }
595 if accepts(ParallelMode::Inner) {
596 return Ok(ParallelMode::Inner);
597 }
598 if accepts(ParallelMode::Sequential) {
599 return Ok(ParallelMode::Sequential);
600 }
601 Err(crate::CpuProviderDomainError::ParallelModeNotSupported {
602 mode: ParallelMode::Inner,
603 })
604 }
605
606 pub(crate) fn preferred_linalg_mode(self, kind: CpuBackendKind) -> ParallelMode {
607 if self.domain.thread_budget().get() == 1 {
608 return ParallelMode::Sequential;
609 }
610 match kind {
611 CpuBackendKind::Faer => self.preferred_engine_mode(),
612 // The linalg operation-family provider has not yet moved onto the
613 // provider capability traits. Preserve its existing ownership
614 // policy until that trait boundary is introduced.
615 CpuBackendKind::Blas => ParallelMode::Inner,
616 }
617 }
618
619 pub(crate) fn provider_default_compatibility_mode(self) -> ParallelMode {
620 if self.domain.thread_budget().get() == 1 {
621 ParallelMode::Sequential
622 } else {
623 ParallelMode::Inner
624 }
625 }
626
627 pub(crate) fn thread_budget(self) -> NonZeroUsize {
628 self.domain.thread_budget()
629 }
630
631 pub(crate) fn supports_outer(self) -> bool {
632 self.domain.thread_budget().get() > 1
633 && self.domain.executor_capabilities().outer_parallelism
634 }
635}
636
637/// Checked element offset and strides for a batched matrix operand.
638///
639/// # Examples
640///
641/// Providers receive this descriptor from a validated request:
642///
643/// ```
644/// use tenferro_cpu::provider::CpuGemmRequest;
645/// # fn inspect(request: &CpuGemmRequest<'_, '_, '_>) {
646/// assert!(request.lhs_layout().row_stride() != 0 || request.rows() <= 1);
647/// # }
648/// ```
649#[derive(Clone, Copy, Debug, PartialEq, Eq)]
650pub struct CpuBatchedMatrixLayout {
651 offset: isize,
652 row_stride: isize,
653 column_stride: isize,
654 batch_stride: isize,
655}
656
657impl CpuBatchedMatrixLayout {
658 #[allow(dead_code)]
659 pub(crate) fn new(
660 offset: isize,
661 row_stride: isize,
662 column_stride: isize,
663 batch_stride: isize,
664 ) -> Self {
665 Self {
666 offset,
667 row_stride,
668 column_stride,
669 batch_stride,
670 }
671 }
672
673 /// Return the checked base element offset.
674 pub fn offset(self) -> isize {
675 self.offset
676 }
677
678 /// Return the row element stride.
679 pub fn row_stride(self) -> isize {
680 self.row_stride
681 }
682
683 /// Return the column element stride.
684 pub fn column_stride(self) -> isize {
685 self.column_stride
686 }
687
688 /// Return the batch element stride.
689 pub fn batch_stride(self) -> isize {
690 self.batch_stride
691 }
692}
693
694/// Validated borrowed GEMM request.
695///
696/// A batch count of one is a single GEMM. A larger batch count is a strided
697/// batched GEMM.
698///
699/// # Examples
700///
701/// ```
702/// use tenferro_cpu::provider::CpuGemmRequest;
703/// # fn inspect(request: &CpuGemmRequest<'_, '_, '_>) {
704/// assert!(request.batch_count() >= 1);
705/// # }
706/// ```
707#[derive(Debug)]
708pub struct CpuGemmRequest<'request, 'input, 'output> {
709 lhs: &'request TensorRead<'input>,
710 rhs: &'request TensorRead<'input>,
711 output: &'request mut TensorWrite<'output>,
712 rows: usize,
713 columns: usize,
714 contracted: usize,
715 batch_count: usize,
716 lhs_layout: CpuBatchedMatrixLayout,
717 rhs_layout: CpuBatchedMatrixLayout,
718 output_layout: CpuBatchedMatrixLayout,
719 accumulation: DotGeneralAccumulation,
720}
721
722pub(crate) struct CpuGemmRequestParts<'request, 'input, 'output> {
723 pub(crate) lhs: &'request TensorRead<'input>,
724 pub(crate) rhs: &'request TensorRead<'input>,
725 pub(crate) output: &'request mut TensorWrite<'output>,
726 pub(crate) rows: usize,
727 pub(crate) columns: usize,
728 pub(crate) contracted: usize,
729 pub(crate) batch_count: usize,
730 pub(crate) lhs_layout: CpuBatchedMatrixLayout,
731 pub(crate) rhs_layout: CpuBatchedMatrixLayout,
732 pub(crate) output_layout: CpuBatchedMatrixLayout,
733 pub(crate) accumulation: DotGeneralAccumulation,
734}
735
736impl<'request, 'input, 'output> CpuGemmRequest<'request, 'input, 'output> {
737 #[allow(clippy::too_many_arguments, dead_code)]
738 pub(crate) fn new(
739 lhs: &'request TensorRead<'input>,
740 rhs: &'request TensorRead<'input>,
741 output: &'request mut TensorWrite<'output>,
742 rows: usize,
743 columns: usize,
744 contracted: usize,
745 batch_count: usize,
746 lhs_layout: CpuBatchedMatrixLayout,
747 rhs_layout: CpuBatchedMatrixLayout,
748 output_layout: CpuBatchedMatrixLayout,
749 accumulation: DotGeneralAccumulation,
750 ) -> Self {
751 Self {
752 lhs,
753 rhs,
754 output,
755 rows,
756 columns,
757 contracted,
758 batch_count,
759 lhs_layout,
760 rhs_layout,
761 output_layout,
762 accumulation,
763 }
764 }
765
766 /// Return the borrowed left input.
767 ///
768 /// # Examples
769 ///
770 /// ```
771 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
772 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
773 /// # let _ = request.lhs();
774 /// # }
775 /// ```
776 pub fn lhs(&self) -> &TensorRead<'input> {
777 self.lhs
778 }
779
780 /// Return the borrowed right input.
781 ///
782 /// # Examples
783 ///
784 /// ```
785 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
786 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
787 /// # let _ = request.rhs();
788 /// # }
789 /// ```
790 pub fn rhs(&self) -> &TensorRead<'input> {
791 self.rhs
792 }
793
794 /// Reborrow the writable output for the duration of the current call.
795 pub fn output(&mut self) -> &mut TensorWrite<'output> {
796 self.output
797 }
798
799 /// Return the number of output rows.
800 ///
801 /// # Examples
802 ///
803 /// ```
804 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
805 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
806 /// # let _ = request.rows();
807 /// # }
808 /// ```
809 pub fn rows(&self) -> usize {
810 self.rows
811 }
812
813 /// Return the number of output columns.
814 ///
815 /// # Examples
816 ///
817 /// ```
818 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
819 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
820 /// # let _ = request.columns();
821 /// # }
822 /// ```
823 pub fn columns(&self) -> usize {
824 self.columns
825 }
826
827 /// Return the contracted dimension.
828 pub fn contracted(&self) -> usize {
829 self.contracted
830 }
831
832 /// Return the number of matrices in this strided batch.
833 pub fn batch_count(&self) -> usize {
834 self.batch_count
835 }
836
837 /// Return the left input matrix layout.
838 pub fn lhs_layout(&self) -> CpuBatchedMatrixLayout {
839 self.lhs_layout
840 }
841
842 /// Return the right input matrix layout.
843 pub fn rhs_layout(&self) -> CpuBatchedMatrixLayout {
844 self.rhs_layout
845 }
846
847 /// Return the output matrix layout.
848 pub fn output_layout(&self) -> CpuBatchedMatrixLayout {
849 self.output_layout
850 }
851
852 /// Return conjugation and alpha/beta update semantics.
853 pub fn accumulation(&self) -> DotGeneralAccumulation {
854 self.accumulation
855 }
856
857 pub(crate) fn into_parts(self) -> CpuGemmRequestParts<'request, 'input, 'output> {
858 CpuGemmRequestParts {
859 lhs: self.lhs,
860 rhs: self.rhs,
861 output: self.output,
862 rows: self.rows,
863 columns: self.columns,
864 contracted: self.contracted,
865 batch_count: self.batch_count,
866 lhs_layout: self.lhs_layout,
867 rhs_layout: self.rhs_layout,
868 output_layout: self.output_layout,
869 accumulation: self.accumulation,
870 }
871 }
872}
873
874/// Validated output-free borrowed GEMM request for full-overwrite destinations.
875///
876/// Mirrors [`CpuGemmRequest`] minus the writable output: the destination is
877/// provided separately as `&mut [MaybeUninit<u8>]` to [`CpuUninitGemmProvider`]
878/// callers. This request is only used for `beta == 0` (full-overwrite)
879/// accumulations, where the destination is never read.
880///
881/// # Examples
882///
883/// ```
884/// use tenferro_cpu::provider::CpuGemmUninitRequest;
885/// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
886/// assert!(request.batch_count() >= 1);
887/// # }
888/// ```
889#[derive(Debug)]
890/// Output-free prepared GEMM request for uninitialized full-overwrite
891/// execution (see `CpuUninitGemmProvider::gemm_into_uninit`).
892///
893/// # Examples
894///
895/// ```
896/// use tenferro_cpu::provider::CpuGemmUninitRequest;
897/// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
898/// # let _ = request.rows();
899/// # let _ = request.columns();
900/// # let _ = request.contracted();
901/// # }
902/// ```
903pub struct CpuGemmUninitRequest<'request, 'input> {
904 lhs: &'request TensorRead<'input>,
905 rhs: &'request TensorRead<'input>,
906 rows: usize,
907 columns: usize,
908 contracted: usize,
909 batch_count: usize,
910 lhs_layout: CpuBatchedMatrixLayout,
911 rhs_layout: CpuBatchedMatrixLayout,
912 output_layout: CpuBatchedMatrixLayout,
913 accumulation: DotGeneralAccumulation,
914}
915
916#[cfg(feature = "cpu-faer")]
917pub(crate) struct CpuGemmUninitRequestParts<'request, 'input> {
918 pub(crate) lhs: &'request TensorRead<'input>,
919 pub(crate) rhs: &'request TensorRead<'input>,
920 pub(crate) rows: usize,
921 pub(crate) columns: usize,
922 pub(crate) contracted: usize,
923 pub(crate) batch_count: usize,
924 pub(crate) lhs_layout: CpuBatchedMatrixLayout,
925 pub(crate) rhs_layout: CpuBatchedMatrixLayout,
926 pub(crate) output_layout: CpuBatchedMatrixLayout,
927 pub(crate) accumulation: DotGeneralAccumulation,
928}
929
930impl<'request, 'input> CpuGemmUninitRequest<'request, 'input> {
931 #[allow(clippy::too_many_arguments, dead_code)]
932 pub(crate) fn new(
933 lhs: &'request TensorRead<'input>,
934 rhs: &'request TensorRead<'input>,
935 rows: usize,
936 columns: usize,
937 contracted: usize,
938 batch_count: usize,
939 lhs_layout: CpuBatchedMatrixLayout,
940 rhs_layout: CpuBatchedMatrixLayout,
941 output_layout: CpuBatchedMatrixLayout,
942 accumulation: DotGeneralAccumulation,
943 ) -> Self {
944 Self {
945 lhs,
946 rhs,
947 rows,
948 columns,
949 contracted,
950 batch_count,
951 lhs_layout,
952 rhs_layout,
953 output_layout,
954 accumulation,
955 }
956 }
957
958 /// Return the `lhs` of this output-free request.
959 ///
960 /// # Examples
961 ///
962 /// ```
963 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
964 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
965 /// # let _ = request.lhs();
966 /// # }
967 /// ```
968 pub fn lhs(&self) -> &TensorRead<'input> {
969 self.lhs
970 }
971
972 /// Return the `rhs` of this output-free request.
973 ///
974 /// # Examples
975 ///
976 /// ```
977 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
978 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
979 /// # let _ = request.rhs();
980 /// # }
981 /// ```
982 pub fn rhs(&self) -> &TensorRead<'input> {
983 self.rhs
984 }
985
986 /// Return the `rows` of this output-free request.
987 ///
988 /// # Examples
989 ///
990 /// ```
991 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
992 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
993 /// # let _ = request.rows();
994 /// # }
995 /// ```
996 pub fn rows(&self) -> usize {
997 self.rows
998 }
999
1000 /// Return the `columns` of this output-free request.
1001 ///
1002 /// # Examples
1003 ///
1004 /// ```
1005 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1006 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1007 /// # let _ = request.columns();
1008 /// # }
1009 /// ```
1010 pub fn columns(&self) -> usize {
1011 self.columns
1012 }
1013
1014 /// Return the `contracted` of this output-free request.
1015 ///
1016 /// # Examples
1017 ///
1018 /// ```
1019 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1020 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1021 /// # let _ = request.contracted();
1022 /// # }
1023 /// ```
1024 pub fn contracted(&self) -> usize {
1025 self.contracted
1026 }
1027
1028 /// Return the `batch_count` of this output-free request.
1029 ///
1030 /// # Examples
1031 ///
1032 /// ```
1033 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1034 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1035 /// # let _ = request.batch_count();
1036 /// # }
1037 /// ```
1038 pub fn batch_count(&self) -> usize {
1039 self.batch_count
1040 }
1041
1042 /// Return the `lhs_layout` of this output-free request.
1043 ///
1044 /// # Examples
1045 ///
1046 /// ```
1047 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1048 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1049 /// # let _ = request.lhs_layout();
1050 /// # }
1051 /// ```
1052 pub fn lhs_layout(&self) -> CpuBatchedMatrixLayout {
1053 self.lhs_layout
1054 }
1055
1056 /// Return the `rhs_layout` of this output-free request.
1057 ///
1058 /// # Examples
1059 ///
1060 /// ```
1061 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1062 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1063 /// # let _ = request.rhs_layout();
1064 /// # }
1065 /// ```
1066 pub fn rhs_layout(&self) -> CpuBatchedMatrixLayout {
1067 self.rhs_layout
1068 }
1069
1070 /// Return the `output_layout` of this output-free request.
1071 ///
1072 /// # Examples
1073 ///
1074 /// ```
1075 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1076 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1077 /// # let _ = request.output_layout();
1078 /// # }
1079 /// ```
1080 pub fn output_layout(&self) -> CpuBatchedMatrixLayout {
1081 self.output_layout
1082 }
1083
1084 /// Return the `accumulation` of this output-free request.
1085 ///
1086 /// # Examples
1087 ///
1088 /// ```
1089 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1090 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1091 /// # let _ = request.accumulation();
1092 /// # }
1093 /// ```
1094 pub fn accumulation(&self) -> DotGeneralAccumulation {
1095 self.accumulation
1096 }
1097
1098 #[cfg(feature = "cpu-faer")]
1099 pub(crate) fn into_parts(self) -> CpuGemmUninitRequestParts<'request, 'input> {
1100 CpuGemmUninitRequestParts {
1101 lhs: self.lhs,
1102 rhs: self.rhs,
1103 rows: self.rows,
1104 columns: self.columns,
1105 contracted: self.contracted,
1106 batch_count: self.batch_count,
1107 lhs_layout: self.lhs_layout,
1108 rhs_layout: self.rhs_layout,
1109 output_layout: self.output_layout,
1110 accumulation: self.accumulation,
1111 }
1112 }
1113}
1114
1115/// Validated borrowed grouped-GEMM request.
1116///
1117/// # Examples
1118///
1119/// ```
1120/// use tenferro_cpu::provider::CpuGroupedGemmRequest;
1121/// # fn inspect(request: &CpuGroupedGemmRequest<'_, '_, '_>) {
1122/// assert_eq!(request.jobs().len(), request.jobs().iter().count());
1123/// # }
1124/// ```
1125#[derive(Debug)]
1126pub struct CpuGroupedGemmRequest<'request, 'input, 'output> {
1127 lhs: &'request TensorRead<'input>,
1128 rhs: &'request TensorRead<'input>,
1129 output: &'request mut TensorWrite<'output>,
1130 jobs: &'request [GroupedGemmJob],
1131 accumulation: DotGeneralAccumulation,
1132}
1133
1134impl<'request, 'input, 'output> CpuGroupedGemmRequest<'request, 'input, 'output> {
1135 #[allow(dead_code)]
1136 pub(crate) fn new(
1137 lhs: &'request TensorRead<'input>,
1138 rhs: &'request TensorRead<'input>,
1139 output: &'request mut TensorWrite<'output>,
1140 jobs: &'request [GroupedGemmJob],
1141 accumulation: DotGeneralAccumulation,
1142 ) -> Self {
1143 Self {
1144 lhs,
1145 rhs,
1146 output,
1147 jobs,
1148 accumulation,
1149 }
1150 }
1151
1152 /// Return the borrowed left input.
1153 pub fn lhs(&self) -> &TensorRead<'input> {
1154 self.lhs
1155 }
1156
1157 /// Return the borrowed right input.
1158 pub fn rhs(&self) -> &TensorRead<'input> {
1159 self.rhs
1160 }
1161
1162 /// Reborrow the writable output.
1163 pub fn output(&mut self) -> &mut TensorWrite<'output> {
1164 self.output
1165 }
1166
1167 /// Return ordered, pairwise-disjoint validated jobs.
1168 pub fn jobs(&self) -> &[GroupedGemmJob] {
1169 self.jobs
1170 }
1171
1172 /// Return shared conjugation and alpha/beta semantics.
1173 pub fn accumulation(&self) -> DotGeneralAccumulation {
1174 self.accumulation
1175 }
1176
1177 pub(crate) fn into_parts(
1178 self,
1179 ) -> (
1180 &'request TensorRead<'input>,
1181 &'request TensorRead<'input>,
1182 &'request mut TensorWrite<'output>,
1183 &'request [GroupedGemmJob],
1184 DotGeneralAccumulation,
1185 ) {
1186 (
1187 self.lhs,
1188 self.rhs,
1189 self.output,
1190 self.jobs,
1191 self.accumulation,
1192 )
1193 }
1194}
1195
1196/// Engine-requested layout materialization.
1197///
1198/// # Examples
1199///
1200/// ```
1201/// use tenferro_cpu::provider::CpuLayoutTransformIntent;
1202/// assert_eq!(
1203/// CpuLayoutTransformIntent::CanonicalColumnMajor,
1204/// CpuLayoutTransformIntent::CanonicalColumnMajor,
1205/// );
1206/// ```
1207#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1208pub enum CpuLayoutTransformIntent {
1209 /// Materialize a compact canonical column-major tensor.
1210 CanonicalColumnMajor,
1211}
1212
1213/// Validated borrowed layout-transform request.
1214///
1215/// # Examples
1216///
1217/// ```
1218/// use tenferro_cpu::provider::{CpuLayoutTransformIntent, CpuLayoutTransformRequest};
1219/// # fn inspect(request: &CpuLayoutTransformRequest<'_, '_, '_>) {
1220/// assert_eq!(request.intent(), CpuLayoutTransformIntent::CanonicalColumnMajor);
1221/// # }
1222/// ```
1223#[derive(Debug)]
1224pub struct CpuLayoutTransformRequest<'request, 'input, 'output> {
1225 input: &'request TensorRead<'input>,
1226 output: &'request mut TensorWrite<'output>,
1227 intent: CpuLayoutTransformIntent,
1228 conjugate: bool,
1229}
1230
1231impl<'request, 'input, 'output> CpuLayoutTransformRequest<'request, 'input, 'output> {
1232 #[allow(dead_code)]
1233 pub(crate) fn new(
1234 input: &'request TensorRead<'input>,
1235 output: &'request mut TensorWrite<'output>,
1236 intent: CpuLayoutTransformIntent,
1237 conjugate: bool,
1238 ) -> Self {
1239 Self {
1240 input,
1241 output,
1242 intent,
1243 conjugate,
1244 }
1245 }
1246
1247 /// Return the borrowed input.
1248 pub fn input(&self) -> &TensorRead<'input> {
1249 self.input
1250 }
1251
1252 /// Reborrow the writable output.
1253 pub fn output(&mut self) -> &mut TensorWrite<'output> {
1254 self.output
1255 }
1256
1257 /// Return the requested materialization intent.
1258 pub fn intent(&self) -> CpuLayoutTransformIntent {
1259 self.intent
1260 }
1261
1262 /// Return whether materialization must conjugate each input element.
1263 ///
1264 /// # Examples
1265 ///
1266 /// ```
1267 /// use tenferro_cpu::provider::CpuLayoutTransformRequest;
1268 /// # fn inspect(request: &CpuLayoutTransformRequest<'_, '_, '_>) {
1269 /// let _must_conjugate = request.conjugate();
1270 /// # }
1271 /// ```
1272 pub fn conjugate(&self) -> bool {
1273 self.conjugate
1274 }
1275
1276 pub(crate) fn into_parts(
1277 self,
1278 ) -> (
1279 &'request TensorRead<'input>,
1280 &'request mut TensorWrite<'output>,
1281 CpuLayoutTransformIntent,
1282 bool,
1283 ) {
1284 (self.input, self.output, self.intent, self.conjugate)
1285 }
1286}
1287
1288/// Validated ordered contraction-role groups.
1289///
1290/// # Examples
1291///
1292/// ```
1293/// use tenferro_cpu::provider::CpuDotGeneralRequest;
1294/// # fn inspect(request: &CpuDotGeneralRequest<'_, '_, '_>) {
1295/// let _pairs = request.axes().contracting_pairs().count();
1296/// # }
1297/// ```
1298#[derive(Clone, Copy, Debug)]
1299pub struct CpuContractionAxes<'a> {
1300 lhs_rank: usize,
1301 rhs_rank: usize,
1302 lhs_contracting: &'a [usize],
1303 rhs_contracting: &'a [usize],
1304 lhs_batch: &'a [usize],
1305 rhs_batch: &'a [usize],
1306 lhs_role_mask: Option<u64>,
1307 rhs_role_mask: Option<u64>,
1308}
1309
1310impl<'a> CpuContractionAxes<'a> {
1311 #[allow(clippy::too_many_arguments, dead_code)]
1312 pub(crate) fn new(
1313 lhs_rank: usize,
1314 rhs_rank: usize,
1315 lhs_contracting: &'a [usize],
1316 rhs_contracting: &'a [usize],
1317 lhs_batch: &'a [usize],
1318 rhs_batch: &'a [usize],
1319 lhs_role_mask: Option<u64>,
1320 rhs_role_mask: Option<u64>,
1321 ) -> Self {
1322 Self {
1323 lhs_rank,
1324 rhs_rank,
1325 lhs_contracting,
1326 rhs_contracting,
1327 lhs_batch,
1328 rhs_batch,
1329 lhs_role_mask,
1330 rhs_role_mask,
1331 }
1332 }
1333
1334 /// Return ordered contracting-axis pairs.
1335 pub fn contracting_pairs(&self) -> impl ExactSizeIterator<Item = (usize, usize)> + '_ {
1336 self.lhs_contracting
1337 .iter()
1338 .copied()
1339 .zip(self.rhs_contracting.iter().copied())
1340 }
1341
1342 /// Return ordered batch-axis pairs.
1343 pub fn batch_pairs(&self) -> impl ExactSizeIterator<Item = (usize, usize)> + '_ {
1344 self.lhs_batch
1345 .iter()
1346 .copied()
1347 .zip(self.rhs_batch.iter().copied())
1348 }
1349
1350 /// Return left axes that are neither contracting nor batch axes.
1351 pub fn lhs_free_axes(&self) -> impl Iterator<Item = usize> + '_ {
1352 (0..self.lhs_rank).filter(move |&axis| !self.lhs_axis_has_role(axis))
1353 }
1354
1355 /// Return right axes that are neither contracting nor batch axes.
1356 pub fn rhs_free_axes(&self) -> impl Iterator<Item = usize> + '_ {
1357 (0..self.rhs_rank).filter(move |&axis| !self.rhs_axis_has_role(axis))
1358 }
1359
1360 fn lhs_axis_has_role(&self, axis: usize) -> bool {
1361 self.lhs_role_mask.map_or_else(
1362 || self.lhs_contracting.contains(&axis) || self.lhs_batch.contains(&axis),
1363 |mask| mask & (1_u64 << axis) != 0,
1364 )
1365 }
1366
1367 fn rhs_axis_has_role(&self, axis: usize) -> bool {
1368 self.rhs_role_mask.map_or_else(
1369 || self.rhs_contracting.contains(&axis) || self.rhs_batch.contains(&axis),
1370 |mask| mask & (1_u64 << axis) != 0,
1371 )
1372 }
1373}
1374
1375/// Validated borrowed semantic binary `dot_general` request.
1376///
1377/// # Examples
1378///
1379/// ```
1380/// use tenferro_cpu::provider::CpuDotGeneralRequest;
1381/// # fn inspect(request: &CpuDotGeneralRequest<'_, '_, '_>) {
1382/// let _ = request.accumulation();
1383/// # }
1384/// ```
1385#[derive(Debug)]
1386pub struct CpuDotGeneralRequest<'request, 'input, 'output> {
1387 lhs: &'request TensorRead<'input>,
1388 rhs: &'request TensorRead<'input>,
1389 output: &'request mut TensorWrite<'output>,
1390 axes: CpuContractionAxes<'request>,
1391 accumulation: DotGeneralAccumulation,
1392}
1393
1394impl<'request, 'input, 'output> CpuDotGeneralRequest<'request, 'input, 'output> {
1395 #[allow(dead_code)]
1396 pub(crate) fn new(
1397 lhs: &'request TensorRead<'input>,
1398 rhs: &'request TensorRead<'input>,
1399 output: &'request mut TensorWrite<'output>,
1400 axes: CpuContractionAxes<'request>,
1401 accumulation: DotGeneralAccumulation,
1402 ) -> Self {
1403 Self {
1404 lhs,
1405 rhs,
1406 output,
1407 axes,
1408 accumulation,
1409 }
1410 }
1411
1412 /// Return the borrowed left input.
1413 pub fn lhs(&self) -> &TensorRead<'input> {
1414 self.lhs
1415 }
1416
1417 /// Return the borrowed right input.
1418 pub fn rhs(&self) -> &TensorRead<'input> {
1419 self.rhs
1420 }
1421
1422 /// Reborrow the writable output.
1423 pub fn output(&mut self) -> &mut TensorWrite<'output> {
1424 self.output
1425 }
1426
1427 /// Return the validated ordered axis groups.
1428 pub fn axes(&self) -> &CpuContractionAxes<'request> {
1429 &self.axes
1430 }
1431
1432 /// Return conjugation and alpha/beta update semantics.
1433 pub fn accumulation(&self) -> DotGeneralAccumulation {
1434 self.accumulation
1435 }
1436
1437 /// Consume the request and return the validated operand borrows.
1438 ///
1439 /// External general-contraction providers use this when they need
1440 /// simultaneous immutable access to both inputs and mutable access to the
1441 /// output.
1442 ///
1443 /// # Examples
1444 ///
1445 /// ```
1446 /// use tenferro_cpu::provider::CpuDotGeneralRequest;
1447 ///
1448 /// fn consume_request(request: CpuDotGeneralRequest<'_, '_, '_>) {
1449 /// let (_lhs, _rhs, _output, _axes, _accumulation) = request.into_parts();
1450 /// }
1451 /// ```
1452 pub fn into_parts(
1453 self,
1454 ) -> (
1455 &'request TensorRead<'input>,
1456 &'request TensorRead<'input>,
1457 &'request mut TensorWrite<'output>,
1458 CpuContractionAxes<'request>,
1459 DotGeneralAccumulation,
1460 ) {
1461 (
1462 self.lhs,
1463 self.rhs,
1464 self.output,
1465 self.axes,
1466 self.accumulation,
1467 )
1468 }
1469}
1470
1471/// Provider for validated GEMM-family requests.
1472///
1473/// # Examples
1474///
1475/// Trait objects are supported directly:
1476///
1477/// ```
1478/// use tenferro_cpu::provider::CpuGemmProvider;
1479/// # fn accepts_provider(_: &dyn CpuGemmProvider) {}
1480/// ```
1481pub trait CpuGemmProvider: fmt::Debug + Send + Sync + 'static {
1482 /// Return immutable count, placement, and fan-out capabilities.
1483 ///
1484 /// This declaration must describe controls actually applied and restored
1485 /// by the provider adapter around each call. Merely discovering a runtime
1486 /// symbol is insufficient. A provider bundle samples this method exactly
1487 /// once during construction and keeps that snapshot for its lifetime; the
1488 /// returned contract must therefore remain valid for the provider object.
1489 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;
1490
1491 /// Execute one validated GEMM.
1492 ///
1493 /// # Errors
1494 ///
1495 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1496 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1497 /// fails. A detected inconsistency in engine-attested request metadata is
1498 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
1499 /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
1500 fn gemm(
1501 &self,
1502 context: &CpuExecutionContext<'_>,
1503 request: CpuGemmRequest<'_, '_, '_>,
1504 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1505
1506 /// Execute one validated strided-batched GEMM.
1507 ///
1508 /// # Errors
1509 ///
1510 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1511 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1512 /// fails. A detected inconsistency in engine-attested request metadata is
1513 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
1514 /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
1515 fn strided_batched_gemm(
1516 &self,
1517 context: &CpuExecutionContext<'_>,
1518 request: CpuGemmRequest<'_, '_, '_>,
1519 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1520
1521 /// Execute one validated grouped GEMM.
1522 ///
1523 /// # Errors
1524 ///
1525 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1526 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1527 /// fails. A detected inconsistency in engine-attested request metadata is
1528 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
1529 /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
1530 fn grouped_gemm(
1531 &self,
1532 context: &CpuExecutionContext<'_>,
1533 request: CpuGroupedGemmRequest<'_, '_, '_>,
1534 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1535
1536 /// Return the structural full-overwrite witness for this provider.
1537 ///
1538 /// Returns `Some(self)` only for types that implement
1539 /// [`CpuUninitGemmProvider`] via an `unsafe impl`, so a `Some` witness is
1540 /// structural proof that the provider asserted the full-overwrite
1541 /// destination contract. Defaults to `None` (opted out).
1542 ///
1543 /// # Examples
1544 ///
1545 /// ```
1546 /// use tenferro_cpu::provider::CpuGemmProvider;
1547 /// # fn inspect(provider: &dyn CpuGemmProvider) {
1548 /// # let _ = provider.uninit_provider();
1549 /// # }
1550 /// ```
1551 fn uninit_provider(&self) -> Option<&dyn CpuUninitGemmProvider> {
1552 None
1553 }
1554}
1555
1556/// Provider for validated GEMM-family requests that fully overwrite their
1557/// destination (only `beta == 0` accumulations).
1558///
1559/// # Safety
1560///
1561/// Implementors assert that [`CpuUninitGemmProvider::gemm_into_uninit`] writes
1562/// every logical output element before returning
1563/// [`CpuProviderOutcome::Executed`]; partial writes make the caller's
1564/// `assume_init` undefined behavior. Implementations must never read the
1565/// destination. This trait is only invoked for `beta == 0` accumulations.
1566///
1567/// # Examples
1568///
1569/// ```
1570/// use tenferro_cpu::provider::CpuUninitGemmProvider;
1571/// # fn accepts_provider(_: &dyn CpuUninitGemmProvider) {}
1572/// ```
1573pub unsafe trait CpuUninitGemmProvider: CpuGemmProvider {
1574 /// Execute one validated full-overwrite GEMM into uninitialized bytes.
1575 ///
1576 /// # Safety
1577 ///
1578 /// Must write every element of the destination represented by
1579 /// `output_bytes` (per `request.output_layout()`) before returning
1580 /// `Executed`; partial writes are UB at the caller's `assume_init`. The
1581 /// destination is never read. Invoked only for `beta == 0` accumulations.
1582 ///
1583 /// # Errors
1584 ///
1585 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1586 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1587 /// fails. A detected inconsistency in engine-attested request metadata is
1588 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
1589 /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
1590 ///
1591 /// # Examples
1592 ///
1593 /// ```
1594 /// use tenferro_cpu::provider::{CpuGemmUninitRequest, CpuProviderOutcome, CpuUninitGemmProvider};
1595 /// use tenferro_cpu::CpuExecutionContext;
1596 /// use std::mem::MaybeUninit;
1597 /// # fn inspect(
1598 /// # provider: &dyn CpuUninitGemmProvider,
1599 /// # context: &CpuExecutionContext<'_>,
1600 /// # request: CpuGemmUninitRequest<'_, '_>,
1601 /// # output_bytes: &mut [MaybeUninit<u8>],
1602 /// # ) -> Option<tenferro_tensor::Result<CpuProviderOutcome>> {
1603 /// # let _ = (provider, context, request, output_bytes);
1604 /// # None
1605 /// # }
1606 /// ```
1607 unsafe fn gemm_into_uninit(
1608 &self,
1609 context: &CpuExecutionContext<'_>,
1610 request: CpuGemmUninitRequest<'_, '_>,
1611 output_bytes: &mut [MaybeUninit<u8>],
1612 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1613}
1614
1615/// Provider for engine-owned tensor materialization.
1616///
1617/// # Examples
1618///
1619/// ```
1620/// use tenferro_cpu::provider::CpuLayoutTransformProvider;
1621/// # fn accepts_provider(_: &dyn CpuLayoutTransformProvider) {}
1622/// ```
1623pub trait CpuLayoutTransformProvider: fmt::Debug + Send + Sync + 'static {
1624 /// Return immutable count, placement, and fan-out capabilities.
1625 ///
1626 /// A provider bundle samples this method exactly once during construction
1627 /// and uses the stored descriptor for all validation and dispatch.
1628 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;
1629
1630 /// Materialize one validated input into a preallocated output.
1631 ///
1632 /// # Errors
1633 ///
1634 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1635 /// [`tenferro_tensor::Error::BackendFailure`] when execution fails. A
1636 /// detected inconsistency in engine-attested layout or range metadata is
1637 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported layouts
1638 /// use [`CpuProviderOutcome::Unsupported`] instead.
1639 fn materialize(
1640 &self,
1641 context: &CpuExecutionContext<'_>,
1642 request: CpuLayoutTransformRequest<'_, '_, '_>,
1643 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1644
1645 /// Return the structural full-overwrite witness for this provider.
1646 ///
1647 /// Returns `Some(self)` only for types that implement
1648 /// [`CpuUninitLayoutTransformProvider`] via an `unsafe impl`, so a `Some`
1649 /// witness is structural proof that the provider asserted the
1650 /// full-overwrite destination contract. Defaults to `None` (opted out).
1651 ///
1652 /// # Examples
1653 ///
1654 /// ```
1655 /// use tenferro_cpu::provider::CpuLayoutTransformProvider;
1656 /// # fn inspect(provider: &dyn CpuLayoutTransformProvider) {
1657 /// # let _ = provider.uninit_provider();
1658 /// # }
1659 /// ```
1660 fn uninit_provider(&self) -> Option<&dyn CpuUninitLayoutTransformProvider> {
1661 None
1662 }
1663}
1664
1665/// Provider for engine-owned tensor materialization into uninitialized bytes.
1666///
1667/// # Safety
1668///
1669/// Implementors assert that [`CpuUninitLayoutTransformProvider::materialize_into_uninit`]
1670/// writes every element of `output_bytes` before returning
1671/// [`CpuProviderOutcome::Executed`]; partial writes make the caller's
1672/// `assume_init` undefined behavior. Implementations must never read the
1673/// destination.
1674///
1675/// # Examples
1676///
1677/// ```
1678/// use tenferro_cpu::provider::CpuUninitLayoutTransformProvider;
1679/// # fn accepts_provider(_: &dyn CpuUninitLayoutTransformProvider) {}
1680/// ```
1681pub unsafe trait CpuUninitLayoutTransformProvider: CpuLayoutTransformProvider {
1682 /// Materialize one validated input into uninitialized bytes.
1683 ///
1684 /// # Safety
1685 ///
1686 /// Must write every element of `output_bytes` (a compact column-major
1687 /// destination of the input shape for
1688 /// [`CpuLayoutTransformIntent::CanonicalColumnMajor`]) before returning
1689 /// `Executed`; partial writes are UB at the caller's `assume_init`. The
1690 /// destination is never read.
1691 ///
1692 /// # Errors
1693 ///
1694 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1695 /// [`tenferro_tensor::Error::BackendFailure`] when execution fails. A
1696 /// detected inconsistency in engine-attested layout or range metadata is
1697 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported layouts
1698 /// use [`CpuProviderOutcome::Unsupported`] instead.
1699 ///
1700 /// # Examples
1701 ///
1702 /// ```
1703 /// use tenferro_cpu::provider::{CpuLayoutTransformIntent, CpuProviderOutcome, CpuUninitLayoutTransformProvider};
1704 /// use tenferro_cpu::CpuExecutionContext;
1705 /// use tenferro_tensor::TensorRead;
1706 /// use std::mem::MaybeUninit;
1707 /// # fn inspect(
1708 /// # provider: &dyn CpuUninitLayoutTransformProvider,
1709 /// # context: &CpuExecutionContext<'_>,
1710 /// # input: &TensorRead<'_>,
1711 /// # intent: CpuLayoutTransformIntent,
1712 /// # output_bytes: &mut [MaybeUninit<u8>],
1713 /// # ) -> Option<tenferro_tensor::Result<CpuProviderOutcome>> {
1714 /// # let _ = (provider, context, input, intent, output_bytes);
1715 /// # None
1716 /// # }
1717 /// ```
1718 unsafe fn materialize_into_uninit(
1719 &self,
1720 context: &CpuExecutionContext<'_>,
1721 input: &TensorRead<'_>,
1722 intent: CpuLayoutTransformIntent,
1723 conjugate: bool,
1724 output_bytes: &mut [MaybeUninit<u8>],
1725 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1726}
1727
1728/// Provider for complete validated binary `dot_general` requests.
1729///
1730/// # Examples
1731///
1732/// ```
1733/// use tenferro_cpu::provider::CpuGeneralContractionProvider;
1734/// # fn accepts_provider(_: &dyn CpuGeneralContractionProvider) {}
1735/// ```
1736pub trait CpuGeneralContractionProvider: fmt::Debug + Send + Sync + 'static {
1737 /// Return immutable count, placement, and fan-out capabilities.
1738 ///
1739 /// A provider bundle samples this method exactly once during construction
1740 /// and uses the stored descriptor for all validation and dispatch.
1741 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;
1742
1743 /// Execute one complete semantic contraction.
1744 ///
1745 /// # Errors
1746 ///
1747 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1748 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1749 /// fails. A detected inconsistency in engine-attested axes or range
1750 /// metadata is returned as [`tenferro_tensor::Error::Validation`].
1751 /// Unsupported contractions use [`CpuProviderOutcome::Unsupported`]
1752 /// instead.
1753 fn dot_general(
1754 &self,
1755 context: &CpuExecutionContext<'_>,
1756 request: CpuDotGeneralRequest<'_, '_, '_>,
1757 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1758}
1759
1760/// Built-in faer GEMM provider.
1761///
1762/// # Examples
1763///
1764/// ```
1765/// use tenferro_cpu::provider::{CpuGemmProvider, FaerGemmProvider};
1766/// let provider: &dyn CpuGemmProvider = &FaerGemmProvider;
1767/// let _ = provider;
1768/// ```
1769#[derive(Clone, Copy, Debug, Default)]
1770pub struct FaerGemmProvider;
1771
1772impl CpuGemmProvider for FaerGemmProvider {
1773 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
1774 engine_worker_capabilities()
1775 }
1776
1777 fn gemm(
1778 &self,
1779 context: &CpuExecutionContext<'_>,
1780 request: CpuGemmRequest<'_, '_, '_>,
1781 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1782 #[cfg(feature = "cpu-faer")]
1783 {
1784 crate::gemm::execute_faer_gemm_request(context, request)
1785 }
1786 #[cfg(not(feature = "cpu-faer"))]
1787 {
1788 let _ = (context, request);
1789 Ok(CpuProviderOutcome::Unsupported(
1790 CpuProviderUnsupported::RuntimeUnavailable,
1791 ))
1792 }
1793 }
1794
1795 fn strided_batched_gemm(
1796 &self,
1797 context: &CpuExecutionContext<'_>,
1798 request: CpuGemmRequest<'_, '_, '_>,
1799 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1800 self.gemm(context, request)
1801 }
1802
1803 fn grouped_gemm(
1804 &self,
1805 context: &CpuExecutionContext<'_>,
1806 request: CpuGroupedGemmRequest<'_, '_, '_>,
1807 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1808 #[cfg(feature = "cpu-faer")]
1809 {
1810 crate::gemm::execute_faer_grouped_request(context, request)
1811 }
1812 #[cfg(not(feature = "cpu-faer"))]
1813 {
1814 let _ = (context, request);
1815 Ok(CpuProviderOutcome::Unsupported(
1816 CpuProviderUnsupported::RuntimeUnavailable,
1817 ))
1818 }
1819 }
1820
1821 fn uninit_provider(&self) -> Option<&dyn CpuUninitGemmProvider> {
1822 Some(self)
1823 }
1824}
1825
1826// SAFETY: `gemm_into_uninit` runs the faer GEMM with `Accum::Replace` semantics
1827// (beta == 0 never reads the destination) and writes zeros for empty
1828// contractions, so every logical output element is written before `Executed`.
1829// See `crate::gemm::execute_faer_gemm_request_into_uninit`.
1830unsafe impl CpuUninitGemmProvider for FaerGemmProvider {
1831 unsafe fn gemm_into_uninit(
1832 &self,
1833 context: &CpuExecutionContext<'_>,
1834 request: CpuGemmUninitRequest<'_, '_>,
1835 output_bytes: &mut [MaybeUninit<u8>],
1836 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1837 #[cfg(feature = "cpu-faer")]
1838 {
1839 crate::gemm::execute_faer_gemm_request_into_uninit(context, request, output_bytes)
1840 }
1841 #[cfg(not(feature = "cpu-faer"))]
1842 {
1843 let _ = (context, request, output_bytes);
1844 Ok(CpuProviderOutcome::Unsupported(
1845 CpuProviderUnsupported::RuntimeUnavailable,
1846 ))
1847 }
1848 }
1849}
1850
1851/// Built-in BLAS GEMM provider.
1852///
1853/// # Examples
1854///
1855/// ```
1856/// use tenferro_cpu::provider::{BlasGemmProvider, CpuGemmProvider};
1857/// let provider: &dyn CpuGemmProvider = &BlasGemmProvider;
1858/// let _ = provider;
1859/// ```
1860#[derive(Clone, Copy, Debug, Default)]
1861pub struct BlasGemmProvider;
1862
1863impl CpuGemmProvider for BlasGemmProvider {
1864 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
1865 #[cfg(feature = "cpu-blas")]
1866 {
1867 builtin_blas_execution_capabilities()
1868 }
1869 #[cfg(not(feature = "cpu-blas"))]
1870 {
1871 // This build returns RuntimeUnavailable without invoking BLAS.
1872 serial_capabilities()
1873 }
1874 }
1875
1876 fn gemm(
1877 &self,
1878 context: &CpuExecutionContext<'_>,
1879 request: CpuGemmRequest<'_, '_, '_>,
1880 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1881 #[cfg(feature = "cpu-blas")]
1882 {
1883 crate::gemm::execute_blas_gemm_request(context, request)
1884 }
1885 #[cfg(not(feature = "cpu-blas"))]
1886 {
1887 let _ = (context, request);
1888 Ok(CpuProviderOutcome::Unsupported(
1889 CpuProviderUnsupported::RuntimeUnavailable,
1890 ))
1891 }
1892 }
1893
1894 fn strided_batched_gemm(
1895 &self,
1896 context: &CpuExecutionContext<'_>,
1897 request: CpuGemmRequest<'_, '_, '_>,
1898 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1899 self.gemm(context, request)
1900 }
1901
1902 fn grouped_gemm(
1903 &self,
1904 context: &CpuExecutionContext<'_>,
1905 request: CpuGroupedGemmRequest<'_, '_, '_>,
1906 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1907 #[cfg(feature = "cpu-blas")]
1908 {
1909 crate::gemm::execute_blas_grouped_request(context, request)
1910 }
1911 #[cfg(not(feature = "cpu-blas"))]
1912 {
1913 let _ = (context, request);
1914 Ok(CpuProviderOutcome::Unsupported(
1915 CpuProviderUnsupported::RuntimeUnavailable,
1916 ))
1917 }
1918 }
1919}
1920
1921/// Built-in strided layout materialization provider.
1922///
1923/// # Examples
1924///
1925/// ```
1926/// use tenferro_cpu::provider::{CpuLayoutTransformProvider, StridedLayoutTransformProvider};
1927/// let provider: &dyn CpuLayoutTransformProvider = &StridedLayoutTransformProvider;
1928/// let _ = provider;
1929/// ```
1930#[derive(Clone, Copy, Debug, Default)]
1931pub struct StridedLayoutTransformProvider;
1932
1933impl CpuLayoutTransformProvider for StridedLayoutTransformProvider {
1934 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
1935 engine_worker_capabilities()
1936 }
1937
1938 fn materialize(
1939 &self,
1940 context: &CpuExecutionContext<'_>,
1941 request: CpuLayoutTransformRequest<'_, '_, '_>,
1942 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1943 context.with_native_parallelism(|| materialize_strided_layout(request))
1944 }
1945
1946 fn uninit_provider(&self) -> Option<&dyn CpuUninitLayoutTransformProvider> {
1947 Some(self)
1948 }
1949}
1950
1951// SAFETY: `materialize_into_uninit` replays the layout-transform copy over the
1952// full destination (every element written via `MaybeUninit` writes from the
1953// source); zero-element destinations are trivially satisfied.
1954unsafe impl CpuUninitLayoutTransformProvider for StridedLayoutTransformProvider {
1955 unsafe fn materialize_into_uninit(
1956 &self,
1957 context: &CpuExecutionContext<'_>,
1958 input: &TensorRead<'_>,
1959 intent: CpuLayoutTransformIntent,
1960 conjugate: bool,
1961 output_bytes: &mut [MaybeUninit<u8>],
1962 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1963 context.with_native_parallelism(|| {
1964 materialize_strided_layout_into_uninit(input, intent, conjugate, output_bytes)
1965 })
1966 }
1967}
1968
1969fn materialize_strided_layout(
1970 request: CpuLayoutTransformRequest<'_, '_, '_>,
1971) -> tenferro_tensor::Result<CpuProviderOutcome> {
1972 let (input, output, _intent, conjugate) = request.into_parts();
1973 if conjugate {
1974 macro_rules! dispatch_conjugated {
1975 ($owned:ident, $view:ident) => {
1976 match (input, &mut *output) {
1977 (
1978 TensorRead::Tensor(Tensor::$owned(input)),
1979 TensorWrite::Tensor(Tensor::$owned(output)),
1980 ) => {
1981 let input = input.as_view();
1982 let mut output = output.as_view_mut();
1983 crate::structural::typed_conjugate_view_into(
1984 &input,
1985 &mut output,
1986 "cpu layout materialization",
1987 )?;
1988 return Ok(CpuProviderOutcome::Executed);
1989 }
1990 (
1991 TensorRead::View(TensorView::$view(input)),
1992 TensorWrite::Tensor(Tensor::$owned(output)),
1993 ) => {
1994 let mut output = output.as_view_mut();
1995 crate::structural::typed_conjugate_view_into(
1996 input,
1997 &mut output,
1998 "cpu layout materialization",
1999 )?;
2000 return Ok(CpuProviderOutcome::Executed);
2001 }
2002 (
2003 TensorRead::Tensor(Tensor::$owned(input)),
2004 TensorWrite::View(TensorViewMut::$view(output)),
2005 ) => {
2006 let input = input.as_view();
2007 crate::structural::typed_conjugate_view_into(
2008 &input,
2009 output,
2010 "cpu layout materialization",
2011 )?;
2012 return Ok(CpuProviderOutcome::Executed);
2013 }
2014 (
2015 TensorRead::View(TensorView::$view(input)),
2016 TensorWrite::View(TensorViewMut::$view(output)),
2017 ) => {
2018 crate::structural::typed_conjugate_view_into(
2019 input,
2020 output,
2021 "cpu layout materialization",
2022 )?;
2023 return Ok(CpuProviderOutcome::Executed);
2024 }
2025 _ => {}
2026 }
2027 };
2028 }
2029 dispatch_conjugated!(F32, F32);
2030 dispatch_conjugated!(F64, F64);
2031 dispatch_conjugated!(C32, C32);
2032 dispatch_conjugated!(C64, C64);
2033 return Ok(CpuProviderOutcome::Unsupported(
2034 CpuProviderUnsupported::DType(input.dtype()),
2035 ));
2036 }
2037 macro_rules! dispatch {
2038 ($owned:ident, $view:ident) => {
2039 match (input, &mut *output) {
2040 (
2041 TensorRead::Tensor(Tensor::$owned(input)),
2042 TensorWrite::Tensor(Tensor::$owned(output)),
2043 ) => {
2044 let input = input.as_view();
2045 let mut output = output.as_view_mut();
2046 crate::structural::typed_copy_view_into(
2047 &input,
2048 &mut output,
2049 "cpu layout materialization",
2050 )?;
2051 return Ok(CpuProviderOutcome::Executed);
2052 }
2053 (
2054 TensorRead::View(TensorView::$view(input)),
2055 TensorWrite::Tensor(Tensor::$owned(output)),
2056 ) => {
2057 let mut output = output.as_view_mut();
2058 crate::structural::typed_copy_view_into(
2059 input,
2060 &mut output,
2061 "cpu layout materialization",
2062 )?;
2063 return Ok(CpuProviderOutcome::Executed);
2064 }
2065 (
2066 TensorRead::Tensor(Tensor::$owned(input)),
2067 TensorWrite::View(TensorViewMut::$view(output)),
2068 ) => {
2069 let input = input.as_view();
2070 crate::structural::typed_copy_view_into(
2071 &input,
2072 output,
2073 "cpu layout materialization",
2074 )?;
2075 return Ok(CpuProviderOutcome::Executed);
2076 }
2077 (
2078 TensorRead::View(TensorView::$view(input)),
2079 TensorWrite::View(TensorViewMut::$view(output)),
2080 ) => {
2081 crate::structural::typed_copy_view_into(
2082 input,
2083 output,
2084 "cpu layout materialization",
2085 )?;
2086 return Ok(CpuProviderOutcome::Executed);
2087 }
2088 _ => {}
2089 }
2090 };
2091 }
2092 dispatch!(F32, F32);
2093 dispatch!(F64, F64);
2094 dispatch!(I32, I32);
2095 dispatch!(I64, I64);
2096 dispatch!(Bool, Bool);
2097 dispatch!(C32, C32);
2098 dispatch!(C64, C64);
2099 Ok(CpuProviderOutcome::Unsupported(
2100 CpuProviderUnsupported::DType(input.dtype()),
2101 ))
2102}
2103
2104/// Full-overwrite layout transform into an uninitialized compact column-major
2105/// destination. Only [`CpuLayoutTransformIntent::CanonicalColumnMajor`] is
2106/// implemented; the destination must be exactly `element_count * size_of::<T>()`
2107/// bytes of the input dtype `T`, aligned for `T`.
2108fn materialize_strided_layout_into_uninit(
2109 input: &TensorRead<'_>,
2110 intent: CpuLayoutTransformIntent,
2111 conjugate: bool,
2112 output_bytes: &mut [MaybeUninit<u8>],
2113) -> tenferro_tensor::Result<CpuProviderOutcome> {
2114 if intent != CpuLayoutTransformIntent::CanonicalColumnMajor {
2115 return Ok(CpuProviderOutcome::Unsupported(
2116 CpuProviderUnsupported::Layout(CpuOperand::Output),
2117 ));
2118 }
2119 macro_rules! dispatch {
2120 ($owned:ident, $view:ident) => {
2121 match input {
2122 TensorRead::Tensor(Tensor::$owned(input)) => {
2123 let input = input.as_view();
2124 crate::structural::typed_copy_into_uninit(
2125 &input,
2126 conjugate,
2127 output_bytes,
2128 "cpu layout materialization",
2129 )?;
2130 return Ok(CpuProviderOutcome::Executed);
2131 }
2132 TensorRead::View(TensorView::$view(input)) => {
2133 crate::structural::typed_copy_into_uninit(
2134 input,
2135 conjugate,
2136 output_bytes,
2137 "cpu layout materialization",
2138 )?;
2139 return Ok(CpuProviderOutcome::Executed);
2140 }
2141 _ => {}
2142 }
2143 };
2144 }
2145 dispatch!(F32, F32);
2146 dispatch!(F64, F64);
2147 dispatch!(C32, C32);
2148 dispatch!(C64, C64);
2149 Ok(CpuProviderOutcome::Unsupported(
2150 CpuProviderUnsupported::DType(input.dtype()),
2151 ))
2152}
2153
2154pub(crate) fn builtin_gemm_provider(kind: CpuBackendKind) -> Arc<dyn CpuGemmProvider> {
2155 match kind {
2156 CpuBackendKind::Faer => Arc::new(FaerGemmProvider),
2157 CpuBackendKind::Blas => Arc::new(BlasGemmProvider),
2158 }
2159}
2160
2161pub(crate) fn builtin_layout_provider() -> Arc<dyn CpuLayoutTransformProvider> {
2162 Arc::new(StridedLayoutTransformProvider)
2163}
2164
2165#[cfg(test)]
2166pub(crate) mod tests;