tenferro_cpu/provider.rs
1//! Object-safe CPU contraction provider contracts.
2//!
3//! Providers synchronously write into engine-owned outputs. Request
4//! constructors are crate-private because only the CPU engine may attest that
5//! tensor metadata and reachable ranges have already been validated.
6
7use core::fmt;
8use std::mem::MaybeUninit;
9use std::num::NonZeroUsize;
10use std::sync::Arc;
11
12use tenferro_tensor::backend::GroupedGemmJob;
13use tenferro_tensor::{
14 DType, DotGeneralAccumulation, Tensor, TensorRead, TensorView, TensorViewMut, TensorWrite,
15};
16
17use crate::arbiter::{with_execution_owner, ResourcePermit};
18use crate::backend::CpuBackendKind;
19use crate::buffer_pool::BufferPool;
20use crate::domain_executor::{indexed_jobs, install_scoped};
21#[cfg(feature = "cpu-blas")]
22use crate::provider_capability::builtin_blas_execution_capabilities;
23#[cfg(not(feature = "cpu-blas"))]
24use crate::provider_capability::serial_capabilities;
25use crate::provider_capability::{engine_worker_capabilities, CpuProviderExecutionCapabilities};
26use crate::resource_domain::CpuResourceDomain;
27use crate::{
28 CpuDomainExecutorError, CpuDomainId, CpuInnerParallelism, CpuPlacementGuarantee, CpuSet,
29};
30
31/// Operand named by a provider capability reason.
32///
33/// # Examples
34///
35/// ```
36/// use tenferro_cpu::provider::CpuOperand;
37/// assert_eq!(CpuOperand::Lhs, CpuOperand::Lhs);
38/// ```
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40pub enum CpuOperand {
41 /// Left input operand.
42 Lhs,
43 /// Right input operand.
44 Rhs,
45 /// Writable output operand.
46 Output,
47}
48
49/// Allocation-free reason that a provider cannot execute a validated request.
50///
51/// An unsupported outcome must be reported before the provider mutates the
52/// output.
53///
54/// # Examples
55///
56/// ```
57/// use tenferro_cpu::provider::CpuProviderUnsupported;
58/// assert_eq!(
59/// CpuProviderUnsupported::RuntimeUnavailable,
60/// CpuProviderUnsupported::RuntimeUnavailable,
61/// );
62/// ```
63#[derive(Clone, Copy, Debug, PartialEq, Eq)]
64#[non_exhaustive]
65pub enum CpuProviderUnsupported {
66 /// The provider does not implement the scalar dtype.
67 DType(DType),
68 /// The provider does not implement the input ranks.
69 Rank {
70 /// Left input rank.
71 lhs: usize,
72 /// Right input rank.
73 rhs: usize,
74 },
75 /// The provider does not implement an operand layout.
76 Layout(CpuOperand),
77 /// The provider cannot implement the requested conjugation.
78 Conjugation,
79 /// The provider cannot implement the requested alpha/beta update.
80 Accumulation,
81 /// The provider cannot implement a strided batch.
82 StridedBatch,
83 /// The provider cannot implement grouped GEMM.
84 Grouped,
85 /// The optional provider runtime is not available in this process.
86 RuntimeUnavailable,
87}
88
89/// Result of attempting a validated request through one provider slot.
90///
91/// # Examples
92///
93/// ```
94/// use tenferro_cpu::provider::{CpuProviderOutcome, CpuProviderUnsupported};
95/// let outcome = CpuProviderOutcome::Unsupported(
96/// CpuProviderUnsupported::RuntimeUnavailable,
97/// );
98/// assert!(matches!(outcome, CpuProviderOutcome::Unsupported(_)));
99/// ```
100#[derive(Clone, Copy, Debug, PartialEq, Eq)]
101#[must_use]
102pub enum CpuProviderOutcome {
103 /// The provider fully executed the request.
104 Executed,
105 /// The provider did not mutate the output and resolution may continue.
106 Unsupported(CpuProviderUnsupported),
107}
108
109/// Parallel scheduling mode selected for one CPU operation.
110///
111/// # Examples
112///
113/// ```
114/// use tenferro_cpu::provider::ParallelMode;
115/// assert_ne!(ParallelMode::Sequential, ParallelMode::Outer);
116/// assert_ne!(ParallelMode::Outer, ParallelMode::Inner);
117/// ```
118#[derive(Clone, Copy, Debug, PartialEq, Eq)]
119pub enum ParallelMode {
120 /// Neither the engine nor the provider may fan out this operation.
121 Sequential,
122 /// The engine owns outer fan-out and delegates Sequential child contexts.
123 /// Providers do not receive an Outer context.
124 Outer,
125 /// One provider kernel may use the selected executor's inner region.
126 Inner,
127}
128
129/// Borrowed execution policy for an already-entered CPU operation.
130///
131/// The context exposes immutable domain facts while keeping the resource lease,
132/// executor object, and the checked executor-entry boundary private. Providers
133/// cannot install or submit work through this value.
134///
135/// # Examples
136///
137/// Providers inspect this value inside a trait method:
138///
139/// ```
140/// use tenferro_cpu::provider::CpuExecutionContext;
141/// # fn inspect(context: &CpuExecutionContext<'_>) {
142/// assert!(context.thread_budget().get() >= 1);
143/// # }
144/// ```
145#[derive(Clone, Copy)]
146pub struct CpuExecutionContext<'a> {
147 domain: &'a CpuResourceDomain,
148 parallel_mode: ParallelMode,
149}
150
151impl fmt::Debug for CpuExecutionContext<'_> {
152 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
153 formatter
154 .debug_struct("CpuExecutionContext")
155 .field("domain_id", &self.domain_id())
156 .field("cpus", &self.cpus())
157 .field("thread_budget", &self.thread_budget())
158 .field("placement_guarantee", &self.placement_guarantee())
159 .field("parallel_mode", &self.parallel_mode())
160 .finish_non_exhaustive()
161 }
162}
163
164impl<'a> CpuExecutionContext<'a> {
165 fn entered(domain: &'a CpuResourceDomain, parallel_mode: ParallelMode) -> Self {
166 Self {
167 domain,
168 parallel_mode,
169 }
170 }
171
172 /// Return the stable identity of the selected CPU resource domain.
173 ///
174 /// # Examples
175 ///
176 /// ```
177 /// use tenferro_cpu::CpuExecutionContext;
178 /// # fn inspect(context: &CpuExecutionContext<'_>) {
179 /// let _domain_id = context.domain_id();
180 /// # }
181 /// ```
182 pub fn domain_id(&self) -> CpuDomainId {
183 self.domain.id()
184 }
185
186 /// Return the selected domain's declared logical CPU set, when present.
187 ///
188 /// # Examples
189 ///
190 /// ```
191 /// use tenferro_cpu::CpuExecutionContext;
192 /// # fn inspect(context: &CpuExecutionContext<'_>) {
193 /// if let Some(cpus) = context.cpus() {
194 /// assert!(!cpus.is_empty());
195 /// }
196 /// # }
197 /// ```
198 pub fn cpus(&self) -> Option<&CpuSet> {
199 self.domain.cpus()
200 }
201
202 /// Return the selected domain's admission contract.
203 ///
204 /// # Examples
205 ///
206 /// ```rust
207 /// use tenferro_cpu::{CpuAdmissionMode, CpuExecutionContext};
208 /// # fn inspect(context: &CpuExecutionContext<'_>) {
209 /// let _mode: CpuAdmissionMode = context.admission_mode();
210 /// # }
211 /// ```
212 pub fn admission_mode(&self) -> crate::CpuAdmissionMode {
213 self.domain.admission_mode()
214 }
215
216 /// Return the non-zero maximum participating-thread budget.
217 ///
218 /// # Examples
219 ///
220 /// ```
221 /// use tenferro_cpu::CpuExecutionContext;
222 /// # fn inspect(context: &CpuExecutionContext<'_>) {
223 /// assert!(context.thread_budget().get() >= 1);
224 /// # }
225 /// ```
226 pub fn thread_budget(&self) -> NonZeroUsize {
227 self.domain.thread_budget()
228 }
229
230 /// Return the strength of the selected domain's placement guarantee, when present.
231 ///
232 /// # Examples
233 ///
234 /// ```
235 /// use tenferro_cpu::CpuExecutionContext;
236 /// # fn inspect(context: &CpuExecutionContext<'_>) {
237 /// let _guarantee = context.placement_guarantee();
238 /// # }
239 /// ```
240 pub fn placement_guarantee(&self) -> Option<CpuPlacementGuarantee> {
241 self.domain.placement_guarantee()
242 }
243
244 /// Return the engine-selected scheduling mode for this entered provider call.
245 ///
246 /// Provider calls observe Sequential or Inner. Outer scheduling creates a
247 /// separate Sequential context inside every submitted child.
248 ///
249 /// # Examples
250 ///
251 /// ```
252 /// use tenferro_cpu::{CpuExecutionContext, ParallelMode};
253 /// # fn inspect(context: &CpuExecutionContext<'_>) {
254 /// assert!(matches!(
255 /// context.parallel_mode(),
256 /// ParallelMode::Sequential | ParallelMode::Inner
257 /// ));
258 /// # }
259 /// ```
260 pub fn parallel_mode(&self) -> ParallelMode {
261 self.parallel_mode
262 }
263
264 /// Materialize a borrowed tensor view for one scoped operation and reclaim
265 /// its temporary host buffer before returning.
266 ///
267 /// Owned tensor inputs are borrowed directly. View inputs are materialized
268 /// from `buffers`, passed to `operation`, and returned to the same pool on
269 /// both success and ordinary error. The receiver is an unforgeable proof
270 /// that the caller is already inside the selected CPU execution domain.
271 ///
272 /// # Errors
273 ///
274 /// Returns [`tenferro_tensor::Error::RuntimeState`] when the view is not
275 /// accessible from CPU host memory, propagates typed view-materialization
276 /// errors, and otherwise returns the error produced by `operation`.
277 ///
278 /// # Examples
279 ///
280 /// ```
281 /// use tenferro_cpu::CpuBackend;
282 /// use tenferro_tensor::{StridedSliceSpec, TensorRead, TensorView, TypedTensor};
283 ///
284 /// let mut backend = CpuBackend::with_threads(1)?;
285 /// let input = TypedTensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
286 /// backend.with_linalg_pool(|context, buffers| {
287 /// let view = input
288 /// .as_view()
289 /// .try_slice(&[StridedSliceSpec::reverse()])?;
290 /// context.with_materialized_tensor_read(
291 /// buffers,
292 /// "example",
293 /// TensorRead::from_view(TensorView::F64(view)),
294 /// |materialized, _| {
295 /// assert_eq!(materialized.as_slice::<f64>().unwrap(), &[2.0, 1.0]);
296 /// Ok(())
297 /// },
298 /// )
299 /// })?;
300 /// # Ok::<(), Box<dyn std::error::Error>>(())
301 /// ```
302 #[doc(hidden)]
303 pub fn with_materialized_tensor_read<R>(
304 &self,
305 buffers: &mut BufferPool,
306 op: &'static str,
307 input: TensorRead<'_>,
308 operation: impl FnOnce(&Tensor, &mut BufferPool) -> tenferro_tensor::Result<R>,
309 ) -> tenferro_tensor::Result<R> {
310 match input {
311 TensorRead::Tensor(tensor) => operation(tensor, buffers),
312 TensorRead::View(view) => {
313 let materialized = self.with_native_parallelism(|| {
314 crate::materialize_tensor_read(buffers, op, TensorRead::View(view))
315 })?;
316 let result = operation(&materialized, buffers);
317 reclaim_tensor(buffers, materialized);
318 result
319 }
320 }
321 }
322
323 /// Reshape a compact tensor while retaining the current execution proof.
324 ///
325 /// This metadata-only helper does not enter an executor or borrow another
326 /// scratch pool.
327 ///
328 /// # Errors
329 ///
330 /// Returns [`tenferro_tensor::Error::Validation`] when the input and output
331 /// shapes have different element counts or the requested layout is invalid.
332 ///
333 /// # Examples
334 ///
335 /// ```
336 /// use tenferro_cpu::CpuBackend;
337 /// use tenferro_tensor::Tensor;
338 ///
339 /// let mut backend = CpuBackend::with_threads(1)?;
340 /// let input = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
341 /// backend.with_linalg_pool(|context, _| {
342 /// let output = context.reshape_tensor(&input, &[2, 1])?;
343 /// assert_eq!(output.shape(), &[2, 1]);
344 /// Ok(())
345 /// })?;
346 /// # Ok::<(), Box<dyn std::error::Error>>(())
347 /// ```
348 #[doc(hidden)]
349 pub fn reshape_tensor(
350 &self,
351 input: &Tensor,
352 shape: &[usize],
353 ) -> tenferro_tensor::Result<Tensor> {
354 crate::structural::reshape(input, shape)
355 }
356
357 /// Return the faer policy selected by this operation context.
358 ///
359 /// This hidden public method is the owner-scoped extension contract used by
360 /// operation-family crates such as `tenferro-linalg`. Keeping the mapping
361 /// here prevents sibling crates from deriving a second CPU threading
362 /// policy.
363 ///
364 /// # Examples
365 ///
366 /// ```
367 /// use tenferro_cpu::CpuExecutionContext;
368 /// # fn inspect(context: &CpuExecutionContext<'_>) {
369 /// let _policy = context.faer_parallelism();
370 /// # }
371 /// ```
372 #[cfg(feature = "cpu-faer")]
373 #[doc(hidden)]
374 pub fn faer_parallelism(self) -> faer::Par {
375 match (
376 self.parallel_mode,
377 self.domain.executor_capabilities().inner_parallelism,
378 ) {
379 (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
380 faer::Par::rayon(self.thread_budget().get())
381 }
382 _ => faer::Par::Seq,
383 }
384 }
385
386 /// Effective native-kernel degree inside this already-entered CPU context.
387 /// Non-Rayon executors and sequential/nested policy use one thread.
388 ///
389 /// # Examples
390 /// ```
391 /// use tenferro_cpu::CpuBackend;
392 /// let mut backend = CpuBackend::with_threads(1)?;
393 /// let threads = backend.with_linalg_pool(|context, _| Ok(context.native_thread_count()))?;
394 /// assert_eq!(threads, 1);
395 /// # Ok::<(), Box<dyn std::error::Error>>(())
396 /// ```
397 #[doc(hidden)]
398 pub fn native_thread_count(&self) -> usize {
399 match (
400 self.parallel_mode,
401 self.domain.executor_capabilities().inner_parallelism,
402 ) {
403 (ParallelMode::Inner, CpuInnerParallelism::Rayon) => self.thread_budget().get(),
404 _ => 1,
405 }
406 }
407
408 pub(crate) fn strided_exec_context(&self) -> strided_kernel::ExecContext {
409 match (
410 self.parallel_mode,
411 self.domain.executor_capabilities().inner_parallelism,
412 ) {
413 (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
414 // This is only an operation-local thread limit for strided's
415 // replay policy. The Rayon pool itself is the already-entered
416 // CpuContext pool installed by `with_native_parallelism`.
417 match strided_kernel::ExecContext::max_threads(self.thread_budget().get()) {
418 Ok(context) => context,
419 // INVARIANT: CpuExecutionContext stores a NonZeroUsize
420 // thread budget, and this branch passes that positive value.
421 Err(_) => unreachable!("CpuExecutionContext has a non-zero thread budget"),
422 }
423 }
424 _ => strided_kernel::ExecContext::serial(),
425 }
426 }
427
428 pub(crate) fn with_native_parallelism<R>(&self, operation: impl FnOnce() -> R) -> R {
429 let policy = match (
430 self.parallel_mode,
431 self.domain.executor_capabilities().inner_parallelism,
432 ) {
433 (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
434 strided_kernel::ExecutionPolicy::Rayon {
435 max_threads: self.thread_budget(),
436 }
437 }
438 _ => strided_kernel::ExecutionPolicy::Sequential,
439 };
440 strided_kernel::with_execution_policy(policy, operation)
441 }
442}
443
444fn reclaim_tensor(buffers: &mut BufferPool, tensor: Tensor) {
445 match tensor {
446 Tensor::F32(tensor) => crate::backend::reclaim_typed(buffers, tensor),
447 Tensor::F64(tensor) => crate::backend::reclaim_typed(buffers, tensor),
448 Tensor::I32(tensor) => crate::backend::reclaim_typed(buffers, tensor),
449 Tensor::I64(tensor) => crate::backend::reclaim_typed(buffers, tensor),
450 Tensor::Bool(tensor) => crate::backend::reclaim_typed(buffers, tensor),
451 Tensor::C32(tensor) => crate::backend::reclaim_typed(buffers, tensor),
452 Tensor::C64(tensor) => crate::backend::reclaim_typed(buffers, tensor),
453 }
454}
455
456/// Crate-private unentered capability for one CPU operation.
457///
458/// This is the only type that owns the resource permit and may cross the
459/// selected domain executor boundary. A [`CpuExecutionContext`] is constructed
460/// only inside an installed job or an outer child job.
461#[derive(Clone, Copy)]
462pub(crate) struct CpuOperationEntry<'a> {
463 domain: &'a CpuResourceDomain,
464 permit: &'a ResourcePermit,
465}
466
467impl<'a> CpuOperationEntry<'a> {
468 pub(crate) fn new(domain: &'a CpuResourceDomain, permit: &'a ResourcePermit) -> Self {
469 Self { domain, permit }
470 }
471
472 pub(crate) fn domain_id(self) -> CpuDomainId {
473 self.domain.id()
474 }
475
476 pub(crate) fn enter<R: Send>(
477 self,
478 parallel_mode: ParallelMode,
479 operation: impl FnOnce(&CpuExecutionContext<'_>) -> R + Send,
480 ) -> Result<R, CpuDomainExecutorError> {
481 if parallel_mode == ParallelMode::Outer {
482 return Err(CpuDomainExecutorError::Scheduling {
483 message: "CPU executor install requires Sequential or Inner mode, got Outer"
484 .to_owned(),
485 });
486 }
487 let owner = self.permit.owner();
488 if crate::backend::execution_scope::is_entered(self.domain, self.permit) {
489 return Ok(with_execution_owner(owner, || {
490 let context = CpuExecutionContext::entered(self.domain, parallel_mode);
491 operation(&context)
492 }));
493 }
494 with_execution_owner(owner, || {
495 install_scoped(self.domain.executor().as_ref(), || {
496 with_execution_owner(owner, || {
497 let context = CpuExecutionContext::entered(self.domain, parallel_mode);
498 operation(&context)
499 })
500 })
501 })
502 }
503
504 pub(crate) fn enter_or_reuse<R: Send>(
505 self,
506 entered: Option<&CpuExecutionContext<'_>>,
507 parallel_mode: ParallelMode,
508 operation: impl FnOnce(&CpuExecutionContext<'_>) -> R + Send,
509 ) -> Result<R, CpuDomainExecutorError> {
510 let Some(entered) = entered else {
511 return self.enter(parallel_mode, operation);
512 };
513 if parallel_mode == ParallelMode::Outer {
514 return Err(CpuDomainExecutorError::Scheduling {
515 message: "entered CPU session requires Sequential or Inner mode, got Outer"
516 .to_owned(),
517 });
518 }
519 if entered.domain_id() != self.domain.id() {
520 return Err(CpuDomainExecutorError::Scheduling {
521 message: format!(
522 "entered CPU session domain {:?} does not match operation domain {:?}",
523 entered.domain_id(),
524 self.domain.id()
525 ),
526 });
527 }
528 let owner = self.permit.owner();
529 Ok(with_execution_owner(owner, || {
530 let context = CpuExecutionContext::entered(self.domain, parallel_mode);
531 operation(&context)
532 }))
533 }
534
535 pub(crate) fn supports_infallible_session_entry(self) -> bool {
536 self.domain.ownership() == crate::CpuDomainOwnership::Managed
537 }
538
539 pub(crate) fn enter_managed_session<R: Send>(
540 self,
541 operation: impl FnOnce(CpuExecutionContext<'a>) -> R + Send,
542 ) -> R {
543 assert!(
544 self.supports_infallible_session_entry(),
545 "managed session entry requires a Tenferro-managed CPU domain"
546 );
547 let mode = self.preferred_engine_mode();
548 self.enter(mode, |_| {
549 operation(CpuExecutionContext::entered(self.domain, mode))
550 })
551 .unwrap_or_else(|error| {
552 panic!("Tenferro-managed CPU executor violated synchronous install contract: {error}")
553 })
554 }
555
556 pub(crate) fn submit_outer(
557 self,
558 len: usize,
559 operation: impl Fn(usize, &CpuExecutionContext<'_>) -> Result<(), CpuDomainExecutorError> + Sync,
560 ) -> Result<(), CpuDomainExecutorError> {
561 if !self.supports_outer() {
562 return Err(CpuDomainExecutorError::Scheduling {
563 message: format!(
564 "CPU domain {:?} does not support Outer mode",
565 self.domain.id()
566 ),
567 });
568 }
569 let owner = self.permit.owner();
570 let lane_count = len.min(self.domain.thread_budget().get());
571 let jobs = indexed_jobs(lane_count, |lane| {
572 // INVARIANT: valid lanes partition `0..len` by residue modulo the
573 // nonzero `lane_count`, so every logical job runs exactly once
574 // while the executor can schedule at most the domain budget.
575 let mut index = lane;
576 while index < len {
577 with_execution_owner(owner, || {
578 let context =
579 CpuExecutionContext::entered(self.domain, ParallelMode::Sequential);
580 operation(index, &context)
581 })?;
582 let Some(next) = index.checked_add(lane_count) else {
583 break;
584 };
585 index = next;
586 }
587 Ok(())
588 });
589 with_execution_owner(owner, || self.domain.executor().submit(&jobs))?;
590 if let Some(index) = jobs.invalid_index_attempt() {
591 return Err(CpuDomainExecutorError::Scheduling {
592 message: format!(
593 "executor requested scoped CPU lane index {index}, but the submission has {lane_count} lanes for {len} logical jobs"
594 ),
595 });
596 }
597 Ok(())
598 }
599
600 pub(crate) fn preferred_engine_mode(self) -> ParallelMode {
601 if self.domain.thread_budget().get() > 1
602 && self.domain.executor_capabilities().inner_parallelism == CpuInnerParallelism::Rayon
603 {
604 ParallelMode::Inner
605 } else {
606 ParallelMode::Sequential
607 }
608 }
609
610 pub(crate) fn preferred_provider_mode(
611 self,
612 accepts: impl Fn(ParallelMode) -> bool,
613 ) -> Result<ParallelMode, crate::CpuProviderDomainError> {
614 if self.domain.thread_budget().get() == 1 {
615 return if accepts(ParallelMode::Sequential) {
616 Ok(ParallelMode::Sequential)
617 } else {
618 Err(crate::CpuProviderDomainError::ParallelModeNotSupported {
619 mode: ParallelMode::Sequential,
620 })
621 };
622 }
623 if accepts(ParallelMode::Inner) {
624 return Ok(ParallelMode::Inner);
625 }
626 if accepts(ParallelMode::Sequential) {
627 return Ok(ParallelMode::Sequential);
628 }
629 Err(crate::CpuProviderDomainError::ParallelModeNotSupported {
630 mode: ParallelMode::Inner,
631 })
632 }
633
634 pub(crate) fn preferred_linalg_mode(self, kind: CpuBackendKind) -> ParallelMode {
635 if self.domain.thread_budget().get() == 1 {
636 return ParallelMode::Sequential;
637 }
638 match kind {
639 CpuBackendKind::Faer => self.preferred_engine_mode(),
640 // The linalg operation-family provider has not yet moved onto the
641 // provider capability traits. Preserve its existing ownership
642 // policy until that trait boundary is introduced.
643 CpuBackendKind::Blas => ParallelMode::Inner,
644 }
645 }
646
647 pub(crate) fn provider_default_compatibility_mode(self) -> ParallelMode {
648 if self.domain.thread_budget().get() == 1 {
649 ParallelMode::Sequential
650 } else {
651 ParallelMode::Inner
652 }
653 }
654
655 pub(crate) fn thread_budget(self) -> NonZeroUsize {
656 self.domain.thread_budget()
657 }
658
659 pub(crate) fn supports_outer(self) -> bool {
660 self.domain.thread_budget().get() > 1
661 && self.domain.executor_capabilities().outer_parallelism
662 }
663}
664
665/// Checked element offset and strides for a batched matrix operand.
666///
667/// # Examples
668///
669/// Providers receive this descriptor from a validated request:
670///
671/// ```
672/// use tenferro_cpu::provider::CpuGemmRequest;
673/// # fn inspect(request: &CpuGemmRequest<'_, '_, '_>) {
674/// assert!(request.lhs_layout().row_stride() != 0 || request.rows() <= 1);
675/// # }
676/// ```
677#[derive(Clone, Copy, Debug, PartialEq, Eq)]
678pub struct CpuBatchedMatrixLayout {
679 offset: isize,
680 row_stride: isize,
681 column_stride: isize,
682 batch_stride: isize,
683}
684
685impl CpuBatchedMatrixLayout {
686 #[allow(dead_code)]
687 pub(crate) fn new(
688 offset: isize,
689 row_stride: isize,
690 column_stride: isize,
691 batch_stride: isize,
692 ) -> Self {
693 Self {
694 offset,
695 row_stride,
696 column_stride,
697 batch_stride,
698 }
699 }
700
701 /// Return the checked base element offset.
702 pub fn offset(self) -> isize {
703 self.offset
704 }
705
706 /// Return the row element stride.
707 pub fn row_stride(self) -> isize {
708 self.row_stride
709 }
710
711 /// Return the column element stride.
712 pub fn column_stride(self) -> isize {
713 self.column_stride
714 }
715
716 /// Return the batch element stride.
717 pub fn batch_stride(self) -> isize {
718 self.batch_stride
719 }
720}
721
722/// Validated borrowed GEMM request.
723///
724/// A batch count of one is a single GEMM. A larger batch count is a strided
725/// batched GEMM.
726///
727/// # Examples
728///
729/// ```
730/// use tenferro_cpu::provider::CpuGemmRequest;
731/// # fn inspect(request: &CpuGemmRequest<'_, '_, '_>) {
732/// assert!(request.batch_count() >= 1);
733/// # }
734/// ```
735#[derive(Debug)]
736pub struct CpuGemmRequest<'request, 'input, 'output> {
737 lhs: &'request TensorRead<'input>,
738 rhs: &'request TensorRead<'input>,
739 output: &'request mut TensorWrite<'output>,
740 rows: usize,
741 columns: usize,
742 contracted: usize,
743 batch_count: usize,
744 lhs_layout: CpuBatchedMatrixLayout,
745 rhs_layout: CpuBatchedMatrixLayout,
746 output_layout: CpuBatchedMatrixLayout,
747 accumulation: DotGeneralAccumulation,
748}
749
750pub(crate) struct CpuGemmRequestParts<'request, 'input, 'output> {
751 pub(crate) lhs: &'request TensorRead<'input>,
752 pub(crate) rhs: &'request TensorRead<'input>,
753 pub(crate) output: &'request mut TensorWrite<'output>,
754 pub(crate) rows: usize,
755 pub(crate) columns: usize,
756 pub(crate) contracted: usize,
757 pub(crate) batch_count: usize,
758 pub(crate) lhs_layout: CpuBatchedMatrixLayout,
759 pub(crate) rhs_layout: CpuBatchedMatrixLayout,
760 pub(crate) output_layout: CpuBatchedMatrixLayout,
761 pub(crate) accumulation: DotGeneralAccumulation,
762}
763
764impl<'request, 'input, 'output> CpuGemmRequest<'request, 'input, 'output> {
765 #[allow(clippy::too_many_arguments, dead_code)]
766 pub(crate) fn new(
767 lhs: &'request TensorRead<'input>,
768 rhs: &'request TensorRead<'input>,
769 output: &'request mut TensorWrite<'output>,
770 rows: usize,
771 columns: usize,
772 contracted: usize,
773 batch_count: usize,
774 lhs_layout: CpuBatchedMatrixLayout,
775 rhs_layout: CpuBatchedMatrixLayout,
776 output_layout: CpuBatchedMatrixLayout,
777 accumulation: DotGeneralAccumulation,
778 ) -> Self {
779 Self {
780 lhs,
781 rhs,
782 output,
783 rows,
784 columns,
785 contracted,
786 batch_count,
787 lhs_layout,
788 rhs_layout,
789 output_layout,
790 accumulation,
791 }
792 }
793
794 /// Return the borrowed left input.
795 ///
796 /// # Examples
797 ///
798 /// ```
799 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
800 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
801 /// # let _ = request.lhs();
802 /// # }
803 /// ```
804 pub fn lhs(&self) -> &TensorRead<'input> {
805 self.lhs
806 }
807
808 /// Return the borrowed right input.
809 ///
810 /// # Examples
811 ///
812 /// ```
813 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
814 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
815 /// # let _ = request.rhs();
816 /// # }
817 /// ```
818 pub fn rhs(&self) -> &TensorRead<'input> {
819 self.rhs
820 }
821
822 /// Reborrow the writable output for the duration of the current call.
823 pub fn output(&mut self) -> &mut TensorWrite<'output> {
824 self.output
825 }
826
827 /// Return the number of output rows.
828 ///
829 /// # Examples
830 ///
831 /// ```
832 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
833 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
834 /// # let _ = request.rows();
835 /// # }
836 /// ```
837 pub fn rows(&self) -> usize {
838 self.rows
839 }
840
841 /// Return the number of output columns.
842 ///
843 /// # Examples
844 ///
845 /// ```
846 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
847 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
848 /// # let _ = request.columns();
849 /// # }
850 /// ```
851 pub fn columns(&self) -> usize {
852 self.columns
853 }
854
855 /// Return the contracted dimension.
856 pub fn contracted(&self) -> usize {
857 self.contracted
858 }
859
860 /// Return the number of matrices in this strided batch.
861 pub fn batch_count(&self) -> usize {
862 self.batch_count
863 }
864
865 /// Return the left input matrix layout.
866 pub fn lhs_layout(&self) -> CpuBatchedMatrixLayout {
867 self.lhs_layout
868 }
869
870 /// Return the right input matrix layout.
871 pub fn rhs_layout(&self) -> CpuBatchedMatrixLayout {
872 self.rhs_layout
873 }
874
875 /// Return the output matrix layout.
876 pub fn output_layout(&self) -> CpuBatchedMatrixLayout {
877 self.output_layout
878 }
879
880 /// Return conjugation and alpha/beta update semantics.
881 pub fn accumulation(&self) -> DotGeneralAccumulation {
882 self.accumulation
883 }
884
885 pub(crate) fn into_parts(self) -> CpuGemmRequestParts<'request, 'input, 'output> {
886 CpuGemmRequestParts {
887 lhs: self.lhs,
888 rhs: self.rhs,
889 output: self.output,
890 rows: self.rows,
891 columns: self.columns,
892 contracted: self.contracted,
893 batch_count: self.batch_count,
894 lhs_layout: self.lhs_layout,
895 rhs_layout: self.rhs_layout,
896 output_layout: self.output_layout,
897 accumulation: self.accumulation,
898 }
899 }
900}
901
902/// Validated output-free borrowed GEMM request for full-overwrite destinations.
903///
904/// Mirrors [`CpuGemmRequest`] minus the writable output: the destination is
905/// provided separately as `&mut [MaybeUninit<u8>]` to [`CpuUninitGemmProvider`]
906/// callers. This request is only used for `beta == 0` (full-overwrite)
907/// accumulations, where the destination is never read.
908///
909/// # Examples
910///
911/// ```
912/// use tenferro_cpu::provider::CpuGemmUninitRequest;
913/// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
914/// assert!(request.batch_count() >= 1);
915/// # }
916/// ```
917#[derive(Debug)]
918/// Output-free prepared GEMM request for uninitialized full-overwrite
919/// execution (see `CpuUninitGemmProvider::gemm_into_uninit`).
920///
921/// # Examples
922///
923/// ```
924/// use tenferro_cpu::provider::CpuGemmUninitRequest;
925/// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
926/// # let _ = request.rows();
927/// # let _ = request.columns();
928/// # let _ = request.contracted();
929/// # }
930/// ```
931pub struct CpuGemmUninitRequest<'request, 'input> {
932 lhs: &'request TensorRead<'input>,
933 rhs: &'request TensorRead<'input>,
934 rows: usize,
935 columns: usize,
936 contracted: usize,
937 batch_count: usize,
938 lhs_layout: CpuBatchedMatrixLayout,
939 rhs_layout: CpuBatchedMatrixLayout,
940 output_layout: CpuBatchedMatrixLayout,
941 accumulation: DotGeneralAccumulation,
942}
943
944#[cfg(feature = "cpu-faer")]
945pub(crate) struct CpuGemmUninitRequestParts<'request, 'input> {
946 pub(crate) lhs: &'request TensorRead<'input>,
947 pub(crate) rhs: &'request TensorRead<'input>,
948 pub(crate) rows: usize,
949 pub(crate) columns: usize,
950 pub(crate) contracted: usize,
951 pub(crate) batch_count: usize,
952 pub(crate) lhs_layout: CpuBatchedMatrixLayout,
953 pub(crate) rhs_layout: CpuBatchedMatrixLayout,
954 pub(crate) output_layout: CpuBatchedMatrixLayout,
955 pub(crate) accumulation: DotGeneralAccumulation,
956}
957
958impl<'request, 'input> CpuGemmUninitRequest<'request, 'input> {
959 #[allow(clippy::too_many_arguments, dead_code)]
960 pub(crate) fn new(
961 lhs: &'request TensorRead<'input>,
962 rhs: &'request TensorRead<'input>,
963 rows: usize,
964 columns: usize,
965 contracted: usize,
966 batch_count: usize,
967 lhs_layout: CpuBatchedMatrixLayout,
968 rhs_layout: CpuBatchedMatrixLayout,
969 output_layout: CpuBatchedMatrixLayout,
970 accumulation: DotGeneralAccumulation,
971 ) -> Self {
972 Self {
973 lhs,
974 rhs,
975 rows,
976 columns,
977 contracted,
978 batch_count,
979 lhs_layout,
980 rhs_layout,
981 output_layout,
982 accumulation,
983 }
984 }
985
986 /// Return the `lhs` of this output-free request.
987 ///
988 /// # Examples
989 ///
990 /// ```
991 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
992 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
993 /// # let _ = request.lhs();
994 /// # }
995 /// ```
996 pub fn lhs(&self) -> &TensorRead<'input> {
997 self.lhs
998 }
999
1000 /// Return the `rhs` of this output-free request.
1001 ///
1002 /// # Examples
1003 ///
1004 /// ```
1005 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1006 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1007 /// # let _ = request.rhs();
1008 /// # }
1009 /// ```
1010 pub fn rhs(&self) -> &TensorRead<'input> {
1011 self.rhs
1012 }
1013
1014 /// Return the `rows` of this output-free request.
1015 ///
1016 /// # Examples
1017 ///
1018 /// ```
1019 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1020 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1021 /// # let _ = request.rows();
1022 /// # }
1023 /// ```
1024 pub fn rows(&self) -> usize {
1025 self.rows
1026 }
1027
1028 /// Return the `columns` of this output-free request.
1029 ///
1030 /// # Examples
1031 ///
1032 /// ```
1033 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1034 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1035 /// # let _ = request.columns();
1036 /// # }
1037 /// ```
1038 pub fn columns(&self) -> usize {
1039 self.columns
1040 }
1041
1042 /// Return the `contracted` of this output-free request.
1043 ///
1044 /// # Examples
1045 ///
1046 /// ```
1047 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1048 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1049 /// # let _ = request.contracted();
1050 /// # }
1051 /// ```
1052 pub fn contracted(&self) -> usize {
1053 self.contracted
1054 }
1055
1056 /// Return the `batch_count` of this output-free request.
1057 ///
1058 /// # Examples
1059 ///
1060 /// ```
1061 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1062 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1063 /// # let _ = request.batch_count();
1064 /// # }
1065 /// ```
1066 pub fn batch_count(&self) -> usize {
1067 self.batch_count
1068 }
1069
1070 /// Return the `lhs_layout` of this output-free request.
1071 ///
1072 /// # Examples
1073 ///
1074 /// ```
1075 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1076 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1077 /// # let _ = request.lhs_layout();
1078 /// # }
1079 /// ```
1080 pub fn lhs_layout(&self) -> CpuBatchedMatrixLayout {
1081 self.lhs_layout
1082 }
1083
1084 /// Return the `rhs_layout` of this output-free request.
1085 ///
1086 /// # Examples
1087 ///
1088 /// ```
1089 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1090 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1091 /// # let _ = request.rhs_layout();
1092 /// # }
1093 /// ```
1094 pub fn rhs_layout(&self) -> CpuBatchedMatrixLayout {
1095 self.rhs_layout
1096 }
1097
1098 /// Return the `output_layout` of this output-free request.
1099 ///
1100 /// # Examples
1101 ///
1102 /// ```
1103 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1104 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1105 /// # let _ = request.output_layout();
1106 /// # }
1107 /// ```
1108 pub fn output_layout(&self) -> CpuBatchedMatrixLayout {
1109 self.output_layout
1110 }
1111
1112 /// Return the `accumulation` of this output-free request.
1113 ///
1114 /// # Examples
1115 ///
1116 /// ```
1117 /// use tenferro_cpu::provider::CpuGemmUninitRequest;
1118 /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
1119 /// # let _ = request.accumulation();
1120 /// # }
1121 /// ```
1122 pub fn accumulation(&self) -> DotGeneralAccumulation {
1123 self.accumulation
1124 }
1125
1126 #[cfg(feature = "cpu-faer")]
1127 pub(crate) fn into_parts(self) -> CpuGemmUninitRequestParts<'request, 'input> {
1128 CpuGemmUninitRequestParts {
1129 lhs: self.lhs,
1130 rhs: self.rhs,
1131 rows: self.rows,
1132 columns: self.columns,
1133 contracted: self.contracted,
1134 batch_count: self.batch_count,
1135 lhs_layout: self.lhs_layout,
1136 rhs_layout: self.rhs_layout,
1137 output_layout: self.output_layout,
1138 accumulation: self.accumulation,
1139 }
1140 }
1141}
1142
1143/// Validated borrowed grouped-GEMM request.
1144///
1145/// # Examples
1146///
1147/// ```
1148/// use tenferro_cpu::provider::CpuGroupedGemmRequest;
1149/// # fn inspect(request: &CpuGroupedGemmRequest<'_, '_, '_>) {
1150/// assert_eq!(request.jobs().len(), request.jobs().iter().count());
1151/// # }
1152/// ```
1153#[derive(Debug)]
1154pub struct CpuGroupedGemmRequest<'request, 'input, 'output> {
1155 lhs: &'request TensorRead<'input>,
1156 rhs: &'request TensorRead<'input>,
1157 output: &'request mut TensorWrite<'output>,
1158 jobs: &'request [GroupedGemmJob],
1159 accumulation: DotGeneralAccumulation,
1160}
1161
1162impl<'request, 'input, 'output> CpuGroupedGemmRequest<'request, 'input, 'output> {
1163 #[allow(dead_code)]
1164 pub(crate) fn new(
1165 lhs: &'request TensorRead<'input>,
1166 rhs: &'request TensorRead<'input>,
1167 output: &'request mut TensorWrite<'output>,
1168 jobs: &'request [GroupedGemmJob],
1169 accumulation: DotGeneralAccumulation,
1170 ) -> Self {
1171 Self {
1172 lhs,
1173 rhs,
1174 output,
1175 jobs,
1176 accumulation,
1177 }
1178 }
1179
1180 /// Return the borrowed left input.
1181 pub fn lhs(&self) -> &TensorRead<'input> {
1182 self.lhs
1183 }
1184
1185 /// Return the borrowed right input.
1186 pub fn rhs(&self) -> &TensorRead<'input> {
1187 self.rhs
1188 }
1189
1190 /// Reborrow the writable output.
1191 pub fn output(&mut self) -> &mut TensorWrite<'output> {
1192 self.output
1193 }
1194
1195 /// Return ordered, pairwise-disjoint validated jobs.
1196 pub fn jobs(&self) -> &[GroupedGemmJob] {
1197 self.jobs
1198 }
1199
1200 /// Return shared conjugation and alpha/beta semantics.
1201 pub fn accumulation(&self) -> DotGeneralAccumulation {
1202 self.accumulation
1203 }
1204
1205 pub(crate) fn into_parts(
1206 self,
1207 ) -> (
1208 &'request TensorRead<'input>,
1209 &'request TensorRead<'input>,
1210 &'request mut TensorWrite<'output>,
1211 &'request [GroupedGemmJob],
1212 DotGeneralAccumulation,
1213 ) {
1214 (
1215 self.lhs,
1216 self.rhs,
1217 self.output,
1218 self.jobs,
1219 self.accumulation,
1220 )
1221 }
1222}
1223
1224/// Engine-requested layout materialization.
1225///
1226/// # Examples
1227///
1228/// ```
1229/// use tenferro_cpu::provider::CpuLayoutTransformIntent;
1230/// assert_eq!(
1231/// CpuLayoutTransformIntent::CanonicalColumnMajor,
1232/// CpuLayoutTransformIntent::CanonicalColumnMajor,
1233/// );
1234/// ```
1235#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1236pub enum CpuLayoutTransformIntent {
1237 /// Materialize a compact canonical column-major tensor.
1238 CanonicalColumnMajor,
1239}
1240
1241/// Validated borrowed layout-transform request.
1242///
1243/// # Examples
1244///
1245/// ```
1246/// use tenferro_cpu::provider::{CpuLayoutTransformIntent, CpuLayoutTransformRequest};
1247/// # fn inspect(request: &CpuLayoutTransformRequest<'_, '_, '_>) {
1248/// assert_eq!(request.intent(), CpuLayoutTransformIntent::CanonicalColumnMajor);
1249/// # }
1250/// ```
1251#[derive(Debug)]
1252pub struct CpuLayoutTransformRequest<'request, 'input, 'output> {
1253 input: &'request TensorRead<'input>,
1254 output: &'request mut TensorWrite<'output>,
1255 intent: CpuLayoutTransformIntent,
1256 conjugate: bool,
1257}
1258
1259impl<'request, 'input, 'output> CpuLayoutTransformRequest<'request, 'input, 'output> {
1260 #[allow(dead_code)]
1261 pub(crate) fn new(
1262 input: &'request TensorRead<'input>,
1263 output: &'request mut TensorWrite<'output>,
1264 intent: CpuLayoutTransformIntent,
1265 conjugate: bool,
1266 ) -> Self {
1267 Self {
1268 input,
1269 output,
1270 intent,
1271 conjugate,
1272 }
1273 }
1274
1275 /// Return the borrowed input.
1276 pub fn input(&self) -> &TensorRead<'input> {
1277 self.input
1278 }
1279
1280 /// Reborrow the writable output.
1281 pub fn output(&mut self) -> &mut TensorWrite<'output> {
1282 self.output
1283 }
1284
1285 /// Return the requested materialization intent.
1286 pub fn intent(&self) -> CpuLayoutTransformIntent {
1287 self.intent
1288 }
1289
1290 /// Return whether materialization must conjugate each input element.
1291 ///
1292 /// # Examples
1293 ///
1294 /// ```
1295 /// use tenferro_cpu::provider::CpuLayoutTransformRequest;
1296 /// # fn inspect(request: &CpuLayoutTransformRequest<'_, '_, '_>) {
1297 /// let _must_conjugate = request.conjugate();
1298 /// # }
1299 /// ```
1300 pub fn conjugate(&self) -> bool {
1301 self.conjugate
1302 }
1303
1304 pub(crate) fn into_parts(
1305 self,
1306 ) -> (
1307 &'request TensorRead<'input>,
1308 &'request mut TensorWrite<'output>,
1309 CpuLayoutTransformIntent,
1310 bool,
1311 ) {
1312 (self.input, self.output, self.intent, self.conjugate)
1313 }
1314}
1315
1316/// Validated ordered contraction-role groups.
1317///
1318/// # Examples
1319///
1320/// ```
1321/// use tenferro_cpu::provider::CpuDotGeneralRequest;
1322/// # fn inspect(request: &CpuDotGeneralRequest<'_, '_, '_>) {
1323/// let _pairs = request.axes().contracting_pairs().count();
1324/// # }
1325/// ```
1326#[derive(Clone, Copy, Debug)]
1327pub struct CpuContractionAxes<'a> {
1328 lhs_rank: usize,
1329 rhs_rank: usize,
1330 lhs_contracting: &'a [usize],
1331 rhs_contracting: &'a [usize],
1332 lhs_batch: &'a [usize],
1333 rhs_batch: &'a [usize],
1334 lhs_role_mask: Option<u64>,
1335 rhs_role_mask: Option<u64>,
1336}
1337
1338impl<'a> CpuContractionAxes<'a> {
1339 #[allow(clippy::too_many_arguments, dead_code)]
1340 pub(crate) fn new(
1341 lhs_rank: usize,
1342 rhs_rank: usize,
1343 lhs_contracting: &'a [usize],
1344 rhs_contracting: &'a [usize],
1345 lhs_batch: &'a [usize],
1346 rhs_batch: &'a [usize],
1347 lhs_role_mask: Option<u64>,
1348 rhs_role_mask: Option<u64>,
1349 ) -> Self {
1350 Self {
1351 lhs_rank,
1352 rhs_rank,
1353 lhs_contracting,
1354 rhs_contracting,
1355 lhs_batch,
1356 rhs_batch,
1357 lhs_role_mask,
1358 rhs_role_mask,
1359 }
1360 }
1361
1362 /// Return ordered contracting-axis pairs.
1363 pub fn contracting_pairs(&self) -> impl ExactSizeIterator<Item = (usize, usize)> + '_ {
1364 self.lhs_contracting
1365 .iter()
1366 .copied()
1367 .zip(self.rhs_contracting.iter().copied())
1368 }
1369
1370 /// Return ordered batch-axis pairs.
1371 pub fn batch_pairs(&self) -> impl ExactSizeIterator<Item = (usize, usize)> + '_ {
1372 self.lhs_batch
1373 .iter()
1374 .copied()
1375 .zip(self.rhs_batch.iter().copied())
1376 }
1377
1378 /// Return left axes that are neither contracting nor batch axes.
1379 pub fn lhs_free_axes(&self) -> impl Iterator<Item = usize> + '_ {
1380 (0..self.lhs_rank).filter(move |&axis| !self.lhs_axis_has_role(axis))
1381 }
1382
1383 /// Return right axes that are neither contracting nor batch axes.
1384 pub fn rhs_free_axes(&self) -> impl Iterator<Item = usize> + '_ {
1385 (0..self.rhs_rank).filter(move |&axis| !self.rhs_axis_has_role(axis))
1386 }
1387
1388 fn lhs_axis_has_role(&self, axis: usize) -> bool {
1389 self.lhs_role_mask.map_or_else(
1390 || self.lhs_contracting.contains(&axis) || self.lhs_batch.contains(&axis),
1391 |mask| mask & (1_u64 << axis) != 0,
1392 )
1393 }
1394
1395 fn rhs_axis_has_role(&self, axis: usize) -> bool {
1396 self.rhs_role_mask.map_or_else(
1397 || self.rhs_contracting.contains(&axis) || self.rhs_batch.contains(&axis),
1398 |mask| mask & (1_u64 << axis) != 0,
1399 )
1400 }
1401}
1402
1403/// Validated borrowed semantic binary `dot_general` request.
1404///
1405/// # Examples
1406///
1407/// ```
1408/// use tenferro_cpu::provider::CpuDotGeneralRequest;
1409/// # fn inspect(request: &CpuDotGeneralRequest<'_, '_, '_>) {
1410/// let _ = request.accumulation();
1411/// # }
1412/// ```
1413#[derive(Debug)]
1414pub struct CpuDotGeneralRequest<'request, 'input, 'output> {
1415 lhs: &'request TensorRead<'input>,
1416 rhs: &'request TensorRead<'input>,
1417 output: &'request mut TensorWrite<'output>,
1418 axes: CpuContractionAxes<'request>,
1419 accumulation: DotGeneralAccumulation,
1420}
1421
1422impl<'request, 'input, 'output> CpuDotGeneralRequest<'request, 'input, 'output> {
1423 #[allow(dead_code)]
1424 pub(crate) fn new(
1425 lhs: &'request TensorRead<'input>,
1426 rhs: &'request TensorRead<'input>,
1427 output: &'request mut TensorWrite<'output>,
1428 axes: CpuContractionAxes<'request>,
1429 accumulation: DotGeneralAccumulation,
1430 ) -> Self {
1431 Self {
1432 lhs,
1433 rhs,
1434 output,
1435 axes,
1436 accumulation,
1437 }
1438 }
1439
1440 /// Return the borrowed left input.
1441 pub fn lhs(&self) -> &TensorRead<'input> {
1442 self.lhs
1443 }
1444
1445 /// Return the borrowed right input.
1446 pub fn rhs(&self) -> &TensorRead<'input> {
1447 self.rhs
1448 }
1449
1450 /// Reborrow the writable output.
1451 pub fn output(&mut self) -> &mut TensorWrite<'output> {
1452 self.output
1453 }
1454
1455 /// Return the validated ordered axis groups.
1456 pub fn axes(&self) -> &CpuContractionAxes<'request> {
1457 &self.axes
1458 }
1459
1460 /// Return conjugation and alpha/beta update semantics.
1461 pub fn accumulation(&self) -> DotGeneralAccumulation {
1462 self.accumulation
1463 }
1464
1465 /// Consume the request and return the validated operand borrows.
1466 ///
1467 /// External general-contraction providers use this when they need
1468 /// simultaneous immutable access to both inputs and mutable access to the
1469 /// output.
1470 ///
1471 /// # Examples
1472 ///
1473 /// ```
1474 /// use tenferro_cpu::provider::CpuDotGeneralRequest;
1475 ///
1476 /// fn consume_request(request: CpuDotGeneralRequest<'_, '_, '_>) {
1477 /// let (_lhs, _rhs, _output, _axes, _accumulation) = request.into_parts();
1478 /// }
1479 /// ```
1480 pub fn into_parts(
1481 self,
1482 ) -> (
1483 &'request TensorRead<'input>,
1484 &'request TensorRead<'input>,
1485 &'request mut TensorWrite<'output>,
1486 CpuContractionAxes<'request>,
1487 DotGeneralAccumulation,
1488 ) {
1489 (
1490 self.lhs,
1491 self.rhs,
1492 self.output,
1493 self.axes,
1494 self.accumulation,
1495 )
1496 }
1497}
1498
1499/// Provider for validated GEMM-family requests.
1500///
1501/// # Examples
1502///
1503/// Trait objects are supported directly:
1504///
1505/// ```
1506/// use tenferro_cpu::provider::CpuGemmProvider;
1507/// # fn accepts_provider(_: &dyn CpuGemmProvider) {}
1508/// ```
1509pub trait CpuGemmProvider: fmt::Debug + Send + Sync + 'static {
1510 /// Return immutable count, placement, and fan-out capabilities.
1511 ///
1512 /// This declaration must describe controls actually applied and restored
1513 /// by the provider adapter around each call. Merely discovering a runtime
1514 /// symbol is insufficient. A provider bundle samples this method exactly
1515 /// once during construction and keeps that snapshot for its lifetime; the
1516 /// returned contract must therefore remain valid for the provider object.
1517 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;
1518
1519 /// Execute one validated GEMM.
1520 ///
1521 /// # Errors
1522 ///
1523 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1524 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1525 /// fails. A detected inconsistency in engine-attested request metadata is
1526 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
1527 /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
1528 fn gemm(
1529 &self,
1530 context: &CpuExecutionContext<'_>,
1531 request: CpuGemmRequest<'_, '_, '_>,
1532 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1533
1534 /// Execute one validated strided-batched GEMM.
1535 ///
1536 /// # Errors
1537 ///
1538 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1539 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1540 /// fails. A detected inconsistency in engine-attested request metadata is
1541 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
1542 /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
1543 fn strided_batched_gemm(
1544 &self,
1545 context: &CpuExecutionContext<'_>,
1546 request: CpuGemmRequest<'_, '_, '_>,
1547 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1548
1549 /// Execute one validated grouped GEMM.
1550 ///
1551 /// # Errors
1552 ///
1553 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1554 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1555 /// fails. A detected inconsistency in engine-attested request metadata is
1556 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
1557 /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
1558 fn grouped_gemm(
1559 &self,
1560 context: &CpuExecutionContext<'_>,
1561 request: CpuGroupedGemmRequest<'_, '_, '_>,
1562 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1563
1564 /// Return the structural full-overwrite witness for this provider.
1565 ///
1566 /// Returns `Some(self)` only for types that implement
1567 /// [`CpuUninitGemmProvider`] via an `unsafe impl`, so a `Some` witness is
1568 /// structural proof that the provider asserted the full-overwrite
1569 /// destination contract. Defaults to `None` (opted out).
1570 ///
1571 /// # Examples
1572 ///
1573 /// ```
1574 /// use tenferro_cpu::provider::CpuGemmProvider;
1575 /// # fn inspect(provider: &dyn CpuGemmProvider) {
1576 /// # let _ = provider.uninit_provider();
1577 /// # }
1578 /// ```
1579 fn uninit_provider(&self) -> Option<&dyn CpuUninitGemmProvider> {
1580 None
1581 }
1582}
1583
1584/// Provider for validated GEMM-family requests that fully overwrite their
1585/// destination (only `beta == 0` accumulations).
1586///
1587/// # Safety
1588///
1589/// Implementors assert that [`CpuUninitGemmProvider::gemm_into_uninit`] writes
1590/// every logical output element before returning
1591/// [`CpuProviderOutcome::Executed`]; partial writes make the caller's
1592/// `assume_init` undefined behavior. Implementations must never read the
1593/// destination. This trait is only invoked for `beta == 0` accumulations.
1594///
1595/// # Examples
1596///
1597/// ```
1598/// use tenferro_cpu::provider::CpuUninitGemmProvider;
1599/// # fn accepts_provider(_: &dyn CpuUninitGemmProvider) {}
1600/// ```
1601pub unsafe trait CpuUninitGemmProvider: CpuGemmProvider {
1602 /// Execute one validated full-overwrite GEMM into uninitialized bytes.
1603 ///
1604 /// # Safety
1605 ///
1606 /// Must write every element of the destination represented by
1607 /// `output_bytes` (per `request.output_layout()`) before returning
1608 /// `Executed`; partial writes are UB at the caller's `assume_init`. The
1609 /// destination is never read. Invoked only for `beta == 0` accumulations.
1610 ///
1611 /// # Errors
1612 ///
1613 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1614 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1615 /// fails. A detected inconsistency in engine-attested request metadata is
1616 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
1617 /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
1618 ///
1619 /// # Examples
1620 ///
1621 /// ```
1622 /// use tenferro_cpu::provider::{CpuGemmUninitRequest, CpuProviderOutcome, CpuUninitGemmProvider};
1623 /// use tenferro_cpu::CpuExecutionContext;
1624 /// use std::mem::MaybeUninit;
1625 /// # fn inspect(
1626 /// # provider: &dyn CpuUninitGemmProvider,
1627 /// # context: &CpuExecutionContext<'_>,
1628 /// # request: CpuGemmUninitRequest<'_, '_>,
1629 /// # output_bytes: &mut [MaybeUninit<u8>],
1630 /// # ) -> Option<tenferro_tensor::Result<CpuProviderOutcome>> {
1631 /// # let _ = (provider, context, request, output_bytes);
1632 /// # None
1633 /// # }
1634 /// ```
1635 unsafe fn gemm_into_uninit(
1636 &self,
1637 context: &CpuExecutionContext<'_>,
1638 request: CpuGemmUninitRequest<'_, '_>,
1639 output_bytes: &mut [MaybeUninit<u8>],
1640 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1641}
1642
1643/// Provider for engine-owned tensor materialization.
1644///
1645/// # Examples
1646///
1647/// ```
1648/// use tenferro_cpu::provider::CpuLayoutTransformProvider;
1649/// # fn accepts_provider(_: &dyn CpuLayoutTransformProvider) {}
1650/// ```
1651pub trait CpuLayoutTransformProvider: fmt::Debug + Send + Sync + 'static {
1652 /// Return immutable count, placement, and fan-out capabilities.
1653 ///
1654 /// A provider bundle samples this method exactly once during construction
1655 /// and uses the stored descriptor for all validation and dispatch.
1656 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;
1657
1658 /// Materialize one validated input into a preallocated output.
1659 ///
1660 /// # Errors
1661 ///
1662 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1663 /// [`tenferro_tensor::Error::BackendFailure`] when execution fails. A
1664 /// detected inconsistency in engine-attested layout or range metadata is
1665 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported layouts
1666 /// use [`CpuProviderOutcome::Unsupported`] instead.
1667 fn materialize(
1668 &self,
1669 context: &CpuExecutionContext<'_>,
1670 request: CpuLayoutTransformRequest<'_, '_, '_>,
1671 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1672
1673 /// Return the structural full-overwrite witness for this provider.
1674 ///
1675 /// Returns `Some(self)` only for types that implement
1676 /// [`CpuUninitLayoutTransformProvider`] via an `unsafe impl`, so a `Some`
1677 /// witness is structural proof that the provider asserted the
1678 /// full-overwrite destination contract. Defaults to `None` (opted out).
1679 ///
1680 /// # Examples
1681 ///
1682 /// ```
1683 /// use tenferro_cpu::provider::CpuLayoutTransformProvider;
1684 /// # fn inspect(provider: &dyn CpuLayoutTransformProvider) {
1685 /// # let _ = provider.uninit_provider();
1686 /// # }
1687 /// ```
1688 fn uninit_provider(&self) -> Option<&dyn CpuUninitLayoutTransformProvider> {
1689 None
1690 }
1691}
1692
1693/// Provider for engine-owned tensor materialization into uninitialized bytes.
1694///
1695/// # Safety
1696///
1697/// Implementors assert that [`CpuUninitLayoutTransformProvider::materialize_into_uninit`]
1698/// writes every element of `output_bytes` before returning
1699/// [`CpuProviderOutcome::Executed`]; partial writes make the caller's
1700/// `assume_init` undefined behavior. Implementations must never read the
1701/// destination.
1702///
1703/// # Examples
1704///
1705/// ```
1706/// use tenferro_cpu::provider::CpuUninitLayoutTransformProvider;
1707/// # fn accepts_provider(_: &dyn CpuUninitLayoutTransformProvider) {}
1708/// ```
1709pub unsafe trait CpuUninitLayoutTransformProvider: CpuLayoutTransformProvider {
1710 /// Materialize one validated input into uninitialized bytes.
1711 ///
1712 /// # Safety
1713 ///
1714 /// Must write every element of `output_bytes` (a compact column-major
1715 /// destination of the input shape for
1716 /// [`CpuLayoutTransformIntent::CanonicalColumnMajor`]) before returning
1717 /// `Executed`; partial writes are UB at the caller's `assume_init`. The
1718 /// destination is never read.
1719 ///
1720 /// # Errors
1721 ///
1722 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1723 /// [`tenferro_tensor::Error::BackendFailure`] when execution fails. A
1724 /// detected inconsistency in engine-attested layout or range metadata is
1725 /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported layouts
1726 /// use [`CpuProviderOutcome::Unsupported`] instead.
1727 ///
1728 /// # Examples
1729 ///
1730 /// ```
1731 /// use tenferro_cpu::provider::{CpuLayoutTransformIntent, CpuProviderOutcome, CpuUninitLayoutTransformProvider};
1732 /// use tenferro_cpu::CpuExecutionContext;
1733 /// use tenferro_tensor::TensorRead;
1734 /// use std::mem::MaybeUninit;
1735 /// # fn inspect(
1736 /// # provider: &dyn CpuUninitLayoutTransformProvider,
1737 /// # context: &CpuExecutionContext<'_>,
1738 /// # input: &TensorRead<'_>,
1739 /// # intent: CpuLayoutTransformIntent,
1740 /// # output_bytes: &mut [MaybeUninit<u8>],
1741 /// # ) -> Option<tenferro_tensor::Result<CpuProviderOutcome>> {
1742 /// # let _ = (provider, context, input, intent, output_bytes);
1743 /// # None
1744 /// # }
1745 /// ```
1746 unsafe fn materialize_into_uninit(
1747 &self,
1748 context: &CpuExecutionContext<'_>,
1749 input: &TensorRead<'_>,
1750 intent: CpuLayoutTransformIntent,
1751 conjugate: bool,
1752 output_bytes: &mut [MaybeUninit<u8>],
1753 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1754}
1755
1756/// Provider for complete validated binary `dot_general` requests.
1757///
1758/// # Examples
1759///
1760/// ```
1761/// use tenferro_cpu::provider::CpuGeneralContractionProvider;
1762/// # fn accepts_provider(_: &dyn CpuGeneralContractionProvider) {}
1763/// ```
1764pub trait CpuGeneralContractionProvider: fmt::Debug + Send + Sync + 'static {
1765 /// Return immutable count, placement, and fan-out capabilities.
1766 ///
1767 /// A provider bundle samples this method exactly once during construction
1768 /// and uses the stored descriptor for all validation and dispatch.
1769 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;
1770
1771 /// Execute one complete semantic contraction.
1772 ///
1773 /// # Errors
1774 ///
1775 /// Returns [`tenferro_tensor::Error::BackendSource`] or
1776 /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
1777 /// fails. A detected inconsistency in engine-attested axes or range
1778 /// metadata is returned as [`tenferro_tensor::Error::Validation`].
1779 /// Unsupported contractions use [`CpuProviderOutcome::Unsupported`]
1780 /// instead.
1781 fn dot_general(
1782 &self,
1783 context: &CpuExecutionContext<'_>,
1784 request: CpuDotGeneralRequest<'_, '_, '_>,
1785 ) -> tenferro_tensor::Result<CpuProviderOutcome>;
1786}
1787
1788/// Built-in faer GEMM provider.
1789///
1790/// # Examples
1791///
1792/// ```
1793/// use tenferro_cpu::provider::{CpuGemmProvider, FaerGemmProvider};
1794/// let provider: &dyn CpuGemmProvider = &FaerGemmProvider;
1795/// let _ = provider;
1796/// ```
1797#[derive(Clone, Copy, Debug, Default)]
1798pub struct FaerGemmProvider;
1799
1800impl CpuGemmProvider for FaerGemmProvider {
1801 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
1802 engine_worker_capabilities()
1803 }
1804
1805 fn gemm(
1806 &self,
1807 context: &CpuExecutionContext<'_>,
1808 request: CpuGemmRequest<'_, '_, '_>,
1809 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1810 #[cfg(feature = "cpu-faer")]
1811 {
1812 crate::gemm::execute_faer_gemm_request(context, request)
1813 }
1814 #[cfg(not(feature = "cpu-faer"))]
1815 {
1816 let _ = (context, request);
1817 Ok(CpuProviderOutcome::Unsupported(
1818 CpuProviderUnsupported::RuntimeUnavailable,
1819 ))
1820 }
1821 }
1822
1823 fn strided_batched_gemm(
1824 &self,
1825 context: &CpuExecutionContext<'_>,
1826 request: CpuGemmRequest<'_, '_, '_>,
1827 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1828 self.gemm(context, request)
1829 }
1830
1831 fn grouped_gemm(
1832 &self,
1833 context: &CpuExecutionContext<'_>,
1834 request: CpuGroupedGemmRequest<'_, '_, '_>,
1835 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1836 #[cfg(feature = "cpu-faer")]
1837 {
1838 crate::gemm::execute_faer_grouped_request(context, request)
1839 }
1840 #[cfg(not(feature = "cpu-faer"))]
1841 {
1842 let _ = (context, request);
1843 Ok(CpuProviderOutcome::Unsupported(
1844 CpuProviderUnsupported::RuntimeUnavailable,
1845 ))
1846 }
1847 }
1848
1849 fn uninit_provider(&self) -> Option<&dyn CpuUninitGemmProvider> {
1850 Some(self)
1851 }
1852}
1853
1854// SAFETY: `gemm_into_uninit` runs the faer GEMM with `Accum::Replace` semantics
1855// (beta == 0 never reads the destination) and writes zeros for empty
1856// contractions, so every logical output element is written before `Executed`.
1857// See `crate::gemm::execute_faer_gemm_request_into_uninit`.
1858unsafe impl CpuUninitGemmProvider for FaerGemmProvider {
1859 unsafe fn gemm_into_uninit(
1860 &self,
1861 context: &CpuExecutionContext<'_>,
1862 request: CpuGemmUninitRequest<'_, '_>,
1863 output_bytes: &mut [MaybeUninit<u8>],
1864 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1865 #[cfg(feature = "cpu-faer")]
1866 {
1867 crate::gemm::execute_faer_gemm_request_into_uninit(context, request, output_bytes)
1868 }
1869 #[cfg(not(feature = "cpu-faer"))]
1870 {
1871 let _ = (context, request, output_bytes);
1872 Ok(CpuProviderOutcome::Unsupported(
1873 CpuProviderUnsupported::RuntimeUnavailable,
1874 ))
1875 }
1876 }
1877}
1878
1879/// Built-in BLAS GEMM provider.
1880///
1881/// # Examples
1882///
1883/// ```
1884/// use tenferro_cpu::provider::{BlasGemmProvider, CpuGemmProvider};
1885/// let provider: &dyn CpuGemmProvider = &BlasGemmProvider;
1886/// let _ = provider;
1887/// ```
1888#[derive(Clone, Copy, Debug, Default)]
1889pub struct BlasGemmProvider;
1890
1891impl CpuGemmProvider for BlasGemmProvider {
1892 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
1893 #[cfg(feature = "cpu-blas")]
1894 {
1895 builtin_blas_execution_capabilities()
1896 }
1897 #[cfg(not(feature = "cpu-blas"))]
1898 {
1899 // This build returns RuntimeUnavailable without invoking BLAS.
1900 serial_capabilities()
1901 }
1902 }
1903
1904 fn gemm(
1905 &self,
1906 context: &CpuExecutionContext<'_>,
1907 request: CpuGemmRequest<'_, '_, '_>,
1908 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1909 #[cfg(feature = "cpu-blas")]
1910 {
1911 crate::gemm::execute_blas_gemm_request(context, request)
1912 }
1913 #[cfg(not(feature = "cpu-blas"))]
1914 {
1915 let _ = (context, request);
1916 Ok(CpuProviderOutcome::Unsupported(
1917 CpuProviderUnsupported::RuntimeUnavailable,
1918 ))
1919 }
1920 }
1921
1922 fn strided_batched_gemm(
1923 &self,
1924 context: &CpuExecutionContext<'_>,
1925 request: CpuGemmRequest<'_, '_, '_>,
1926 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1927 self.gemm(context, request)
1928 }
1929
1930 fn grouped_gemm(
1931 &self,
1932 context: &CpuExecutionContext<'_>,
1933 request: CpuGroupedGemmRequest<'_, '_, '_>,
1934 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1935 #[cfg(feature = "cpu-blas")]
1936 {
1937 crate::gemm::execute_blas_grouped_request(context, request)
1938 }
1939 #[cfg(not(feature = "cpu-blas"))]
1940 {
1941 let _ = (context, request);
1942 Ok(CpuProviderOutcome::Unsupported(
1943 CpuProviderUnsupported::RuntimeUnavailable,
1944 ))
1945 }
1946 }
1947}
1948
1949/// Built-in strided layout materialization provider.
1950///
1951/// # Examples
1952///
1953/// ```
1954/// use tenferro_cpu::provider::{CpuLayoutTransformProvider, StridedLayoutTransformProvider};
1955/// let provider: &dyn CpuLayoutTransformProvider = &StridedLayoutTransformProvider;
1956/// let _ = provider;
1957/// ```
1958#[derive(Clone, Copy, Debug, Default)]
1959pub struct StridedLayoutTransformProvider;
1960
1961impl CpuLayoutTransformProvider for StridedLayoutTransformProvider {
1962 fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
1963 engine_worker_capabilities()
1964 }
1965
1966 fn materialize(
1967 &self,
1968 context: &CpuExecutionContext<'_>,
1969 request: CpuLayoutTransformRequest<'_, '_, '_>,
1970 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1971 context.with_native_parallelism(|| materialize_strided_layout(request))
1972 }
1973
1974 fn uninit_provider(&self) -> Option<&dyn CpuUninitLayoutTransformProvider> {
1975 Some(self)
1976 }
1977}
1978
1979// SAFETY: `materialize_into_uninit` replays the layout-transform copy over the
1980// full destination (every element written via `MaybeUninit` writes from the
1981// source); zero-element destinations are trivially satisfied.
1982unsafe impl CpuUninitLayoutTransformProvider for StridedLayoutTransformProvider {
1983 unsafe fn materialize_into_uninit(
1984 &self,
1985 context: &CpuExecutionContext<'_>,
1986 input: &TensorRead<'_>,
1987 intent: CpuLayoutTransformIntent,
1988 conjugate: bool,
1989 output_bytes: &mut [MaybeUninit<u8>],
1990 ) -> tenferro_tensor::Result<CpuProviderOutcome> {
1991 context.with_native_parallelism(|| {
1992 materialize_strided_layout_into_uninit(input, intent, conjugate, output_bytes)
1993 })
1994 }
1995}
1996
1997fn materialize_strided_layout(
1998 request: CpuLayoutTransformRequest<'_, '_, '_>,
1999) -> tenferro_tensor::Result<CpuProviderOutcome> {
2000 let (input, output, _intent, conjugate) = request.into_parts();
2001 if conjugate {
2002 macro_rules! dispatch_conjugated {
2003 ($owned:ident, $view:ident) => {
2004 match (input, &mut *output) {
2005 (
2006 TensorRead::Tensor(Tensor::$owned(input)),
2007 TensorWrite::Tensor(Tensor::$owned(output)),
2008 ) => {
2009 let input = input.as_view();
2010 let mut output = output.as_view_mut();
2011 crate::structural::typed_conjugate_view_into(
2012 &input,
2013 &mut output,
2014 "cpu layout materialization",
2015 )?;
2016 return Ok(CpuProviderOutcome::Executed);
2017 }
2018 (
2019 TensorRead::View(TensorView::$view(input)),
2020 TensorWrite::Tensor(Tensor::$owned(output)),
2021 ) => {
2022 let mut output = output.as_view_mut();
2023 crate::structural::typed_conjugate_view_into(
2024 input,
2025 &mut output,
2026 "cpu layout materialization",
2027 )?;
2028 return Ok(CpuProviderOutcome::Executed);
2029 }
2030 (
2031 TensorRead::Tensor(Tensor::$owned(input)),
2032 TensorWrite::View(TensorViewMut::$view(output)),
2033 ) => {
2034 let input = input.as_view();
2035 crate::structural::typed_conjugate_view_into(
2036 &input,
2037 output,
2038 "cpu layout materialization",
2039 )?;
2040 return Ok(CpuProviderOutcome::Executed);
2041 }
2042 (
2043 TensorRead::View(TensorView::$view(input)),
2044 TensorWrite::View(TensorViewMut::$view(output)),
2045 ) => {
2046 crate::structural::typed_conjugate_view_into(
2047 input,
2048 output,
2049 "cpu layout materialization",
2050 )?;
2051 return Ok(CpuProviderOutcome::Executed);
2052 }
2053 _ => {}
2054 }
2055 };
2056 }
2057 dispatch_conjugated!(F32, F32);
2058 dispatch_conjugated!(F64, F64);
2059 dispatch_conjugated!(C32, C32);
2060 dispatch_conjugated!(C64, C64);
2061 return Ok(CpuProviderOutcome::Unsupported(
2062 CpuProviderUnsupported::DType(input.dtype()),
2063 ));
2064 }
2065 macro_rules! dispatch {
2066 ($owned:ident, $view:ident) => {
2067 match (input, &mut *output) {
2068 (
2069 TensorRead::Tensor(Tensor::$owned(input)),
2070 TensorWrite::Tensor(Tensor::$owned(output)),
2071 ) => {
2072 let input = input.as_view();
2073 let mut output = output.as_view_mut();
2074 crate::structural::typed_copy_view_into(
2075 &input,
2076 &mut output,
2077 "cpu layout materialization",
2078 )?;
2079 return Ok(CpuProviderOutcome::Executed);
2080 }
2081 (
2082 TensorRead::View(TensorView::$view(input)),
2083 TensorWrite::Tensor(Tensor::$owned(output)),
2084 ) => {
2085 let mut output = output.as_view_mut();
2086 crate::structural::typed_copy_view_into(
2087 input,
2088 &mut output,
2089 "cpu layout materialization",
2090 )?;
2091 return Ok(CpuProviderOutcome::Executed);
2092 }
2093 (
2094 TensorRead::Tensor(Tensor::$owned(input)),
2095 TensorWrite::View(TensorViewMut::$view(output)),
2096 ) => {
2097 let input = input.as_view();
2098 crate::structural::typed_copy_view_into(
2099 &input,
2100 output,
2101 "cpu layout materialization",
2102 )?;
2103 return Ok(CpuProviderOutcome::Executed);
2104 }
2105 (
2106 TensorRead::View(TensorView::$view(input)),
2107 TensorWrite::View(TensorViewMut::$view(output)),
2108 ) => {
2109 crate::structural::typed_copy_view_into(
2110 input,
2111 output,
2112 "cpu layout materialization",
2113 )?;
2114 return Ok(CpuProviderOutcome::Executed);
2115 }
2116 _ => {}
2117 }
2118 };
2119 }
2120 dispatch!(F32, F32);
2121 dispatch!(F64, F64);
2122 dispatch!(I32, I32);
2123 dispatch!(I64, I64);
2124 dispatch!(Bool, Bool);
2125 dispatch!(C32, C32);
2126 dispatch!(C64, C64);
2127 Ok(CpuProviderOutcome::Unsupported(
2128 CpuProviderUnsupported::DType(input.dtype()),
2129 ))
2130}
2131
2132/// Full-overwrite layout transform into an uninitialized compact column-major
2133/// destination. Only [`CpuLayoutTransformIntent::CanonicalColumnMajor`] is
2134/// implemented; the destination must be exactly `element_count * size_of::<T>()`
2135/// bytes of the input dtype `T`, aligned for `T`.
2136fn materialize_strided_layout_into_uninit(
2137 input: &TensorRead<'_>,
2138 intent: CpuLayoutTransformIntent,
2139 conjugate: bool,
2140 output_bytes: &mut [MaybeUninit<u8>],
2141) -> tenferro_tensor::Result<CpuProviderOutcome> {
2142 if intent != CpuLayoutTransformIntent::CanonicalColumnMajor {
2143 return Ok(CpuProviderOutcome::Unsupported(
2144 CpuProviderUnsupported::Layout(CpuOperand::Output),
2145 ));
2146 }
2147 macro_rules! dispatch {
2148 ($owned:ident, $view:ident) => {
2149 match input {
2150 TensorRead::Tensor(Tensor::$owned(input)) => {
2151 let input = input.as_view();
2152 crate::structural::typed_copy_into_uninit(
2153 &input,
2154 conjugate,
2155 output_bytes,
2156 "cpu layout materialization",
2157 )?;
2158 return Ok(CpuProviderOutcome::Executed);
2159 }
2160 TensorRead::View(TensorView::$view(input)) => {
2161 crate::structural::typed_copy_into_uninit(
2162 input,
2163 conjugate,
2164 output_bytes,
2165 "cpu layout materialization",
2166 )?;
2167 return Ok(CpuProviderOutcome::Executed);
2168 }
2169 _ => {}
2170 }
2171 };
2172 }
2173 dispatch!(F32, F32);
2174 dispatch!(F64, F64);
2175 dispatch!(C32, C32);
2176 dispatch!(C64, C64);
2177 Ok(CpuProviderOutcome::Unsupported(
2178 CpuProviderUnsupported::DType(input.dtype()),
2179 ))
2180}
2181
2182pub(crate) fn builtin_gemm_provider(kind: CpuBackendKind) -> Arc<dyn CpuGemmProvider> {
2183 match kind {
2184 CpuBackendKind::Faer => Arc::new(FaerGemmProvider),
2185 CpuBackendKind::Blas => Arc::new(BlasGemmProvider),
2186 }
2187}
2188
2189pub(crate) fn builtin_layout_provider() -> Arc<dyn CpuLayoutTransformProvider> {
2190 Arc::new(StridedLayoutTransformProvider)
2191}
2192
2193#[cfg(test)]
2194pub(crate) mod tests;