tenferro_cpu/

provider.rs

//! Object-safe CPU contraction provider contracts.
//!
//! Providers synchronously write into engine-owned outputs. Request
//! constructors are crate-private because only the CPU engine may attest that
//! tensor metadata and reachable ranges have already been validated.

use core::fmt;
use std::mem::MaybeUninit;
use std::num::NonZeroUsize;
use std::sync::Arc;

use tenferro_tensor::backend::GroupedGemmJob;
use tenferro_tensor::{
    DType, DotGeneralAccumulation, Tensor, TensorRead, TensorView, TensorViewMut, TensorWrite,
};

use crate::arbiter::{with_execution_owner, ResourcePermit};
use crate::backend::CpuBackendKind;
use crate::buffer_pool::BufferPool;
use crate::domain_executor::{indexed_jobs, install_scoped};
/// The Rust scalar type behind a preset variant name a macro received.
macro_rules! preset_scalar {
    (F32) => {
        f32
    };
    (F64) => {
        f64
    };
    (I32) => {
        i32
    };
    (I64) => {
        i64
    };
    (Bool) => {
        bool
    };
    (C32) => {
        num_complex::Complex32
    };
    (C64) => {
        num_complex::Complex64
    };
}

#[cfg(feature = "cpu-blas")]
use crate::provider_capability::builtin_blas_execution_capabilities;
#[cfg(not(feature = "cpu-blas"))]
use crate::provider_capability::serial_capabilities;
use crate::provider_capability::{engine_worker_capabilities, CpuProviderExecutionCapabilities};
use crate::resource_domain::CpuResourceDomain;
use crate::{CpuDomainExecutorError, CpuDomainId, CpuInnerParallelism, CpuSet};

/// Operand named by a provider capability reason.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuOperand;
/// assert_eq!(CpuOperand::Lhs, CpuOperand::Lhs);
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CpuOperand {
    /// Left input operand.
    Lhs,
    /// Right input operand.
    Rhs,
    /// Writable output operand.
    Output,
}

/// Allocation-free reason that a provider cannot execute a validated request.
///
/// An unsupported outcome must be reported before the provider mutates the
/// output.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuProviderUnsupported;
/// assert_eq!(
///     CpuProviderUnsupported::RuntimeUnavailable,
///     CpuProviderUnsupported::RuntimeUnavailable,
/// );
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum CpuProviderUnsupported {
    /// The provider does not implement the scalar dtype.
    DType(DType),
    /// The provider does not implement the input ranks.
    Rank {
        /// Left input rank.
        lhs: usize,
        /// Right input rank.
        rhs: usize,
    },
    /// The provider does not implement an operand layout.
    Layout(CpuOperand),
    /// The provider cannot implement the requested conjugation.
    Conjugation,
    /// The provider cannot implement the requested alpha/beta update.
    Accumulation,
    /// The provider cannot implement a strided batch.
    StridedBatch,
    /// The provider cannot implement grouped GEMM.
    Grouped,
    /// The optional provider runtime is not available in this process.
    RuntimeUnavailable,
}

/// Result of attempting a validated request through one provider slot.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::{CpuProviderOutcome, CpuProviderUnsupported};
/// let outcome = CpuProviderOutcome::Unsupported(
///     CpuProviderUnsupported::RuntimeUnavailable,
/// );
/// assert!(matches!(outcome, CpuProviderOutcome::Unsupported(_)));
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[must_use]
pub enum CpuProviderOutcome {
    /// The provider fully executed the request.
    Executed,
    /// The provider did not mutate the output and resolution may continue.
    Unsupported(CpuProviderUnsupported),
}

/// Parallel scheduling mode selected for one CPU operation.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::ParallelMode;
/// assert_ne!(ParallelMode::Sequential, ParallelMode::Outer);
/// assert_ne!(ParallelMode::Outer, ParallelMode::Inner);
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ParallelMode {
    /// Neither the engine nor the provider may fan out this operation.
    Sequential,
    /// The engine owns outer fan-out and delegates Sequential child contexts.
    /// Providers do not receive an Outer context.
    Outer,
    /// One provider kernel may use the selected executor's inner region.
    Inner,
}

/// Borrowed execution policy for an already-entered CPU operation.
///
/// The context exposes immutable domain facts while keeping the resource lease,
/// executor object, and the checked executor-entry boundary private. Providers
/// cannot install or submit work through this value.
///
/// # Examples
///
/// Providers inspect this value inside a trait method:
///
/// ```
/// use tenferro_cpu::provider::CpuExecutionContext;
/// # fn inspect(context: &CpuExecutionContext<'_>) {
/// assert!(context.thread_budget().get() >= 1);
/// # }
/// ```
#[derive(Clone, Copy)]
pub struct CpuExecutionContext<'a> {
    domain: &'a CpuResourceDomain,
    parallel_mode: ParallelMode,
    // Set only for a child of tenferro's own outer fan-out (`submit_outer` or
    // `with_outer_lanes`), never inferred from running on a Rayon worker.
    outer_fan_out: bool,
    batch_policy: crate::CpuBatchPolicy,
}

impl fmt::Debug for CpuExecutionContext<'_> {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter
            .debug_struct("CpuExecutionContext")
            .field("domain_id", &self.domain_id())
            .field("cpus", &self.cpus())
            .field("thread_budget", &self.thread_budget())
            .field("parallel_mode", &self.parallel_mode())
            .field("outer_fan_out", &self.outer_fan_out)
            .finish_non_exhaustive()
    }
}

impl<'a> CpuExecutionContext<'a> {
    fn entered(
        domain: &'a CpuResourceDomain,
        parallel_mode: ParallelMode,
        batch_policy: crate::CpuBatchPolicy,
    ) -> Self {
        Self {
            domain,
            parallel_mode,
            outer_fan_out: false,
            batch_policy,
        }
    }

    /// A child of tenferro's outer fan-out: sequential, with fan-out active.
    fn outer_child(domain: &'a CpuResourceDomain, batch_policy: crate::CpuBatchPolicy) -> Self {
        Self {
            domain,
            parallel_mode: ParallelMode::Sequential,
            outer_fan_out: true,
            batch_policy,
        }
    }

    /// The effective batch policy for batched work in this context.
    ///
    /// It is the backend default unless a session scope overrides it; see
    /// [`crate::with_batch_policy`].
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::{with_cpu_exec_session, CpuBackend, CpuBatchStrategy};
    /// use tenferro_tensor::BackendSessionHost;
    ///
    /// let mut backend = CpuBackend::with_threads(1)?;
    /// let strategy = backend.with_backend_session(|session| {
    ///     with_cpu_exec_session(session, |cpu| {
    ///         cpu.with_linalg_pool(|context, _| Ok(context.batch_policy().strategy()))
    ///     })
    ///     .expect("a CPU backend session")
    /// })??;
    /// assert_eq!(strategy, CpuBatchStrategy::Auto);
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    #[must_use]
    pub fn batch_policy(&self) -> crate::CpuBatchPolicy {
        self.batch_policy
    }

    /// This context with a scoped batch-policy override.
    pub(crate) fn with_batch_policy(mut self, batch_policy: crate::CpuBatchPolicy) -> Self {
        self.batch_policy = batch_policy;
        self
    }

    /// Whether [`Self::with_outer_lanes`] can fan out from this context.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::{with_cpu_exec_session, CpuBackend};
    /// use tenferro_tensor::BackendSessionHost;
    ///
    /// let mut backend = CpuBackend::with_threads(1)?;
    /// let fans_out = backend.with_backend_session(|session| {
    ///     with_cpu_exec_session(session, |cpu| {
    ///         cpu.with_linalg_pool(|context, _| Ok(context.can_fan_out_lanes()))
    ///     })
    ///     .expect("a CPU backend session")
    /// })??;
    /// assert!(!fans_out);
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    #[must_use]
    pub fn can_fan_out_lanes(&self) -> bool {
        self.parallel_mode == ParallelMode::Inner
            && self.domain.executor_capabilities().inner_parallelism == CpuInnerParallelism::Rayon
            && self.thread_budget().get() > 1
    }

    /// Whether this context is one lane of tenferro's own outer fan-out.
    ///
    /// Such a lane runs concurrently with its siblings. Its mode is
    /// [`ParallelMode::Sequential`], so it may not start inner parallel work, and
    /// a provider whose declared parallelism is an independent runtime (the
    /// default for external BLAS/LAPACK) is rejected before dispatch. Running on
    /// a Rayon worker is not by itself outer fan-out.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::{with_cpu_exec_session, CpuBackend};
    /// use tenferro_tensor::BackendSessionHost;
    ///
    /// let mut backend = CpuBackend::with_threads(1)?;
    /// let lane = backend.with_backend_session(|session| {
    ///     with_cpu_exec_session(session, |cpu| {
    ///         cpu.with_linalg_pool(|context, _| Ok(context.is_outer_fan_out_lane()))
    ///     })
    ///     .expect("a CPU backend session")
    /// })??;
    /// assert!(!lane);
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    #[must_use]
    pub fn is_outer_fan_out_lane(&self) -> bool {
        self.outer_fan_out
    }

    /// Run independent lane jobs as tenferro-owned outer fan-out inside this
    /// already-entered context's own Rayon region.
    ///
    /// Each item of `jobs` (typically one disjoint chunk of a batch) runs once
    /// and receives a lane context: [`Self::is_outer_fan_out_lane`] is true and
    /// the mode is [`ParallelMode::Sequential`], so the lane neither starts
    /// inner parallel work nor reaches an independent-runtime provider. Fan-out
    /// needs an [`ParallelMode::Inner`] context of a Rayon executor with more
    /// than one thread; otherwise the jobs run in order on the calling thread,
    /// still with a lane context. No second pool is created: jobs run on the
    /// Rayon pool this context was entered in.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::{with_cpu_exec_session, CpuBackend};
    /// use tenferro_tensor::BackendSessionHost;
    ///
    /// let mut backend = CpuBackend::with_threads(2)?;
    /// let mut data = vec![1.0_f64; 8];
    /// backend.with_backend_session(|session| {
    ///     with_cpu_exec_session(session, |cpu| {
    ///         cpu.with_linalg_pool(|context, _| {
    ///             context.with_outer_lanes(data.chunks_mut(3), |chunk, lane| {
    ///                 assert!(lane.is_outer_fan_out_lane());
    ///                 chunk.iter_mut().for_each(|value| *value *= 2.0);
    ///             });
    ///             Ok(())
    ///         })
    ///     })
    ///     .expect("a CPU backend session")
    /// })??;
    /// assert_eq!(data, [2.0; 8]);
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    pub fn with_outer_lanes<I>(
        &self,
        jobs: I,
        job: impl Fn(I::Item, &CpuExecutionContext<'a>) + Sync,
    ) where
        I: IntoIterator,
        I::IntoIter: Send,
        I::Item: Send,
    {
        let child = Self::outer_child(self.domain, self.batch_policy);
        if !self.can_fan_out_lanes() {
            for item in jobs {
                job(item, &child);
            }
            return;
        }
        let job = &job;
        let jobs = jobs.into_iter();
        rayon::scope(|scope| {
            for item in jobs {
                scope.spawn(move |_| job(item, &child));
            }
        });
    }

    /// Return the stable identity of the selected CPU resource domain.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::CpuExecutionContext;
    /// # fn inspect(context: &CpuExecutionContext<'_>) {
    /// let _domain_id = context.domain_id();
    /// # }
    /// ```
    pub fn domain_id(&self) -> CpuDomainId {
        self.domain.id()
    }

    /// Return the selected domain's declared logical CPU set, when present.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::CpuExecutionContext;
    /// # fn inspect(context: &CpuExecutionContext<'_>) {
    /// if let Some(cpus) = context.cpus() {
    ///     assert!(!cpus.is_empty());
    /// }
    /// # }
    /// ```
    pub fn cpus(&self) -> Option<&CpuSet> {
        self.domain.cpus()
    }

    /// Return the selected domain's admission contract.
    ///
    /// # Examples
    ///
    /// ```rust
    /// use tenferro_cpu::{CpuAdmissionMode, CpuExecutionContext};
    /// # fn inspect(context: &CpuExecutionContext<'_>) {
    /// let _mode: CpuAdmissionMode = context.admission_mode();
    /// # }
    /// ```
    pub fn admission_mode(&self) -> crate::CpuAdmissionMode {
        self.domain.admission_mode()
    }

    /// Return the non-zero maximum participating-thread budget.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::CpuExecutionContext;
    /// # fn inspect(context: &CpuExecutionContext<'_>) {
    /// assert!(context.thread_budget().get() >= 1);
    /// # }
    /// ```
    pub fn thread_budget(&self) -> NonZeroUsize {
        self.domain.thread_budget()
    }

    /// Return the engine-selected scheduling mode for this entered provider call.
    ///
    /// Provider calls observe Sequential or Inner. Outer scheduling creates a
    /// separate Sequential context inside every submitted child.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::{CpuExecutionContext, ParallelMode};
    /// # fn inspect(context: &CpuExecutionContext<'_>) {
    /// assert!(matches!(
    ///     context.parallel_mode(),
    ///     ParallelMode::Sequential | ParallelMode::Inner
    /// ));
    /// # }
    /// ```
    pub fn parallel_mode(&self) -> ParallelMode {
        self.parallel_mode
    }

    /// Materialize a borrowed tensor view for one scoped operation and reclaim
    /// its temporary host buffer before returning.
    ///
    /// Owned tensor inputs are borrowed directly. View inputs are materialized
    /// from `buffers`, passed to `operation`, and returned to the same pool on
    /// both success and ordinary error. The receiver is an unforgeable proof
    /// that the caller is already inside the selected CPU execution domain.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::RuntimeState`] when the view is not
    /// accessible from CPU host memory, propagates typed view-materialization
    /// errors, and otherwise returns the error produced by `operation`.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::{with_cpu_exec_session, CpuBackend};
    /// use tenferro_tensor::{
    ///     BackendSessionHost, StridedSliceSpec, TensorRead, TensorView, TypedTensor,
    /// };
    ///
    /// let mut backend = CpuBackend::with_threads(1)?;
    /// let input = TypedTensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
    /// backend.with_backend_session(|session| {
    ///     with_cpu_exec_session(session, |cpu| {
    ///         cpu.with_linalg_pool(|context, buffers| {
    ///             let view = input
    ///                 .as_view()
    ///                 .slice_view(&[StridedSliceSpec::reverse()])?;
    ///             context.with_materialized_tensor_read(
    ///                 buffers,
    ///                 "example",
    ///                 TensorRead::from_view(TensorView::F64(view)),
    ///                 |materialized, _| {
    ///                     assert_eq!(materialized.as_slice::<f64>().unwrap(), &[2.0, 1.0]);
    ///                     Ok(())
    ///                 },
    ///             )
    ///         })
    ///     })
    ///     .expect("a CPU backend session")
    /// })??;
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    #[doc(hidden)]
    pub fn with_materialized_tensor_read<R>(
        &self,
        buffers: &mut BufferPool,
        op: &'static str,
        input: TensorRead<'_>,
        operation: impl FnOnce(&Tensor, &mut BufferPool) -> tenferro_tensor::Result<R>,
    ) -> tenferro_tensor::Result<R> {
        match input {
            TensorRead::Tensor(tensor) => operation(tensor, buffers),
            TensorRead::View(view) => {
                let materialized = self.with_native_parallelism(|| {
                    crate::materialize_tensor_read(buffers, op, TensorRead::View(view))
                })?;
                let result = operation(&materialized, buffers);
                crate::backend::reclaim_tensor(buffers, materialized);
                result
            }
        }
    }

    /// Reshape a compact tensor while retaining the current execution proof.
    ///
    /// This metadata-only helper does not enter an executor or borrow another
    /// scratch pool.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::Validation`] when the input and output
    /// shapes have different element counts or the requested layout is invalid.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::{with_cpu_exec_session, CpuBackend};
    /// use tenferro_tensor::{BackendSessionHost, Tensor};
    ///
    /// let mut backend = CpuBackend::with_threads(1)?;
    /// let input = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
    /// backend.with_backend_session(|session| {
    ///     with_cpu_exec_session(session, |cpu| {
    ///         cpu.with_linalg_pool(|context, _| {
    ///             let output = context.reshape_tensor(&input, &[2, 1])?;
    ///             assert_eq!(output.shape(), &[2, 1]);
    ///             Ok(())
    ///         })
    ///     })
    ///     .expect("a CPU backend session")
    /// })??;
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    #[doc(hidden)]
    pub fn reshape_tensor(
        &self,
        input: &Tensor,
        shape: &[usize],
    ) -> tenferro_tensor::Result<Tensor> {
        crate::structural::reshape(input, shape)
    }

    /// Return the faer policy selected by this operation context.
    ///
    /// This hidden public method is the owner-scoped extension contract used by
    /// operation-family crates such as `tenferro-linalg`. Keeping the mapping
    /// here prevents sibling crates from deriving a second CPU threading
    /// policy.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::CpuExecutionContext;
    /// # fn inspect(context: &CpuExecutionContext<'_>) {
    /// let _policy = context.faer_parallelism();
    /// # }
    /// ```
    #[cfg(feature = "cpu-faer")]
    #[doc(hidden)]
    pub fn faer_parallelism(self) -> faer::Par {
        match (
            self.parallel_mode,
            self.domain.executor_capabilities().inner_parallelism,
        ) {
            (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
                faer::Par::rayon(self.thread_budget().get())
            }
            _ => faer::Par::Seq,
        }
    }

    /// The Rayon pool this context's inner parallel region runs on.
    ///
    /// `Some` exactly when the context owns an inner region: parallel mode
    /// [`ParallelMode::Inner`], a Rayon-backed executor, and a thread budget
    /// above one (the same gate as the faer policy). The caller is already on
    /// a worker of this pool, so work installed or scoped on it runs in
    /// place. A provider running its own kernels on the pool must use at most
    /// [`CpuExecutionContext::thread_budget`] threads, which can be smaller
    /// than the pool, and declares
    /// [`crate::CpuThreadCountControl::PerCallUpperBound`] with
    /// [`crate::CpuPlacementControl::EngineWorkers`].
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::{with_cpu_exec_session, CpuBackend};
    /// use tenferro_tensor::BackendSessionHost;
    /// let mut backend = CpuBackend::with_threads(2)?;
    /// let workers = backend.with_backend_session(|session| {
    ///     with_cpu_exec_session(session, |cpu| {
    ///         cpu.with_linalg_pool(|context, _| {
    ///             Ok(context.rayon_pool().map(|pool| pool.current_num_threads()))
    ///         })
    ///     })
    ///     .expect("a CPU backend session")
    /// })??;
    /// assert_eq!(workers, Some(2));
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    pub fn rayon_pool(&self) -> Option<&'a rayon::ThreadPool> {
        match (
            self.parallel_mode,
            self.domain.executor_capabilities().inner_parallelism,
        ) {
            (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
                self.domain.executor().rayon_pool()
            }
            _ => None,
        }
    }

    /// Effective native-kernel degree inside this already-entered CPU context.
    /// Non-Rayon executors and sequential/nested policy use one thread.
    ///
    /// # Examples
    /// ```
    /// use tenferro_cpu::{with_cpu_exec_session, CpuBackend};
    /// use tenferro_tensor::BackendSessionHost;
    /// let mut backend = CpuBackend::with_threads(1)?;
    /// let threads = backend.with_backend_session(|session| {
    ///     with_cpu_exec_session(session, |cpu| {
    ///         cpu.with_linalg_pool(|context, _| Ok(context.native_thread_count()))
    ///     })
    ///     .expect("a CPU backend session")
    /// })??;
    /// assert_eq!(threads, 1);
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    #[doc(hidden)]
    pub fn native_thread_count(&self) -> usize {
        match (
            self.parallel_mode,
            self.domain.executor_capabilities().inner_parallelism,
        ) {
            (ParallelMode::Inner, CpuInnerParallelism::Rayon) => self.thread_budget().get(),
            _ => 1,
        }
    }

    pub(crate) fn strided_exec_context(&self) -> strided_kernel::ExecContext {
        match (
            self.parallel_mode,
            self.domain.executor_capabilities().inner_parallelism,
        ) {
            (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
                // This is only an operation-local thread limit for strided's
                // replay policy. The Rayon pool itself is the already-entered
                // CpuContext pool installed by `with_native_parallelism`.
                match strided_kernel::ExecContext::max_threads(self.thread_budget().get()) {
                    Ok(context) => context,
                    // INVARIANT: CpuExecutionContext stores a NonZeroUsize
                    // thread budget, and this branch passes that positive value.
                    Err(_) => unreachable!("CpuExecutionContext has a non-zero thread budget"),
                }
            }
            _ => strided_kernel::ExecContext::serial(),
        }
    }

    pub(crate) fn with_native_parallelism<R>(&self, operation: impl FnOnce() -> R) -> R {
        let policy = match (
            self.parallel_mode,
            self.domain.executor_capabilities().inner_parallelism,
        ) {
            (ParallelMode::Inner, CpuInnerParallelism::Rayon) if self.thread_budget().get() > 1 => {
                strided_kernel::ExecutionPolicy::Rayon {
                    max_threads: self.thread_budget(),
                }
            }
            _ => strided_kernel::ExecutionPolicy::Sequential,
        };
        strided_kernel::with_execution_policy(policy, operation)
    }
}

/// Proof that every provider an outer fan-out's lanes can reach accepts
/// concurrent sequential calls.
///
/// [`CpuOperationEntry::submit_outer`] takes this value, so fan-out cannot be
/// submitted before the reachable delegates' declarations are checked: a
/// declared nesting violation fails before any lane runs or writes output.
#[derive(Debug)]
pub(crate) struct OuterFanOutChecked(());

/// Check the reachable delegates of an outer fan-out before submitting it.
///
/// Every capability listed must accept [`ParallelMode::Outer`] (sequential,
/// concurrent-call safe, worker-local). An implementation whose parallelism is
/// an independent runtime, the default declaration for external BLAS/LAPACK,
/// is rejected.
pub(crate) fn check_outer_fan_out_delegates<'c>(
    delegates: impl IntoIterator<Item = &'c crate::CpuProviderExecutionCapabilities>,
) -> Result<OuterFanOutChecked, crate::CpuProviderDomainError> {
    for capabilities in delegates {
        if !capabilities.accepts_mode(ParallelMode::Outer) {
            return Err(crate::CpuProviderDomainError::ParallelModeNotSupported {
                mode: ParallelMode::Outer,
            });
        }
    }
    Ok(OuterFanOutChecked(()))
}

/// Where an outer fan-out runs: submitted to the domain executor from an
/// unentered operation, or as lanes of an already-entered Inner context.
#[derive(Clone, Copy)]
pub(crate) enum CpuOuterFanOut<'a> {
    Executor(CpuOperationEntry<'a>),
    Lanes(CpuExecutionContext<'a>),
}

impl CpuOuterFanOut<'_> {
    /// Run `len` indexed jobs, each with a lane context, and return the first
    /// job error.
    pub(crate) fn submit(
        self,
        checked: OuterFanOutChecked,
        len: usize,
        operation: impl Fn(usize, &CpuExecutionContext<'_>) -> Result<(), CpuDomainExecutorError> + Sync,
    ) -> Result<(), CpuDomainExecutorError> {
        match self {
            Self::Executor(entry) => entry.submit_outer(checked, len, operation),
            Self::Lanes(context) => {
                let first_error = std::sync::Mutex::new(None);
                context.with_outer_lanes(0..len, |index, lane| {
                    if let Err(error) = operation(index, lane) {
                        first_error
                            .lock()
                            .unwrap_or_else(std::sync::PoisonError::into_inner)
                            .get_or_insert(error);
                    }
                });
                match first_error
                    .into_inner()
                    .unwrap_or_else(std::sync::PoisonError::into_inner)
                {
                    Some(error) => Err(error),
                    None => Ok(()),
                }
            }
        }
    }
}

/// Crate-private unentered capability for one CPU operation.
///
/// This is the only type that owns the resource permit and may cross the
/// selected domain executor boundary. A [`CpuExecutionContext`] is constructed
/// only inside an installed job or an outer child job.
#[derive(Clone, Copy)]
pub(crate) struct CpuOperationEntry<'a> {
    domain: &'a CpuResourceDomain,
    permit: &'a ResourcePermit,
    batch_policy: crate::CpuBatchPolicy,
}

impl<'a> CpuOperationEntry<'a> {
    pub(crate) fn new(domain: &'a CpuResourceDomain, permit: &'a ResourcePermit) -> Self {
        Self {
            domain,
            permit,
            batch_policy: crate::CpuBatchPolicy::default(),
        }
    }

    /// This entry with a different effective batch policy.
    pub(crate) fn with_batch_policy(mut self, batch_policy: crate::CpuBatchPolicy) -> Self {
        self.batch_policy = batch_policy;
        self
    }

    pub(crate) fn batch_policy(self) -> crate::CpuBatchPolicy {
        self.batch_policy
    }

    pub(crate) fn domain_id(self) -> CpuDomainId {
        self.domain.id()
    }

    pub(crate) fn enter<R: Send>(
        self,
        parallel_mode: ParallelMode,
        operation: impl FnOnce(&CpuExecutionContext<'_>) -> R + Send,
    ) -> Result<R, CpuDomainExecutorError> {
        if parallel_mode == ParallelMode::Outer {
            return Err(CpuDomainExecutorError::Scheduling {
                message: "CPU executor install requires Sequential or Inner mode, got Outer"
                    .to_owned(),
            });
        }
        let owner = self.permit.owner();
        if crate::backend::execution_scope::is_entered(self.domain, self.permit) {
            return Ok(with_execution_owner(owner, || {
                let context =
                    CpuExecutionContext::entered(self.domain, parallel_mode, self.batch_policy);
                operation(&context)
            }));
        }
        with_execution_owner(owner, || {
            install_scoped(self.domain.executor().as_ref(), || {
                with_execution_owner(owner, || {
                    let context =
                        CpuExecutionContext::entered(self.domain, parallel_mode, self.batch_policy);
                    operation(&context)
                })
            })
        })
    }

    pub(crate) fn enter_or_reuse<R: Send>(
        self,
        entered: Option<&CpuExecutionContext<'_>>,
        parallel_mode: ParallelMode,
        operation: impl FnOnce(&CpuExecutionContext<'_>) -> R + Send,
    ) -> Result<R, CpuDomainExecutorError> {
        let Some(entered) = entered else {
            return self.enter(parallel_mode, operation);
        };
        if parallel_mode == ParallelMode::Outer {
            return Err(CpuDomainExecutorError::Scheduling {
                message: "entered CPU session requires Sequential or Inner mode, got Outer"
                    .to_owned(),
            });
        }
        if entered.domain_id() != self.domain.id() {
            return Err(CpuDomainExecutorError::Scheduling {
                message: format!(
                    "entered CPU session domain {:?} does not match operation domain {:?}",
                    entered.domain_id(),
                    self.domain.id()
                ),
            });
        }
        let owner = self.permit.owner();
        // A lane of outer fan-out keeps its fan-out fact, and it may not widen
        // its sequential policy into inner parallelism: that would nest a second
        // fan-out inside every sibling lane.
        let context = if entered.is_outer_fan_out_lane() {
            CpuExecutionContext::outer_child(self.domain, self.batch_policy)
        } else {
            CpuExecutionContext::entered(self.domain, parallel_mode, self.batch_policy)
        };
        Ok(with_execution_owner(owner, || operation(&context)))
    }

    /// Whether a backend session enters this domain's executor once for its
    /// whole callback. Externally managed domains enter per operation instead.
    pub(crate) fn enters_executor_per_session(self) -> bool {
        self.domain.ownership() == crate::CpuDomainOwnership::Managed
    }

    /// Enter a Tenferro-managed executor once for a whole backend session.
    ///
    /// Callers check [`Self::enters_executor_per_session`] first; the managed
    /// Rayon executor's synchronous install does not fail for Sequential or
    /// Inner mode, but its typed error is still reported rather than hidden.
    pub(crate) fn enter_managed_session<R: Send>(
        self,
        operation: impl FnOnce(CpuExecutionContext<'a>) -> R + Send,
    ) -> Result<R, tenferro_tensor::SessionEntryError> {
        let mode = self.preferred_engine_mode();
        self.enter(mode, |_| {
            operation(CpuExecutionContext::entered(
                self.domain,
                mode,
                self.batch_policy,
            ))
        })
        .map_err(|error| tenferro_tensor::SessionEntryError::Executor {
            backend: crate::backend::CPU_BACKEND,
            source: Box::new(error),
        })
    }

    pub(crate) fn submit_outer(
        self,
        _delegates: OuterFanOutChecked,
        len: usize,
        operation: impl Fn(usize, &CpuExecutionContext<'_>) -> Result<(), CpuDomainExecutorError> + Sync,
    ) -> Result<(), CpuDomainExecutorError> {
        if !self.supports_outer() {
            return Err(CpuDomainExecutorError::Scheduling {
                message: format!(
                    "CPU domain {:?} does not support Outer mode",
                    self.domain.id()
                ),
            });
        }
        let owner = self.permit.owner();
        let lane_count = len.min(self.domain.thread_budget().get());
        let jobs = indexed_jobs(lane_count, |lane| {
            // INVARIANT: valid lanes partition `0..len` by residue modulo the
            // nonzero `lane_count`, so every logical job runs exactly once
            // while the executor can schedule at most the domain budget.
            let mut index = lane;
            while index < len {
                with_execution_owner(owner, || {
                    let context = CpuExecutionContext::outer_child(self.domain, self.batch_policy);
                    operation(index, &context)
                })?;
                let Some(next) = index.checked_add(lane_count) else {
                    break;
                };
                index = next;
            }
            Ok(())
        });
        with_execution_owner(owner, || self.domain.executor().submit(&jobs))?;
        if let Some(index) = jobs.invalid_index_attempt() {
            return Err(CpuDomainExecutorError::Scheduling {
                message: format!(
                    "executor requested scoped CPU lane index {index}, but the submission has {lane_count} lanes for {len} logical jobs"
                ),
            });
        }
        Ok(())
    }

    pub(crate) fn preferred_engine_mode(self) -> ParallelMode {
        if self.domain.thread_budget().get() > 1
            && self.domain.executor_capabilities().inner_parallelism == CpuInnerParallelism::Rayon
        {
            ParallelMode::Inner
        } else {
            ParallelMode::Sequential
        }
    }

    pub(crate) fn preferred_provider_mode(
        self,
        accepts: impl Fn(ParallelMode) -> bool,
    ) -> Result<ParallelMode, crate::CpuProviderDomainError> {
        if self.domain.thread_budget().get() == 1 {
            return if accepts(ParallelMode::Sequential) {
                Ok(ParallelMode::Sequential)
            } else {
                Err(crate::CpuProviderDomainError::ParallelModeNotSupported {
                    mode: ParallelMode::Sequential,
                })
            };
        }
        if accepts(ParallelMode::Inner) {
            return Ok(ParallelMode::Inner);
        }
        if accepts(ParallelMode::Sequential) {
            return Ok(ParallelMode::Sequential);
        }
        Err(crate::CpuProviderDomainError::ParallelModeNotSupported {
            mode: ParallelMode::Inner,
        })
    }

    pub(crate) fn provider_default_compatibility_mode(self) -> ParallelMode {
        if self.domain.thread_budget().get() == 1 {
            ParallelMode::Sequential
        } else {
            ParallelMode::Inner
        }
    }

    pub(crate) fn thread_budget(self) -> NonZeroUsize {
        self.domain.thread_budget()
    }

    pub(crate) fn supports_outer(self) -> bool {
        self.domain.thread_budget().get() > 1
            && self.domain.executor_capabilities().outer_parallelism
    }
}

/// Checked element offset and strides for a batched matrix operand.
///
/// # Examples
///
/// Providers receive this descriptor from a validated request:
///
/// ```
/// use tenferro_cpu::provider::CpuGemmRequest;
/// # fn inspect(request: &CpuGemmRequest<'_, '_, '_>) {
/// assert!(request.lhs_layout().row_stride() != 0 || request.rows() <= 1);
/// # }
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CpuBatchedMatrixLayout {
    offset: isize,
    row_stride: isize,
    column_stride: isize,
    batch_stride: isize,
}

impl CpuBatchedMatrixLayout {
    #[allow(dead_code)]
    pub(crate) fn new(
        offset: isize,
        row_stride: isize,
        column_stride: isize,
        batch_stride: isize,
    ) -> Self {
        Self {
            offset,
            row_stride,
            column_stride,
            batch_stride,
        }
    }

    /// Return the checked base element offset.
    pub fn offset(self) -> isize {
        self.offset
    }

    /// Return the row element stride.
    pub fn row_stride(self) -> isize {
        self.row_stride
    }

    /// Return the column element stride.
    pub fn column_stride(self) -> isize {
        self.column_stride
    }

    /// Return the batch element stride.
    pub fn batch_stride(self) -> isize {
        self.batch_stride
    }
}

/// Whether a provider may, must, or must not execute a batch of GEMMs with
/// one vendor batch call (`cblas_?gemm_batch`).
///
/// The engine resolves this from the effective [`crate::CpuBatchPolicy`]
/// before any output write. A provider without a vendor batch routine treats
/// [`Self::Allowed`] and [`Self::Forbidden`] alike and reports
/// [`CpuProviderUnsupported::RuntimeUnavailable`] for [`Self::Required`].
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuVendorBatch;
///
/// let allowed = CpuVendorBatch::Allowed { max_item_dim: 16 };
/// assert!(allowed.permits([[4, 4, 4], [8, 8, 8]]));
/// assert!(!allowed.permits([[4, 4, 32], [8, 8, 8]]));
/// assert!(!CpuVendorBatch::Forbidden.permits([[1, 1, 1], [1, 1, 1]]));
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum CpuVendorBatch {
    /// Use a vendor batch call when there is more than one item and every
    /// item's `m`, `n` and `k` are at most `max_item_dim`.
    Allowed {
        /// Largest per-item dimension for which a vendor batch call is used.
        max_item_dim: usize,
    },
    /// Execute the whole batch with one vendor batch call.
    Required,
    /// Never use a vendor batch call.
    Forbidden,
}

impl Default for CpuVendorBatch {
    fn default() -> Self {
        Self::Allowed {
            max_item_dim: crate::CpuBatchThresholds::default().vendor_batch_max_item_dim(),
        }
    }
}

impl CpuVendorBatch {
    /// Whether this control selects a vendor batch call for GEMMs of the given
    /// `[m, n, k]` dimensions; [`Self::Required`] always does.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuVendorBatch;
    /// assert!(CpuVendorBatch::Required.permits([[64, 64, 64]]));
    /// ```
    #[must_use]
    pub fn permits(self, dims: impl IntoIterator<Item = [usize; 3]>) -> bool {
        match self {
            Self::Allowed { max_item_dim } => crate::CpuBatchThresholds::default()
                .with_vendor_batch_max_item_dim(max_item_dim)
                .auto_uses_vendor_batch(dims),
            Self::Required => true,
            Self::Forbidden => false,
        }
    }
}

/// Validated borrowed GEMM request.
///
/// A batch count of one is a single GEMM. A larger batch count is a strided
/// batched GEMM.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuGemmRequest;
/// # fn inspect(request: &CpuGemmRequest<'_, '_, '_>) {
/// assert!(request.batch_count() >= 1);
/// # }
/// ```
#[derive(Debug)]
pub struct CpuGemmRequest<'request, 'input, 'output> {
    lhs: &'request TensorRead<'input>,
    rhs: &'request TensorRead<'input>,
    output: &'request mut TensorWrite<'output>,
    rows: usize,
    columns: usize,
    contracted: usize,
    batch_count: usize,
    lhs_layout: CpuBatchedMatrixLayout,
    rhs_layout: CpuBatchedMatrixLayout,
    output_layout: CpuBatchedMatrixLayout,
    accumulation: DotGeneralAccumulation,
    vendor_batch: CpuVendorBatch,
}

pub(crate) struct CpuGemmRequestParts<'request, 'input, 'output> {
    pub(crate) lhs: &'request TensorRead<'input>,
    pub(crate) rhs: &'request TensorRead<'input>,
    pub(crate) output: &'request mut TensorWrite<'output>,
    pub(crate) rows: usize,
    pub(crate) columns: usize,
    pub(crate) contracted: usize,
    pub(crate) batch_count: usize,
    pub(crate) lhs_layout: CpuBatchedMatrixLayout,
    pub(crate) rhs_layout: CpuBatchedMatrixLayout,
    pub(crate) output_layout: CpuBatchedMatrixLayout,
    pub(crate) accumulation: DotGeneralAccumulation,
    // Read only by the BLAS provider; faer checks the request accessor.
    #[cfg(feature = "cpu-blas")]
    pub(crate) vendor_batch: CpuVendorBatch,
}

impl<'request, 'input, 'output> CpuGemmRequest<'request, 'input, 'output> {
    #[allow(clippy::too_many_arguments, dead_code)]
    pub(crate) fn new(
        lhs: &'request TensorRead<'input>,
        rhs: &'request TensorRead<'input>,
        output: &'request mut TensorWrite<'output>,
        rows: usize,
        columns: usize,
        contracted: usize,
        batch_count: usize,
        lhs_layout: CpuBatchedMatrixLayout,
        rhs_layout: CpuBatchedMatrixLayout,
        output_layout: CpuBatchedMatrixLayout,
        accumulation: DotGeneralAccumulation,
    ) -> Self {
        Self {
            lhs,
            rhs,
            output,
            rows,
            columns,
            contracted,
            batch_count,
            lhs_layout,
            rhs_layout,
            output_layout,
            accumulation,
            vendor_batch: CpuVendorBatch::default(),
        }
    }

    /// This request with the engine-resolved vendor-batch control.
    pub(crate) fn with_vendor_batch(mut self, vendor_batch: CpuVendorBatch) -> Self {
        self.vendor_batch = vendor_batch;
        self
    }

    /// Return whether a batch may, must or must not use a vendor batch call.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::{CpuGemmRequest, CpuVendorBatch};
    /// # fn inspect(request: &CpuGemmRequest<'_, '_, '_>) {
    /// let _ = request.vendor_batch() == CpuVendorBatch::Forbidden;
    /// # }
    /// ```
    pub fn vendor_batch(&self) -> CpuVendorBatch {
        self.vendor_batch
    }

    /// Return the borrowed left input.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.lhs();
    /// # }
    /// ```
    pub fn lhs(&self) -> &TensorRead<'input> {
        self.lhs
    }

    /// Return the borrowed right input.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.rhs();
    /// # }
    /// ```
    pub fn rhs(&self) -> &TensorRead<'input> {
        self.rhs
    }

    /// Reborrow the writable output for the duration of the current call.
    pub fn output(&mut self) -> &mut TensorWrite<'output> {
        self.output
    }

    /// Return the number of output rows.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.rows();
    /// # }
    /// ```
    pub fn rows(&self) -> usize {
        self.rows
    }

    /// Return the number of output columns.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.columns();
    /// # }
    /// ```
    pub fn columns(&self) -> usize {
        self.columns
    }

    /// Return the contracted dimension.
    pub fn contracted(&self) -> usize {
        self.contracted
    }

    /// Return the number of matrices in this strided batch.
    pub fn batch_count(&self) -> usize {
        self.batch_count
    }

    /// Return the left input matrix layout.
    pub fn lhs_layout(&self) -> CpuBatchedMatrixLayout {
        self.lhs_layout
    }

    /// Return the right input matrix layout.
    pub fn rhs_layout(&self) -> CpuBatchedMatrixLayout {
        self.rhs_layout
    }

    /// Return the output matrix layout.
    pub fn output_layout(&self) -> CpuBatchedMatrixLayout {
        self.output_layout
    }

    /// Return conjugation and alpha/beta update semantics.
    pub fn accumulation(&self) -> DotGeneralAccumulation {
        self.accumulation
    }

    pub(crate) fn into_parts(self) -> CpuGemmRequestParts<'request, 'input, 'output> {
        CpuGemmRequestParts {
            #[cfg(feature = "cpu-blas")]
            vendor_batch: self.vendor_batch,
            lhs: self.lhs,
            rhs: self.rhs,
            output: self.output,
            rows: self.rows,
            columns: self.columns,
            contracted: self.contracted,
            batch_count: self.batch_count,
            lhs_layout: self.lhs_layout,
            rhs_layout: self.rhs_layout,
            output_layout: self.output_layout,
            accumulation: self.accumulation,
        }
    }
}

/// Validated output-free borrowed GEMM request for full-overwrite destinations.
///
/// Mirrors [`CpuGemmRequest`] minus the writable output: the destination is
/// provided separately as `&mut [MaybeUninit<u8>]` to [`CpuUninitGemmProvider`]
/// callers. This request is only used for `beta == 0` (full-overwrite)
/// accumulations, where the destination is never read.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuGemmUninitRequest;
/// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
/// assert!(request.batch_count() >= 1);
/// # }
/// ```
#[derive(Debug)]
/// Output-free prepared GEMM request for uninitialized full-overwrite
/// execution (see `CpuUninitGemmProvider::gemm_into_uninit`).
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuGemmUninitRequest;
/// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
/// #     let _ = request.rows();
/// #     let _ = request.columns();
/// #     let _ = request.contracted();
/// # }
/// ```
pub struct CpuGemmUninitRequest<'request, 'input> {
    lhs: &'request TensorRead<'input>,
    rhs: &'request TensorRead<'input>,
    rows: usize,
    columns: usize,
    contracted: usize,
    batch_count: usize,
    lhs_layout: CpuBatchedMatrixLayout,
    rhs_layout: CpuBatchedMatrixLayout,
    output_layout: CpuBatchedMatrixLayout,
    accumulation: DotGeneralAccumulation,
}

#[cfg(any(
    feature = "cpu-faer",
    all(feature = "cpu-blas", not(feature = "provider-inject"))
))]
pub(crate) struct CpuGemmUninitRequestParts<'request, 'input> {
    pub(crate) lhs: &'request TensorRead<'input>,
    pub(crate) rhs: &'request TensorRead<'input>,
    pub(crate) rows: usize,
    pub(crate) columns: usize,
    pub(crate) contracted: usize,
    pub(crate) batch_count: usize,
    pub(crate) lhs_layout: CpuBatchedMatrixLayout,
    pub(crate) rhs_layout: CpuBatchedMatrixLayout,
    pub(crate) output_layout: CpuBatchedMatrixLayout,
    pub(crate) accumulation: DotGeneralAccumulation,
}

impl<'request, 'input> CpuGemmUninitRequest<'request, 'input> {
    #[allow(clippy::too_many_arguments, dead_code)]
    pub(crate) fn new(
        lhs: &'request TensorRead<'input>,
        rhs: &'request TensorRead<'input>,
        rows: usize,
        columns: usize,
        contracted: usize,
        batch_count: usize,
        lhs_layout: CpuBatchedMatrixLayout,
        rhs_layout: CpuBatchedMatrixLayout,
        output_layout: CpuBatchedMatrixLayout,
        accumulation: DotGeneralAccumulation,
    ) -> Self {
        Self {
            lhs,
            rhs,
            rows,
            columns,
            contracted,
            batch_count,
            lhs_layout,
            rhs_layout,
            output_layout,
            accumulation,
        }
    }

    /// Return the `lhs` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.lhs();
    /// # }
    /// ```
    pub fn lhs(&self) -> &TensorRead<'input> {
        self.lhs
    }

    /// Return the `rhs` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.rhs();
    /// # }
    /// ```
    pub fn rhs(&self) -> &TensorRead<'input> {
        self.rhs
    }

    /// Return the `rows` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.rows();
    /// # }
    /// ```
    pub fn rows(&self) -> usize {
        self.rows
    }

    /// Return the `columns` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.columns();
    /// # }
    /// ```
    pub fn columns(&self) -> usize {
        self.columns
    }

    /// Return the `contracted` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.contracted();
    /// # }
    /// ```
    pub fn contracted(&self) -> usize {
        self.contracted
    }

    /// Return the `batch_count` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.batch_count();
    /// # }
    /// ```
    pub fn batch_count(&self) -> usize {
        self.batch_count
    }

    /// Return the `lhs_layout` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.lhs_layout();
    /// # }
    /// ```
    pub fn lhs_layout(&self) -> CpuBatchedMatrixLayout {
        self.lhs_layout
    }

    /// Return the `rhs_layout` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.rhs_layout();
    /// # }
    /// ```
    pub fn rhs_layout(&self) -> CpuBatchedMatrixLayout {
        self.rhs_layout
    }

    /// Return the `output_layout` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.output_layout();
    /// # }
    /// ```
    pub fn output_layout(&self) -> CpuBatchedMatrixLayout {
        self.output_layout
    }

    /// Return the `accumulation` of this output-free request.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmUninitRequest;
    /// # fn inspect(request: &CpuGemmUninitRequest<'_, '_>) {
    /// #     let _ = request.accumulation();
    /// # }
    /// ```
    pub fn accumulation(&self) -> DotGeneralAccumulation {
        self.accumulation
    }

    #[cfg(any(
        feature = "cpu-faer",
        all(feature = "cpu-blas", not(feature = "provider-inject"))
    ))]
    pub(crate) fn into_parts(self) -> CpuGemmUninitRequestParts<'request, 'input> {
        CpuGemmUninitRequestParts {
            lhs: self.lhs,
            rhs: self.rhs,
            rows: self.rows,
            columns: self.columns,
            contracted: self.contracted,
            batch_count: self.batch_count,
            lhs_layout: self.lhs_layout,
            rhs_layout: self.rhs_layout,
            output_layout: self.output_layout,
            accumulation: self.accumulation,
        }
    }
}

/// Validated borrowed grouped-GEMM request.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuGroupedGemmRequest;
/// # fn inspect(request: &CpuGroupedGemmRequest<'_, '_, '_>) {
/// assert_eq!(request.jobs().len(), request.jobs().iter().count());
/// # }
/// ```
#[derive(Debug)]
pub struct CpuGroupedGemmRequest<'request, 'input, 'output> {
    lhs: &'request TensorRead<'input>,
    rhs: &'request TensorRead<'input>,
    output: &'request mut TensorWrite<'output>,
    jobs: &'request [GroupedGemmJob],
    accumulation: DotGeneralAccumulation,
    vendor_batch: CpuVendorBatch,
}

impl<'request, 'input, 'output> CpuGroupedGemmRequest<'request, 'input, 'output> {
    /// This request with the engine-resolved vendor-batch control.
    pub(crate) fn with_vendor_batch(mut self, vendor_batch: CpuVendorBatch) -> Self {
        self.vendor_batch = vendor_batch;
        self
    }

    /// Return whether the jobs may, must or must not use a vendor batch call.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::{CpuGroupedGemmRequest, CpuVendorBatch};
    /// # fn inspect(request: &CpuGroupedGemmRequest<'_, '_, '_>) {
    /// let _ = request.vendor_batch() == CpuVendorBatch::Required;
    /// # }
    /// ```
    pub fn vendor_batch(&self) -> CpuVendorBatch {
        self.vendor_batch
    }

    #[allow(dead_code)]
    pub(crate) fn new(
        lhs: &'request TensorRead<'input>,
        rhs: &'request TensorRead<'input>,
        output: &'request mut TensorWrite<'output>,
        jobs: &'request [GroupedGemmJob],
        accumulation: DotGeneralAccumulation,
    ) -> Self {
        Self {
            lhs,
            rhs,
            output,
            jobs,
            accumulation,
            vendor_batch: CpuVendorBatch::default(),
        }
    }

    /// Return the borrowed left input.
    pub fn lhs(&self) -> &TensorRead<'input> {
        self.lhs
    }

    /// Return the borrowed right input.
    pub fn rhs(&self) -> &TensorRead<'input> {
        self.rhs
    }

    /// Reborrow the writable output.
    pub fn output(&mut self) -> &mut TensorWrite<'output> {
        self.output
    }

    /// Return ordered, pairwise-disjoint validated jobs.
    pub fn jobs(&self) -> &[GroupedGemmJob] {
        self.jobs
    }

    /// Return shared conjugation and alpha/beta semantics.
    pub fn accumulation(&self) -> DotGeneralAccumulation {
        self.accumulation
    }

    pub(crate) fn into_parts(
        self,
    ) -> (
        &'request TensorRead<'input>,
        &'request TensorRead<'input>,
        &'request mut TensorWrite<'output>,
        &'request [GroupedGemmJob],
        DotGeneralAccumulation,
    ) {
        (
            self.lhs,
            self.rhs,
            self.output,
            self.jobs,
            self.accumulation,
        )
    }
}

/// Engine-requested layout materialization.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuLayoutTransformIntent;
/// assert_eq!(
///     CpuLayoutTransformIntent::CanonicalColumnMajor,
///     CpuLayoutTransformIntent::CanonicalColumnMajor,
/// );
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CpuLayoutTransformIntent {
    /// Materialize a compact canonical column-major tensor.
    CanonicalColumnMajor,
}

/// Validated borrowed layout-transform request.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::{CpuLayoutTransformIntent, CpuLayoutTransformRequest};
/// # fn inspect(request: &CpuLayoutTransformRequest<'_, '_, '_>) {
/// assert_eq!(request.intent(), CpuLayoutTransformIntent::CanonicalColumnMajor);
/// # }
/// ```
#[derive(Debug)]
pub struct CpuLayoutTransformRequest<'request, 'input, 'output> {
    input: &'request TensorRead<'input>,
    output: &'request mut TensorWrite<'output>,
    intent: CpuLayoutTransformIntent,
    conjugate: bool,
}

impl<'request, 'input, 'output> CpuLayoutTransformRequest<'request, 'input, 'output> {
    #[allow(dead_code)]
    pub(crate) fn new(
        input: &'request TensorRead<'input>,
        output: &'request mut TensorWrite<'output>,
        intent: CpuLayoutTransformIntent,
        conjugate: bool,
    ) -> Self {
        Self {
            input,
            output,
            intent,
            conjugate,
        }
    }

    /// Return the borrowed input.
    pub fn input(&self) -> &TensorRead<'input> {
        self.input
    }

    /// Reborrow the writable output.
    pub fn output(&mut self) -> &mut TensorWrite<'output> {
        self.output
    }

    /// Return the requested materialization intent.
    pub fn intent(&self) -> CpuLayoutTransformIntent {
        self.intent
    }

    /// Return whether materialization must conjugate each input element.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuLayoutTransformRequest;
    /// # fn inspect(request: &CpuLayoutTransformRequest<'_, '_, '_>) {
    /// let _must_conjugate = request.conjugate();
    /// # }
    /// ```
    pub fn conjugate(&self) -> bool {
        self.conjugate
    }

    pub(crate) fn into_parts(
        self,
    ) -> (
        &'request TensorRead<'input>,
        &'request mut TensorWrite<'output>,
        CpuLayoutTransformIntent,
        bool,
    ) {
        (self.input, self.output, self.intent, self.conjugate)
    }
}

/// Validated ordered contraction-role groups.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuDotGeneralRequest;
/// # fn inspect(request: &CpuDotGeneralRequest<'_, '_, '_>) {
/// let _pairs = request.axes().contracting_pairs().count();
/// # }
/// ```
#[derive(Clone, Copy, Debug)]
pub struct CpuContractionAxes<'a> {
    lhs_rank: usize,
    rhs_rank: usize,
    lhs_contracting: &'a [usize],
    rhs_contracting: &'a [usize],
    lhs_batch: &'a [usize],
    rhs_batch: &'a [usize],
    lhs_role_mask: Option<u64>,
    rhs_role_mask: Option<u64>,
}

impl<'a> CpuContractionAxes<'a> {
    #[allow(clippy::too_many_arguments, dead_code)]
    pub(crate) fn new(
        lhs_rank: usize,
        rhs_rank: usize,
        lhs_contracting: &'a [usize],
        rhs_contracting: &'a [usize],
        lhs_batch: &'a [usize],
        rhs_batch: &'a [usize],
        lhs_role_mask: Option<u64>,
        rhs_role_mask: Option<u64>,
    ) -> Self {
        Self {
            lhs_rank,
            rhs_rank,
            lhs_contracting,
            rhs_contracting,
            lhs_batch,
            rhs_batch,
            lhs_role_mask,
            rhs_role_mask,
        }
    }

    /// Return ordered contracting-axis pairs.
    pub fn contracting_pairs(&self) -> impl ExactSizeIterator<Item = (usize, usize)> + '_ {
        self.lhs_contracting
            .iter()
            .copied()
            .zip(self.rhs_contracting.iter().copied())
    }

    /// Return ordered batch-axis pairs.
    pub fn batch_pairs(&self) -> impl ExactSizeIterator<Item = (usize, usize)> + '_ {
        self.lhs_batch
            .iter()
            .copied()
            .zip(self.rhs_batch.iter().copied())
    }

    /// Return left axes that are neither contracting nor batch axes.
    pub fn lhs_free_axes(&self) -> impl Iterator<Item = usize> + '_ {
        (0..self.lhs_rank).filter(move |&axis| !self.lhs_axis_has_role(axis))
    }

    /// Return right axes that are neither contracting nor batch axes.
    pub fn rhs_free_axes(&self) -> impl Iterator<Item = usize> + '_ {
        (0..self.rhs_rank).filter(move |&axis| !self.rhs_axis_has_role(axis))
    }

    fn lhs_axis_has_role(&self, axis: usize) -> bool {
        self.lhs_role_mask.map_or_else(
            || self.lhs_contracting.contains(&axis) || self.lhs_batch.contains(&axis),
            |mask| mask & (1_u64 << axis) != 0,
        )
    }

    fn rhs_axis_has_role(&self, axis: usize) -> bool {
        self.rhs_role_mask.map_or_else(
            || self.rhs_contracting.contains(&axis) || self.rhs_batch.contains(&axis),
            |mask| mask & (1_u64 << axis) != 0,
        )
    }
}

/// Validated borrowed semantic binary `dot_general` request.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuDotGeneralRequest;
/// # fn inspect(request: &CpuDotGeneralRequest<'_, '_, '_>) {
/// let _ = request.accumulation();
/// # }
/// ```
#[derive(Debug)]
pub struct CpuDotGeneralRequest<'request, 'input, 'output> {
    lhs: &'request TensorRead<'input>,
    rhs: &'request TensorRead<'input>,
    output: &'request mut TensorWrite<'output>,
    axes: CpuContractionAxes<'request>,
    accumulation: DotGeneralAccumulation,
}

impl<'request, 'input, 'output> CpuDotGeneralRequest<'request, 'input, 'output> {
    #[allow(dead_code)]
    pub(crate) fn new(
        lhs: &'request TensorRead<'input>,
        rhs: &'request TensorRead<'input>,
        output: &'request mut TensorWrite<'output>,
        axes: CpuContractionAxes<'request>,
        accumulation: DotGeneralAccumulation,
    ) -> Self {
        Self {
            lhs,
            rhs,
            output,
            axes,
            accumulation,
        }
    }

    /// Return the borrowed left input.
    pub fn lhs(&self) -> &TensorRead<'input> {
        self.lhs
    }

    /// Return the borrowed right input.
    pub fn rhs(&self) -> &TensorRead<'input> {
        self.rhs
    }

    /// Reborrow the writable output.
    pub fn output(&mut self) -> &mut TensorWrite<'output> {
        self.output
    }

    /// Return the validated ordered axis groups.
    pub fn axes(&self) -> &CpuContractionAxes<'request> {
        &self.axes
    }

    /// Return conjugation and alpha/beta update semantics.
    pub fn accumulation(&self) -> DotGeneralAccumulation {
        self.accumulation
    }

    /// Consume the request and return the validated operand borrows.
    ///
    /// External general-contraction providers use this when they need
    /// simultaneous immutable access to both inputs and mutable access to the
    /// output.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuDotGeneralRequest;
    ///
    /// fn consume_request(request: CpuDotGeneralRequest<'_, '_, '_>) {
    ///     let (_lhs, _rhs, _output, _axes, _accumulation) = request.into_parts();
    /// }
    /// ```
    pub fn into_parts(
        self,
    ) -> (
        &'request TensorRead<'input>,
        &'request TensorRead<'input>,
        &'request mut TensorWrite<'output>,
        CpuContractionAxes<'request>,
        DotGeneralAccumulation,
    ) {
        (
            self.lhs,
            self.rhs,
            self.output,
            self.axes,
            self.accumulation,
        )
    }
}

/// Provider for validated GEMM-family requests.
///
/// # Examples
///
/// Trait objects are supported directly:
///
/// ```
/// use tenferro_cpu::provider::CpuGemmProvider;
/// # fn accepts_provider(_: &dyn CpuGemmProvider) {}
/// ```
pub trait CpuGemmProvider: fmt::Debug + Send + Sync + 'static {
    /// Return immutable count, placement, and fan-out capabilities.
    ///
    /// This declaration must describe controls actually applied and restored
    /// by the provider adapter around each call. Merely discovering a runtime
    /// symbol is insufficient. A provider bundle samples this method exactly
    /// once during construction and keeps that snapshot for its lifetime; the
    /// returned contract must therefore remain valid for the provider object.
    fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;

    /// Execute one validated GEMM.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::BackendSource`] or
    /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
    /// fails. A detected inconsistency in engine-attested request metadata is
    /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
    /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
    fn gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome>;

    /// Execute one validated strided-batched GEMM.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::BackendSource`] or
    /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
    /// fails. A detected inconsistency in engine-attested request metadata is
    /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
    /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
    fn strided_batched_gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome>;

    /// Execute one validated grouped GEMM.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::BackendSource`] or
    /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
    /// fails. A detected inconsistency in engine-attested request metadata is
    /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
    /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
    fn grouped_gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGroupedGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome>;

    /// Return the structural full-overwrite witness for this provider.
    ///
    /// Returns `Some(self)` only for types that implement
    /// [`CpuUninitGemmProvider`] via an `unsafe impl`, so a `Some` witness is
    /// structural proof that the provider asserted the full-overwrite
    /// destination contract. Defaults to `None` (opted out).
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuGemmProvider;
    /// # fn inspect(provider: &dyn CpuGemmProvider) {
    /// #     let _ = provider.uninit_provider();
    /// # }
    /// ```
    fn uninit_provider(&self) -> Option<&dyn CpuUninitGemmProvider> {
        None
    }
}

/// Provider for validated GEMM-family requests that fully overwrite their
/// destination (only `beta == 0` accumulations).
///
/// # Safety
///
/// Implementors assert that [`CpuUninitGemmProvider::gemm_into_uninit`] writes
/// every logical output element before returning
/// [`CpuProviderOutcome::Executed`]; partial writes make the caller's
/// `assume_init` undefined behavior. Implementations must never read the
/// destination. This trait is only invoked for `beta == 0` accumulations.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuUninitGemmProvider;
/// # fn accepts_provider(_: &dyn CpuUninitGemmProvider) {}
/// ```
pub unsafe trait CpuUninitGemmProvider: CpuGemmProvider {
    /// Execute one validated full-overwrite GEMM into uninitialized bytes.
    ///
    /// # Safety
    ///
    /// Must write every element of the destination represented by
    /// `output_bytes` (per `request.output_layout()`) before returning
    /// `Executed`; partial writes are UB at the caller's `assume_init`. The
    /// destination is never read. Invoked only for `beta == 0` accumulations.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::BackendSource`] or
    /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
    /// fails. A detected inconsistency in engine-attested request metadata is
    /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported
    /// capabilities use [`CpuProviderOutcome::Unsupported`] instead.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::{CpuGemmUninitRequest, CpuProviderOutcome, CpuUninitGemmProvider};
    /// use tenferro_cpu::CpuExecutionContext;
    /// use std::mem::MaybeUninit;
    /// # fn inspect(
    /// #     provider: &dyn CpuUninitGemmProvider,
    /// #     context: &CpuExecutionContext<'_>,
    /// #     request: CpuGemmUninitRequest<'_, '_>,
    /// #     output_bytes: &mut [MaybeUninit<u8>],
    /// # ) -> Option<tenferro_tensor::Result<CpuProviderOutcome>> {
    /// #     let _ = (provider, context, request, output_bytes);
    /// #     None
    /// # }
    /// ```
    unsafe fn gemm_into_uninit(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmUninitRequest<'_, '_>,
        output_bytes: &mut [MaybeUninit<u8>],
    ) -> tenferro_tensor::Result<CpuProviderOutcome>;
}

/// Provider for engine-owned tensor materialization.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuLayoutTransformProvider;
/// # fn accepts_provider(_: &dyn CpuLayoutTransformProvider) {}
/// ```
pub trait CpuLayoutTransformProvider: fmt::Debug + Send + Sync + 'static {
    /// Return immutable count, placement, and fan-out capabilities.
    ///
    /// A provider bundle samples this method exactly once during construction
    /// and uses the stored descriptor for all validation and dispatch.
    fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;

    /// Materialize one validated input into a preallocated output.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::BackendSource`] or
    /// [`tenferro_tensor::Error::BackendFailure`] when execution fails. A
    /// detected inconsistency in engine-attested layout or range metadata is
    /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported layouts
    /// use [`CpuProviderOutcome::Unsupported`] instead.
    fn materialize(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuLayoutTransformRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome>;

    /// Return the structural full-overwrite witness for this provider.
    ///
    /// Returns `Some(self)` only for types that implement
    /// [`CpuUninitLayoutTransformProvider`] via an `unsafe impl`, so a `Some`
    /// witness is structural proof that the provider asserted the
    /// full-overwrite destination contract. Defaults to `None` (opted out).
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::CpuLayoutTransformProvider;
    /// # fn inspect(provider: &dyn CpuLayoutTransformProvider) {
    /// #     let _ = provider.uninit_provider();
    /// # }
    /// ```
    fn uninit_provider(&self) -> Option<&dyn CpuUninitLayoutTransformProvider> {
        None
    }
}

/// Provider for engine-owned tensor materialization into uninitialized bytes.
///
/// # Safety
///
/// Implementors assert that [`CpuUninitLayoutTransformProvider::materialize_into_uninit`]
/// writes every element of `output_bytes` before returning
/// [`CpuProviderOutcome::Executed`]; partial writes make the caller's
/// `assume_init` undefined behavior. Implementations must never read the
/// destination.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuUninitLayoutTransformProvider;
/// # fn accepts_provider(_: &dyn CpuUninitLayoutTransformProvider) {}
/// ```
pub unsafe trait CpuUninitLayoutTransformProvider: CpuLayoutTransformProvider {
    /// Materialize one validated input into uninitialized bytes.
    ///
    /// # Safety
    ///
    /// Must write every element of `output_bytes` (a compact column-major
    /// destination of the input shape for
    /// [`CpuLayoutTransformIntent::CanonicalColumnMajor`]) before returning
    /// `Executed`; partial writes are UB at the caller's `assume_init`. The
    /// destination is never read.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::BackendSource`] or
    /// [`tenferro_tensor::Error::BackendFailure`] when execution fails. A
    /// detected inconsistency in engine-attested layout or range metadata is
    /// returned as [`tenferro_tensor::Error::Validation`]. Unsupported layouts
    /// use [`CpuProviderOutcome::Unsupported`] instead.
    ///
    /// # Examples
    ///
    /// ```
    /// use tenferro_cpu::provider::{CpuLayoutTransformIntent, CpuProviderOutcome, CpuUninitLayoutTransformProvider};
    /// use tenferro_cpu::CpuExecutionContext;
    /// use tenferro_tensor::TensorRead;
    /// use std::mem::MaybeUninit;
    /// # fn inspect(
    /// #     provider: &dyn CpuUninitLayoutTransformProvider,
    /// #     context: &CpuExecutionContext<'_>,
    /// #     input: &TensorRead<'_>,
    /// #     intent: CpuLayoutTransformIntent,
    /// #     output_bytes: &mut [MaybeUninit<u8>],
    /// # ) -> Option<tenferro_tensor::Result<CpuProviderOutcome>> {
    /// #     let _ = (provider, context, input, intent, output_bytes);
    /// #     None
    /// # }
    /// ```
    unsafe fn materialize_into_uninit(
        &self,
        context: &CpuExecutionContext<'_>,
        input: &TensorRead<'_>,
        intent: CpuLayoutTransformIntent,
        conjugate: bool,
        output_bytes: &mut [MaybeUninit<u8>],
    ) -> tenferro_tensor::Result<CpuProviderOutcome>;
}

/// Provider for complete validated binary `dot_general` requests.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::CpuGeneralContractionProvider;
/// # fn accepts_provider(_: &dyn CpuGeneralContractionProvider) {}
/// ```
pub trait CpuGeneralContractionProvider: fmt::Debug + Send + Sync + 'static {
    /// Return immutable count, placement, and fan-out capabilities.
    ///
    /// A provider bundle samples this method exactly once during construction
    /// and uses the stored descriptor for all validation and dispatch.
    fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities;

    /// Execute one complete semantic contraction.
    ///
    /// # Errors
    ///
    /// Returns [`tenferro_tensor::Error::BackendSource`] or
    /// [`tenferro_tensor::Error::BackendFailure`] when the provider runtime
    /// fails. A detected inconsistency in engine-attested axes or range
    /// metadata is returned as [`tenferro_tensor::Error::Validation`].
    /// Unsupported contractions use [`CpuProviderOutcome::Unsupported`]
    /// instead.
    fn dot_general(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuDotGeneralRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome>;
}

/// Built-in faer GEMM provider.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::{CpuGemmProvider, FaerGemmProvider};
/// let provider: &dyn CpuGemmProvider = &FaerGemmProvider;
/// let _ = provider;
/// ```
#[derive(Clone, Copy, Debug, Default)]
pub struct FaerGemmProvider;

impl CpuGemmProvider for FaerGemmProvider {
    fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
        engine_worker_capabilities()
    }

    fn gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        // faer has no vendor batch routine; a forced whole-batch call is
        // reported instead of silently becoming a per-item loop.
        if request.vendor_batch() == CpuVendorBatch::Required {
            return Ok(CpuProviderOutcome::Unsupported(
                CpuProviderUnsupported::RuntimeUnavailable,
            ));
        }
        #[cfg(feature = "cpu-faer")]
        {
            crate::gemm::execute_faer_gemm_request(context, request)
        }
        #[cfg(not(feature = "cpu-faer"))]
        {
            let _ = (context, request);
            Ok(CpuProviderOutcome::Unsupported(
                CpuProviderUnsupported::RuntimeUnavailable,
            ))
        }
    }

    fn strided_batched_gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        self.gemm(context, request)
    }

    fn grouped_gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGroupedGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        if request.vendor_batch() == CpuVendorBatch::Required {
            return Ok(CpuProviderOutcome::Unsupported(
                CpuProviderUnsupported::RuntimeUnavailable,
            ));
        }
        #[cfg(feature = "cpu-faer")]
        {
            crate::gemm::execute_faer_grouped_request(context, request)
        }
        #[cfg(not(feature = "cpu-faer"))]
        {
            let _ = (context, request);
            Ok(CpuProviderOutcome::Unsupported(
                CpuProviderUnsupported::RuntimeUnavailable,
            ))
        }
    }

    fn uninit_provider(&self) -> Option<&dyn CpuUninitGemmProvider> {
        Some(self)
    }
}

// SAFETY: `gemm_into_uninit` runs the faer GEMM with `Accum::Replace` semantics
// (beta == 0 never reads the destination) and writes zeros for empty
// contractions, so every logical output element is written before `Executed`.
// See `crate::gemm::execute_faer_gemm_request_into_uninit`.
unsafe impl CpuUninitGemmProvider for FaerGemmProvider {
    unsafe fn gemm_into_uninit(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmUninitRequest<'_, '_>,
        output_bytes: &mut [MaybeUninit<u8>],
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        #[cfg(feature = "cpu-faer")]
        {
            crate::gemm::execute_faer_gemm_request_into_uninit(context, request, output_bytes)
        }
        #[cfg(not(feature = "cpu-faer"))]
        {
            let _ = (context, request, output_bytes);
            Ok(CpuProviderOutcome::Unsupported(
                CpuProviderUnsupported::RuntimeUnavailable,
            ))
        }
    }
}

/// Built-in BLAS GEMM provider.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::{BlasGemmProvider, CpuGemmProvider};
/// let provider: &dyn CpuGemmProvider = &BlasGemmProvider;
/// let _ = provider;
/// ```
#[derive(Clone, Copy, Debug, Default)]
pub struct BlasGemmProvider;

impl CpuGemmProvider for BlasGemmProvider {
    fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
        #[cfg(feature = "cpu-blas")]
        {
            builtin_blas_execution_capabilities()
        }
        #[cfg(not(feature = "cpu-blas"))]
        {
            // This build returns RuntimeUnavailable without invoking BLAS.
            serial_capabilities()
        }
    }

    fn gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        #[cfg(feature = "cpu-blas")]
        {
            crate::gemm::execute_blas_gemm_request(context, request)
        }
        #[cfg(not(feature = "cpu-blas"))]
        {
            let _ = (context, request);
            Ok(CpuProviderOutcome::Unsupported(
                CpuProviderUnsupported::RuntimeUnavailable,
            ))
        }
    }

    fn strided_batched_gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        self.gemm(context, request)
    }

    fn grouped_gemm(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGroupedGemmRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        #[cfg(feature = "cpu-blas")]
        {
            crate::gemm::execute_blas_grouped_request(context, request)
        }
        #[cfg(not(feature = "cpu-blas"))]
        {
            let _ = (context, request);
            Ok(CpuProviderOutcome::Unsupported(
                CpuProviderUnsupported::RuntimeUnavailable,
            ))
        }
    }

    fn uninit_provider(&self) -> Option<&dyn CpuUninitGemmProvider> {
        // Injected pointers currently promise ABI compatibility, not the
        // stronger full-overwrite witness. Preserve their initialized path.
        #[cfg(feature = "provider-inject")]
        {
            None
        }
        #[cfg(not(feature = "provider-inject"))]
        {
            Some(self)
        }
    }
}

// SAFETY: the implementation rejects nonzero beta, validates destination byte
// length/alignment, and uses BLAS's beta=0 full-overwrite contract through raw
// pointers. Empty contractions explicitly initialize their output to zero.
#[cfg(not(feature = "provider-inject"))]
unsafe impl CpuUninitGemmProvider for BlasGemmProvider {
    unsafe fn gemm_into_uninit(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuGemmUninitRequest<'_, '_>,
        output_bytes: &mut [MaybeUninit<u8>],
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        #[cfg(feature = "cpu-blas")]
        {
            crate::gemm::execute_blas_gemm_request_into_uninit(context, request, output_bytes)
        }
        #[cfg(not(feature = "cpu-blas"))]
        {
            let _ = (context, request, output_bytes);
            Ok(CpuProviderOutcome::Unsupported(
                CpuProviderUnsupported::RuntimeUnavailable,
            ))
        }
    }
}

/// Built-in strided layout materialization provider.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::provider::{CpuLayoutTransformProvider, StridedLayoutTransformProvider};
/// let provider: &dyn CpuLayoutTransformProvider = &StridedLayoutTransformProvider;
/// let _ = provider;
/// ```
#[derive(Clone, Copy, Debug, Default)]
pub struct StridedLayoutTransformProvider;

impl CpuLayoutTransformProvider for StridedLayoutTransformProvider {
    fn execution_capabilities(&self) -> CpuProviderExecutionCapabilities {
        engine_worker_capabilities()
    }

    fn materialize(
        &self,
        context: &CpuExecutionContext<'_>,
        request: CpuLayoutTransformRequest<'_, '_, '_>,
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        context.with_native_parallelism(|| materialize_strided_layout(request))
    }

    fn uninit_provider(&self) -> Option<&dyn CpuUninitLayoutTransformProvider> {
        Some(self)
    }
}

// SAFETY: `materialize_into_uninit` replays the layout-transform copy over the
// full destination (every element written via `MaybeUninit` writes from the
// source); zero-element destinations are trivially satisfied.
unsafe impl CpuUninitLayoutTransformProvider for StridedLayoutTransformProvider {
    unsafe fn materialize_into_uninit(
        &self,
        context: &CpuExecutionContext<'_>,
        input: &TensorRead<'_>,
        intent: CpuLayoutTransformIntent,
        conjugate: bool,
        output_bytes: &mut [MaybeUninit<u8>],
    ) -> tenferro_tensor::Result<CpuProviderOutcome> {
        context.with_native_parallelism(|| {
            materialize_strided_layout_into_uninit(input, intent, conjugate, output_bytes)
        })
    }
}

fn materialize_strided_layout(
    request: CpuLayoutTransformRequest<'_, '_, '_>,
) -> tenferro_tensor::Result<CpuProviderOutcome> {
    let (input, output, _intent, conjugate) = request.into_parts();
    if conjugate {
        macro_rules! dispatch_conjugated {
            ($owned:ident, $view:ident) => {
                match (input, &mut *output) {
                    (
                        TensorRead::Tensor(input),
                        TensorWrite::Tensor(output),
                    ) if input.dtype() == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype()
                        && output.dtype() == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype() =>
                    {
                        let input = input.as_typed::<preset_scalar!($owned)>()
                            .expect("the dtype guard selects this arm");
                        let output = output.as_typed_mut::<preset_scalar!($owned)>()
                            .expect("the dtype guard selects this arm");
                        let input = input.as_view();
                        let mut output = output.as_view_mut();
                        crate::structural::typed_conjugate_view_into(
                            &input,
                            &mut output,
                            "cpu layout materialization",
                        )?;
                        return Ok(CpuProviderOutcome::Executed);
                    }
                    (
                        TensorRead::View(TensorView::$view(input)),
                        TensorWrite::Tensor(output),
                    ) if output.dtype() == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype() => {
                        let output = output.as_typed_mut::<preset_scalar!($owned)>()
                            .expect("the dtype guard selects this arm");
                        let mut output = output.as_view_mut();
                        crate::structural::typed_conjugate_view_into(
                            input,
                            &mut output,
                            "cpu layout materialization",
                        )?;
                        return Ok(CpuProviderOutcome::Executed);
                    }
                    (
                        TensorRead::Tensor(input),
                        TensorWrite::View(TensorViewMut::$view(output)),
                    ) if input.dtype() == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype() => {
                        let input = input.as_typed::<preset_scalar!($owned)>()
                            .expect("the dtype guard selects this arm");
                        let input = input.as_view();
                        crate::structural::typed_conjugate_view_into(
                            &input,
                            output,
                            "cpu layout materialization",
                        )?;
                        return Ok(CpuProviderOutcome::Executed);
                    }
                    (
                        TensorRead::View(TensorView::$view(input)),
                        TensorWrite::View(TensorViewMut::$view(output)),
                    ) => {
                        crate::structural::typed_conjugate_view_into(
                            input,
                            output,
                            "cpu layout materialization",
                        )?;
                        return Ok(CpuProviderOutcome::Executed);
                    }
                    _ => {}
                }
            };
        }
        dispatch_conjugated!(F32, F32);
        dispatch_conjugated!(F64, F64);
        dispatch_conjugated!(C32, C32);
        dispatch_conjugated!(C64, C64);
        return Ok(CpuProviderOutcome::Unsupported(
            CpuProviderUnsupported::DType(input.dtype()),
        ));
    }
    macro_rules! dispatch {
        ($owned:ident, $view:ident) => {
            match (input, &mut *output) {
                (TensorRead::Tensor(input), TensorWrite::Tensor(output))
                    if input.dtype()
                        == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype()
                        && output.dtype()
                            == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype(
                            ) =>
                {
                    let input = input
                        .as_typed::<preset_scalar!($owned)>()
                        .expect("the dtype guard selects this arm");
                    let output = output
                        .as_typed_mut::<preset_scalar!($owned)>()
                        .expect("the dtype guard selects this arm");
                    let input = input.as_view();
                    let mut output = output.as_view_mut();
                    crate::structural::typed_copy_view_into(
                        &input,
                        &mut output,
                        "cpu layout materialization",
                    )?;
                    return Ok(CpuProviderOutcome::Executed);
                }
                (TensorRead::View(TensorView::$view(input)), TensorWrite::Tensor(output))
                    if output.dtype()
                        == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype() =>
                {
                    let output = output
                        .as_typed_mut::<preset_scalar!($owned)>()
                        .expect("the dtype guard selects this arm");
                    let mut output = output.as_view_mut();
                    crate::structural::typed_copy_view_into(
                        input,
                        &mut output,
                        "cpu layout materialization",
                    )?;
                    return Ok(CpuProviderOutcome::Executed);
                }
                (TensorRead::Tensor(input), TensorWrite::View(TensorViewMut::$view(output)))
                    if input.dtype()
                        == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype() =>
                {
                    let input = input
                        .as_typed::<preset_scalar!($owned)>()
                        .expect("the dtype guard selects this arm");
                    let input = input.as_view();
                    crate::structural::typed_copy_view_into(
                        &input,
                        output,
                        "cpu layout materialization",
                    )?;
                    return Ok(CpuProviderOutcome::Executed);
                }
                (
                    TensorRead::View(TensorView::$view(input)),
                    TensorWrite::View(TensorViewMut::$view(output)),
                ) => {
                    crate::structural::typed_copy_view_into(
                        input,
                        output,
                        "cpu layout materialization",
                    )?;
                    return Ok(CpuProviderOutcome::Executed);
                }
                _ => {}
            }
        };
    }
    dispatch!(F32, F32);
    dispatch!(F64, F64);
    dispatch!(I32, I32);
    dispatch!(I64, I64);
    dispatch!(Bool, Bool);
    dispatch!(C32, C32);
    dispatch!(C64, C64);
    Ok(CpuProviderOutcome::Unsupported(
        CpuProviderUnsupported::DType(input.dtype()),
    ))
}

/// Full-overwrite layout transform into an uninitialized compact column-major
/// destination. Only [`CpuLayoutTransformIntent::CanonicalColumnMajor`] is
/// implemented; the destination must be exactly `element_count * size_of::<T>()`
/// bytes of the input dtype `T`, aligned for `T`.
fn materialize_strided_layout_into_uninit(
    input: &TensorRead<'_>,
    intent: CpuLayoutTransformIntent,
    conjugate: bool,
    output_bytes: &mut [MaybeUninit<u8>],
) -> tenferro_tensor::Result<CpuProviderOutcome> {
    if intent != CpuLayoutTransformIntent::CanonicalColumnMajor {
        return Ok(CpuProviderOutcome::Unsupported(
            CpuProviderUnsupported::Layout(CpuOperand::Output),
        ));
    }
    macro_rules! dispatch {
        ($owned:ident, $view:ident) => {
            match input {
                TensorRead::Tensor(input)
                    if input.dtype()
                        == <preset_scalar!($owned) as tenferro_tensor::TensorScalar>::dtype() =>
                {
                    let input = input
                        .as_typed::<preset_scalar!($owned)>()
                        .expect("the dtype guard selects this arm");
                    let input = input.as_view();
                    crate::structural::typed_copy_into_uninit(
                        &input,
                        conjugate,
                        output_bytes,
                        "cpu layout materialization",
                    )?;
                    return Ok(CpuProviderOutcome::Executed);
                }
                TensorRead::View(TensorView::$view(input)) => {
                    crate::structural::typed_copy_into_uninit(
                        input,
                        conjugate,
                        output_bytes,
                        "cpu layout materialization",
                    )?;
                    return Ok(CpuProviderOutcome::Executed);
                }
                _ => {}
            }
        };
    }
    dispatch!(F32, F32);
    dispatch!(F64, F64);
    dispatch!(C32, C32);
    dispatch!(C64, C64);
    Ok(CpuProviderOutcome::Unsupported(
        CpuProviderUnsupported::DType(input.dtype()),
    ))
}

pub(crate) fn builtin_gemm_provider(kind: CpuBackendKind) -> Arc<dyn CpuGemmProvider> {
    match kind {
        CpuBackendKind::Faer => Arc::new(FaerGemmProvider),
        CpuBackendKind::Blas => Arc::new(BlasGemmProvider),
    }
}

pub(crate) fn builtin_layout_provider() -> Arc<dyn CpuLayoutTransformProvider> {
    Arc::new(StridedLayoutTransformProvider)
}

#[cfg(test)]
pub(crate) mod tests;