tensor4all_tensorbackend/

tenferro_bridge.rs

//! Bridge helpers between tensor4all storage snapshots and tenferro tensors.

use std::cell::RefCell;
use std::cmp::Reverse;
use std::collections::{HashMap, HashSet};
use std::env;
use std::sync::Arc;
use std::time::{Duration, Instant};

use anyhow::{anyhow, ensure, Result};
use num_complex::{Complex32, Complex64};
use omeco::ScoreFunction;
use tenferro::{
    program::CoreSemanticOp, CompiledGraph, DType, GraphCompiler, Tensor as NativeTensor,
    TensorRead, TensorScalar, TensorSessionOpsExt, TraceContext,
};
use tenferro_einsum::{
    ConcreteEinsumPlan, ContractionOptimizerOptions, ContractionTree, EinsumSubscripts, Subscripts,
    TraceContextEinsumExt,
};
use tenferro_linalg::TensorLinalgExt;

use crate::any_scalar::promote_scalar_native;
pub(crate) use crate::einsum_ids::{
    build_binary_einsum_ids, checked_native_einsum_labels, common_dtype, convert_native_tensor_in,
};
/// Error returned by the storage/tensor bridge helpers.
///
/// Wraps the underlying tensor-element or backend diagnostic, preserving its
/// source chain.
///
/// # Remedies
/// - Dtype mismatch: extract with the element type matching the native tensor
///   dtype, or promote explicitly before the call.
/// - Shape mismatch: validate the native tensor shape against the payload
///   contract.
/// - Backend failure: the wrapped source chain identifies the failing stage.
#[derive(Debug, thiserror::Error)]
#[error("native tensor bridge operation failed: {source}")]
pub struct BridgeError {
    /// Original tensor-element or backend diagnostic.
    #[source]
    pub source: anyhow::Error,
}

impl From<anyhow::Error> for BridgeError {
    fn from(source: anyhow::Error) -> Self {
        Self { source }
    }
}

fn native_tensor_from_vec<T: TensorScalar>(
    shape: Vec<usize>,
    values: Vec<T>,
) -> std::result::Result<NativeTensor, BridgeError> {
    NativeTensor::from_vec_col_major(shape, values)
        .map_err(|source| BridgeError::from(anyhow::Error::new(source)))
}

use crate::context::{
    default_context, default_engine_buffer_pool_stats, reset_default_engine,
    reset_default_engine_buffer_pool, with_default_graph_compiler, with_default_session,
};
use crate::memory::release_process_allocator_cached_memory;
use crate::storage::{storage_dtype_dispatch, Storage, StorageScalar};
use crate::tensor_element::TensorElement;
use crate::BackendScalar;

/// Read-only native tensor input that can either borrow external payload data
/// or own a temporary materialized tensor.
// Both tenferro handle variants are large; boxing the slightly larger one would
// add an allocation to the eager einsum path for little enum-size reduction.
#[allow(clippy::large_enum_variant)]
pub enum NativeTensorReadInput<'a> {
    /// Borrowed read-only tensor input.
    Borrowed(TensorRead<'a>),
    /// Owned temporary tensor input.
    Owned(NativeTensor),
}

impl<'a> NativeTensorReadInput<'a> {
    /// Return this input as a read-only tenferro tensor input.
    pub fn as_read(&'a self) -> TensorRead<'a> {
        match self {
            Self::Borrowed(read) => read.clone(),
            Self::Owned(tensor) => TensorRead::from_tensor(tensor),
        }
    }

    /// Return the scalar dtype of this input.
    pub fn dtype(&self) -> DType {
        match self {
            Self::Borrowed(read) => read.dtype(),
            Self::Owned(tensor) => tensor.dtype(),
        }
    }

    /// Return the tensor shape of this input.
    pub fn shape(&self) -> &[usize] {
        match self {
            Self::Borrowed(read) => read.shape(),
            Self::Owned(tensor) => tensor.shape(),
        }
    }
}

#[cfg(test)]
use std::cell::Cell;

#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
enum NativeEinsumPath {
    Owned,
    Borrowed,
    BorrowedWithConversions,
}

#[derive(Debug, Clone, Hash, PartialEq, Eq)]
struct NativeOperandSignature {
    shape: Vec<usize>,
    ids: Vec<u32>,
    dtype: DType,
}

#[derive(Debug, Clone, Hash, PartialEq, Eq)]
struct NativeEinsumSignature {
    path: NativeEinsumPath,
    operands: Vec<NativeOperandSignature>,
    output_ids: Vec<u32>,
}

#[derive(Debug, Default, Clone)]
struct NativeEinsumProfileEntry {
    calls: usize,
    total_time: Duration,
}

/// Entries kept before the cache is cleared. Real workloads cycle through
/// hundreds to a few thousand distinct contraction signatures (#891: a
/// 400-shape QTT cycle re-prepared every plan at 256), so the bound sits above
/// that working set while still capping the per-thread cache.
const CONCRETE_EINSUM_PLAN_CACHE_CAPACITY: usize = 4096;

thread_local! {
    static NATIVE_EINSUM_PROFILE_STATE: RefCell<HashMap<NativeEinsumSignature, NativeEinsumProfileEntry>> =
        RefCell::new(HashMap::new());
    static NATIVE_EINSUM_TRACE_STATE: RefCell<HashSet<NativeEinsumSignature>> =
        RefCell::new(HashSet::new());
    static CONCRETE_EINSUM_PLAN_CACHE: RefCell<HashMap<NativeEinsumSignature, Arc<ConcreteEinsumPlan>>> =
        RefCell::new(HashMap::new());
}

#[cfg(test)]
thread_local! {
    static FORCE_NATIVE_EINSUM_PROFILE: Cell<bool> = const { Cell::new(false) };
}

fn native_einsum_profile_enabled() -> bool {
    #[cfg(test)]
    if FORCE_NATIVE_EINSUM_PROFILE.with(Cell::get) {
        return true;
    }
    env::var("T4A_PROFILE_NATIVE_EINSUM").is_ok()
}

fn native_einsum_path_trace_enabled() -> bool {
    env::var("T4A_TRACE_NATIVE_EINSUM_PATHS").is_ok()
}

fn native_einsum_path_trace_min_bytes() -> usize {
    env::var("T4A_TRACE_NATIVE_EINSUM_MIN_BYTES")
        .ok()
        .and_then(|value| value.parse().ok())
        .unwrap_or(0)
}

fn native_einsum_path_trace_max_signatures() -> usize {
    env::var("T4A_TRACE_NATIVE_EINSUM_MAX_SIGNATURES")
        .ok()
        .and_then(|value| value.parse().ok())
        .unwrap_or(64)
}

fn native_einsum_pool_trace_enabled() -> bool {
    env::var("T4A_TRACE_NATIVE_EINSUM_POOL").is_ok()
}

fn native_einsum_pool_trace_min_output_bytes() -> usize {
    env::var("T4A_TRACE_NATIVE_EINSUM_POOL_MIN_OUTPUT_BYTES")
        .ok()
        .and_then(|value| value.parse().ok())
        .unwrap_or(0)
}

fn native_einsum_pool_trace_min_retained_bytes() -> usize {
    env::var("T4A_TRACE_NATIVE_EINSUM_POOL_MIN_RETAINED_BYTES")
        .ok()
        .and_then(|value| value.parse().ok())
        .unwrap_or(0)
}

fn reset_native_einsum_engine_after_call() -> bool {
    env::var("T4A_RESET_NATIVE_EINSUM_ENGINE_AFTER_CALL").is_ok()
}

fn reset_native_einsum_buffer_pool_after_call() -> bool {
    env::var("T4A_RESET_NATIVE_EINSUM_BUFFER_POOL_AFTER_CALL").is_ok()
}

fn release_allocator_after_native_einsum_call() -> bool {
    env::var("T4A_RELEASE_ALLOCATOR_AFTER_NATIVE_EINSUM_CALL").is_ok()
}

#[cfg(test)]
pub(crate) fn set_native_einsum_profile_enabled_for_tests(enabled: bool) {
    FORCE_NATIVE_EINSUM_PROFILE.with(|slot| slot.set(enabled));
}

fn native_einsum_signature(
    path: NativeEinsumPath,
    operands: &[(&NativeTensor, &[u32])],
    output_ids: &[u32],
) -> NativeEinsumSignature {
    NativeEinsumSignature {
        path,
        operands: operands
            .iter()
            .map(|(tensor, ids)| NativeOperandSignature {
                shape: tensor.shape().to_vec(),
                ids: ids.to_vec(),
                dtype: tensor.dtype(),
            })
            .collect(),
        output_ids: output_ids.to_vec(),
    }
}

fn concrete_einsum_dtype_supported(dtype: DType) -> bool {
    matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64)
}

fn concrete_einsum_signature(
    inputs: &[TensorRead<'_>],
    subscripts: &Subscripts,
) -> NativeEinsumSignature {
    // ConcreteEinsumPlan owns only dtype, shape, labels, and its contraction
    // tree. Per-call TensorRead strides are validated and consumed at execute.
    NativeEinsumSignature {
        path: NativeEinsumPath::Borrowed,
        operands: inputs
            .iter()
            .zip(subscripts.inputs.iter())
            .map(|(tensor, ids)| NativeOperandSignature {
                shape: tensor.shape().to_vec(),
                ids: ids.clone(),
                dtype: tensor.dtype(),
            })
            .collect(),
        output_ids: subscripts.output.clone(),
    }
}

fn cached_concrete_einsum_plan(
    inputs: &[TensorRead<'_>],
    subscripts: &Subscripts,
    einsum_subscripts: &EinsumSubscripts,
) -> Result<Arc<ConcreteEinsumPlan>> {
    let signature = concrete_einsum_signature(inputs, subscripts);
    if let Some(plan) =
        CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow().get(&signature).cloned())
    {
        return Ok(plan);
    }

    let plan = Arc::new(
        ConcreteEinsumPlan::prepare_read_subscripts(inputs, einsum_subscripts)
            .map_err(|error| anyhow!("native einsum session plan preparation failed: {error}"))?,
    );
    CONCRETE_EINSUM_PLAN_CACHE.with(|cache| {
        let mut cache = cache.borrow_mut();
        // ponytail: bounded clear-on-full cache; use an LRU only if shape churn is measured.
        if cache.len() >= CONCRETE_EINSUM_PLAN_CACHE_CAPACITY {
            cache.clear();
        }
        cache.insert(signature, Arc::clone(&plan));
    });
    Ok(plan)
}

#[cfg(test)]
fn reset_concrete_einsum_plan_cache() {
    CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow_mut().clear());
}

#[cfg(test)]
fn concrete_einsum_plan_cache_len() -> usize {
    CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow().len())
}

fn record_native_einsum_profile(
    path: NativeEinsumPath,
    operands: &[(&NativeTensor, &[u32])],
    output_ids: &[u32],
    elapsed: Duration,
) {
    if !native_einsum_profile_enabled() {
        return;
    }
    let signature = native_einsum_signature(path, operands, output_ids);
    NATIVE_EINSUM_PROFILE_STATE.with(|state| {
        let mut state = state.borrow_mut();
        let entry = state.entry(signature).or_default();
        entry.calls += 1;
        entry.total_time += elapsed;
    });
}

fn native_slice<'a, T: TensorScalar>(
    tensor: &'a NativeTensor,
    label: &'static str,
) -> Result<&'a [T]> {
    tensor
        .as_slice::<T>()
        .map_err(|error| anyhow!("{label}: {error}"))
}

fn dtype_size_bytes(dtype: DType) -> usize {
    match dtype {
        DType::F32 => 4,
        DType::F64 => 8,
        DType::C32 => 8,
        DType::C64 => 16,
        DType::I32 => 4,
        DType::I64 => 8,
        DType::Bool => 1,
        // External scalar kinds have no element width this crate can know. They
        // are rejected at every entry point that can materialize storage, so this
        // accounting arm is unreachable for a tensor that passed those checks.
        DType::External(_) => 0,
    }
}

fn native_tensor_bytes(tensor: &NativeTensor) -> usize {
    tensor
        .shape()
        .iter()
        .copied()
        .fold(1usize, usize::saturating_mul)
        .saturating_mul(dtype_size_bytes(tensor.dtype()))
}

fn format_label(label: u32) -> String {
    char::from_u32(label).map_or_else(|| label.to_string(), |label| label.to_string())
}

fn format_labels(labels: &[u32]) -> String {
    if labels.is_empty() {
        "scalar".to_string()
    } else {
        labels
            .iter()
            .map(|&label| format_label(label))
            .collect::<Vec<_>>()
            .join("")
    }
}

fn label_dims(subscripts: &Subscripts, shapes: &[Vec<usize>]) -> Result<HashMap<u32, usize>> {
    let mut dims = HashMap::new();
    for (labels, shape) in subscripts.inputs.iter().zip(shapes.iter()) {
        ensure!(
            labels.len() == shape.len(),
            "einsum labels {:?} do not match shape {:?}",
            labels,
            shape
        );
        for (&label, &dim) in labels.iter().zip(shape.iter()) {
            if let Some(previous) = dims.insert(label, dim) {
                ensure!(
                    previous == dim,
                    "inconsistent dimension for einsum label {}: {} vs {}",
                    format_label(label),
                    previous,
                    dim
                );
            }
        }
    }
    Ok(dims)
}

fn labels_size(labels: &[u32], dims: &HashMap<u32, usize>) -> usize {
    labels.iter().fold(1usize, |size, label| {
        size.saturating_mul(dims.get(label).copied().unwrap_or(1))
    })
}

fn union_labels(lhs: &[u32], rhs: &[u32]) -> Vec<u32> {
    let mut seen = HashSet::new();
    let mut labels = Vec::new();
    for &label in lhs.iter().chain(rhs.iter()) {
        if seen.insert(label) {
            labels.push(label);
        }
    }
    labels
}

#[derive(Debug)]
struct NativeEinsumPlanReport {
    lines: Vec<String>,
    peak_intermediate_bytes: usize,
}

fn time_optimized_contraction_options() -> ContractionOptimizerOptions {
    ContractionOptimizerOptions {
        score: ScoreFunction::time_optimized(),
        ..ContractionOptimizerOptions::default()
    }
}

fn native_einsum_plan_report_with_options(
    signature: &NativeEinsumSignature,
    optimizer_name: &'static str,
    options: &ContractionOptimizerOptions,
) -> Result<NativeEinsumPlanReport> {
    let input_ids = signature
        .operands
        .iter()
        .map(|operand| operand.ids.as_slice())
        .collect::<Vec<_>>();
    let subscripts_string = build_einsum_subscripts(&input_ids, &signature.output_ids)?;
    let subscripts = Subscripts {
        inputs: input_ids.iter().map(|ids| ids.to_vec()).collect(),
        output: signature.output_ids.clone(),
    };
    let shapes = signature
        .operands
        .iter()
        .map(|operand| operand.shape.clone())
        .collect::<Vec<_>>();
    let shape_refs = shapes.iter().map(Vec::as_slice).collect::<Vec<_>>();
    let tree = ContractionTree::optimize_with_options(&subscripts, &shape_refs, options)
        .map_err(|e| anyhow!("failed to optimize native einsum path: {e}"))?;
    let dims = label_dims(&subscripts, &shapes)?;
    let dtype = signature
        .operands
        .first()
        .map(|operand| operand.dtype)
        .unwrap_or(DType::F64);
    let dtype_size = dtype_size_bytes(dtype);

    let mut lines = Vec::new();
    lines.push(format!(
        "optimizer={optimizer_name} subscripts={subscripts_string} dtype={dtype:?} steps={}",
        tree.step_count()
    ));
    let mut peak_intermediate_elems = 1usize;
    for step in 0..tree.step_count() {
        let Some((left, right)) = tree.step_pair(step) else {
            continue;
        };
        let Some((lhs, rhs, out)) = tree.step_subscripts(step) else {
            continue;
        };
        let lhs_elems = labels_size(lhs, &dims);
        let rhs_elems = labels_size(rhs, &dims);
        let out_elems = labels_size(out, &dims);
        let flop_index_elems = labels_size(&union_labels(lhs, rhs), &dims);
        peak_intermediate_elems = peak_intermediate_elems.max(out_elems);
        lines.push(format!(
            "  step {step:02}: pair=({left},{right}) {}[{}] x {}[{}] -> {}[{}]  flop_index={}  intermediate={} elems ({:.3} MiB)",
            format_labels(lhs),
            lhs_elems,
            format_labels(rhs),
            rhs_elems,
            format_labels(out),
            out_elems,
            flop_index_elems,
            out_elems,
            out_elems as f64 * dtype_size as f64 / (1024.0 * 1024.0),
        ));
    }
    let peak_intermediate_bytes = peak_intermediate_elems.saturating_mul(dtype_size);
    lines.push(format!(
        "  peak_intermediate={} elems ({:.3} MiB)",
        peak_intermediate_elems,
        peak_intermediate_bytes as f64 / (1024.0 * 1024.0)
    ));

    Ok(NativeEinsumPlanReport {
        lines,
        peak_intermediate_bytes,
    })
}

fn native_einsum_time_optimized_plan_report(
    signature: &NativeEinsumSignature,
) -> Result<NativeEinsumPlanReport> {
    native_einsum_plan_report_with_options(
        signature,
        "time_optimized",
        &time_optimized_contraction_options(),
    )
}

fn native_einsum_balanced_plan_report(
    signature: &NativeEinsumSignature,
) -> Result<NativeEinsumPlanReport> {
    native_einsum_plan_report_with_options(
        signature,
        "balanced_default",
        &ContractionOptimizerOptions::default(),
    )
}

fn maybe_trace_native_einsum_path(
    path: NativeEinsumPath,
    operands: &[(&NativeTensor, &[u32])],
    output_ids: &[u32],
) {
    if !native_einsum_path_trace_enabled() {
        return;
    }
    let signature = native_einsum_signature(path, operands, output_ids);
    let report = match native_einsum_time_optimized_plan_report(&signature) {
        Ok(report) if report.peak_intermediate_bytes >= native_einsum_path_trace_min_bytes() => {
            report
        }
        Ok(_) => return,
        Err(err) => {
            eprintln!("native_einsum path trace failed: {err:#}");
            return;
        }
    };

    let max_signatures = native_einsum_path_trace_max_signatures();
    let should_trace = NATIVE_EINSUM_TRACE_STATE.with(|state| {
        let mut state = state.borrow_mut();
        if state.len() >= max_signatures || state.contains(&signature) {
            false
        } else {
            state.insert(signature.clone());
            true
        }
    });
    if !should_trace {
        return;
    }

    eprintln!("=== native_einsum Path Trace ===");
    eprintln!(
        "path={:?} output_ids={:?}",
        signature.path, signature.output_ids
    );
    for operand in &signature.operands {
        eprintln!(
            "  operand shape={:?} ids={:?} dtype={:?}",
            operand.shape, operand.ids, operand.dtype
        );
    }
    for line in report.lines {
        eprintln!("{line}");
    }
    if env::var("T4A_TRACE_NATIVE_EINSUM_COMPARE_BALANCED").is_ok() {
        match native_einsum_balanced_plan_report(&signature) {
            Ok(balanced) => {
                for line in balanced.lines {
                    eprintln!("{line}");
                }
            }
            Err(err) => eprintln!("balanced native_einsum path trace failed: {err:#}"),
        }
    }
}

/// Reset the aggregated native einsum profile.
pub fn reset_native_einsum_profile() {
    NATIVE_EINSUM_PROFILE_STATE.with(|state| state.borrow_mut().clear());
    NATIVE_EINSUM_TRACE_STATE.with(|state| state.borrow_mut().clear());
}

/// Print and clear the aggregated native einsum profile.
pub fn print_and_reset_native_einsum_profile() {
    if !native_einsum_profile_enabled() {
        return;
    }
    NATIVE_EINSUM_PROFILE_STATE.with(|state| {
        let mut entries: Vec<_> = state
            .borrow()
            .iter()
            .map(|(k, v)| (k.clone(), v.clone()))
            .collect();
        state.borrow_mut().clear();
        entries.sort_by_key(|(_, entry)| Reverse(entry.total_time));

        eprintln!("=== native_einsum Profile ===");
        for (idx, (signature, entry)) in entries.into_iter().take(20).enumerate() {
            eprintln!(
                "#{idx:02} path={:?} calls={} total={:.3}s per_call={:.3}us output_ids={:?}",
                signature.path,
                entry.calls,
                entry.total_time.as_secs_f64(),
                entry.total_time.as_secs_f64() * 1e6 / entry.calls as f64,
                signature.output_ids,
            );
            for operand in &signature.operands {
                eprintln!(
                    "     shape={:?} ids={:?} dtype={:?}",
                    operand.shape, operand.ids, operand.dtype
                );
            }
            match native_einsum_time_optimized_plan_report(&signature) {
                Ok(report) => {
                    for line in report.lines {
                        eprintln!("     {line}");
                    }
                }
                Err(err) => eprintln!("     path report failed: {err:#}"),
            }
        }
    });
}

fn convert_tensor(tensor: &NativeTensor, to: DType) -> Result<NativeTensor> {
    with_default_session(|session| convert_native_tensor_in(session, tensor, to))
        .map_err(|e| anyhow::Error::new(e).context(format!("tensor conversion to {to:?} failed")))
}

fn ids_to_subscript(ids: &[u32]) -> Result<String> {
    const LETTERS: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
    let mut out = String::with_capacity(ids.len());
    for &id in ids {
        let idx = usize::try_from(id).unwrap_or(usize::MAX);
        let letter = LETTERS
            .get(idx)
            .ok_or_else(|| anyhow!("einsum label {id} exceeds supported label range"))?;
        out.push(char::from(*letter));
    }
    Ok(out)
}

fn build_einsum_subscripts(operands: &[&[u32]], output_ids: &[u32]) -> Result<String> {
    let inputs = operands
        .iter()
        .map(|ids| ids_to_subscript(ids))
        .collect::<Result<Vec<_>>>()?;
    Ok(format!(
        "{}->{}",
        inputs.join(","),
        ids_to_subscript(output_ids)?
    ))
}

fn compile_native_einsum_program(
    compiler: &mut GraphCompiler,
    input_specs: impl IntoIterator<Item = tenferro::program::ProgramInputSpec>,
    subscripts: &EinsumSubscripts,
) -> Result<tenferro::CompiledGraph> {
    let input_specs = input_specs.into_iter().collect::<Vec<_>>();
    let target = common_dtype(
        input_specs
            .iter()
            .map(|spec| spec.metadata().dtype())
            .collect::<Vec<_>>()
            .as_slice(),
    );
    let mut trace = TraceContext::new();
    let trace_inputs = input_specs
        .into_iter()
        .map(|spec| {
            let dtype = spec.metadata().dtype();
            let input = trace
                .input(spec)
                .map_err(|e| anyhow!("native einsum input tracing failed: {e}"))?;
            if dtype == target {
                return Ok(input);
            }
            trace
                .add_op(
                    CoreSemanticOp::Convert {
                        from: dtype,
                        to: target,
                    },
                    &[input],
                )
                .map_err(|e| anyhow!("native einsum promotion tracing failed: {e}"))?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("native einsum promotion returned no output"))
        })
        .collect::<Result<Vec<_>>>()?;
    let output = trace
        .einsum_subscripts(&trace_inputs, subscripts)
        .map_err(|e| anyhow!("native einsum tracing failed: {e}"))?;
    let graph = trace
        .finish(&[output])
        .map_err(|e| anyhow!("native einsum graph finalization failed: {e}"))?;
    compiler
        .compile_traced_graph(&graph)
        .map_err(|e| anyhow!("native einsum graph compilation failed: {e}"))
}

fn run_cached_native_einsum(
    subscripts: &EinsumSubscripts,
    build: impl FnOnce(&mut GraphCompiler) -> Result<CompiledGraph>,
    execute: impl FnOnce(&crate::context::CpuExecutionContext, &CompiledGraph) -> Result<NativeTensor>,
) -> Result<NativeTensor> {
    let trace_pool = native_einsum_pool_trace_enabled();
    let pool_before = trace_pool
        .then(default_engine_buffer_pool_stats)
        .transpose()?;
    // Compile under the compiler lock only; the graph then runs under the
    // context's engine lock. Holding both would make a compile wait for a graph
    // that is running.
    let program = with_default_graph_compiler(build)?;
    let result = execute(default_context(), &program)?;
    if trace_pool {
        let pool_after = default_engine_buffer_pool_stats()?;
        let output_bytes = native_tensor_bytes(&result);
        let retained_threshold = native_einsum_pool_trace_min_retained_bytes();
        if pool_after != pool_before.unwrap_or_default()
            && pool_after.capacity_bytes >= retained_threshold
            || output_bytes >= native_einsum_pool_trace_min_output_bytes()
        {
            let before = pool_before.unwrap_or_default();
            eprintln!(
                "native_einsum pool subscripts={subscripts:?} before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB output_shape={:?} output_bytes={:.3} MiB",
                before.buffers,
                before.capacity_bytes as f64 / (1024.0 * 1024.0),
                pool_after.buffers,
                pool_after.capacity_bytes as f64 / (1024.0 * 1024.0),
                result.shape(),
                output_bytes as f64 / (1024.0 * 1024.0),
            );
        }
    }
    if reset_native_einsum_engine_after_call() {
        let before_reset = trace_pool
            .then(default_engine_buffer_pool_stats)
            .transpose()?;
        reset_default_engine()?;
        if trace_pool
            && before_reset.unwrap_or_default().capacity_bytes
                >= native_einsum_pool_trace_min_retained_bytes()
        {
            let before = before_reset.unwrap_or_default();
            let after = default_engine_buffer_pool_stats()?;
            eprintln!(
                "native_einsum engine_reset before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB",
                before.buffers,
                before.capacity_bytes as f64 / (1024.0 * 1024.0),
                after.buffers,
                after.capacity_bytes as f64 / (1024.0 * 1024.0),
            );
        }
    } else if reset_native_einsum_buffer_pool_after_call() {
        let before_clear = trace_pool
            .then(default_engine_buffer_pool_stats)
            .transpose()?;
        reset_default_engine_buffer_pool()?;
        if trace_pool
            && before_clear.unwrap_or_default().capacity_bytes
                >= native_einsum_pool_trace_min_retained_bytes()
        {
            let before = before_clear.unwrap_or_default();
            let after = default_engine_buffer_pool_stats()?;
            eprintln!(
                "native_einsum pool_reset before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB",
                before.buffers,
                before.capacity_bytes as f64 / (1024.0 * 1024.0),
                after.buffers,
                after.capacity_bytes as f64 / (1024.0 * 1024.0),
            );
        }
    }
    if release_allocator_after_native_einsum_call() {
        let report = release_process_allocator_cached_memory();
        if trace_pool && (report.released_bytes.unwrap_or(0) > 0 || report.success == Some(true)) {
            eprintln!(
                "native_einsum allocator_pressure_relief supported={} released_bytes={:?} success={:?}",
                report.supported,
                report.released_bytes,
                report.success,
            );
        }
    }
    Ok(result)
}

fn cached_einsum_native_tensors(
    inputs: &[&NativeTensor],
    subscripts: &EinsumSubscripts,
) -> Result<NativeTensor> {
    run_cached_native_einsum(
        subscripts,
        |compiler| {
            compile_native_einsum_program(
                compiler,
                inputs.iter().map(|tensor| {
                    tenferro::program::ProgramInputSpec::new(
                        tensor.dtype(),
                        tensor.shape().iter().copied().map(Into::into),
                    )
                }),
                subscripts,
            )
        },
        |context, program| single_output(context.run_graph(program, inputs)),
    )
}

fn cached_einsum_native_reads(
    inputs: &[TensorRead<'_>],
    subscripts: &Subscripts,
) -> Result<NativeTensor> {
    let einsum_subscripts = EinsumSubscripts::from(subscripts);
    if inputs.first().is_some_and(|first| {
        concrete_einsum_dtype_supported(first.dtype())
            && inputs.iter().all(|input| input.dtype() == first.dtype())
    }) {
        let plan = cached_concrete_einsum_plan(inputs, subscripts, &einsum_subscripts)?;
        // The plan reports `tenferro_einsum::Error`, so the session-entry
        // rejection is mapped here rather than through the tenferro-tensor-typed
        // session helper.
        return default_context()
            .with_session(|engine| plan.execute_read(inputs, engine))
            .map_err(|entry| {
                anyhow::Error::new(entry).context("native einsum session entry failed")
            })?
            .map_err(|error| anyhow!("native einsum session execution failed: {error}"));
    }

    run_cached_native_einsum(
        &einsum_subscripts,
        |compiler| {
            compile_native_einsum_program(
                compiler,
                inputs.iter().map(|tensor| {
                    tenferro::program::ProgramInputSpec::new(
                        tensor.dtype(),
                        tensor.shape().iter().copied().map(Into::into),
                    )
                }),
                &einsum_subscripts,
            )
        },
        |context, program| single_output(context.run_graph_reads(program, inputs)),
    )
    .map_err(|e| anyhow!("native read einsum failed: {e}"))
}

/// Reduce a graph's output list to the single tensor an einsum produces.
fn single_output(
    outputs: std::result::Result<Vec<NativeTensor>, crate::context::CpuExecutionContextError>,
) -> Result<NativeTensor> {
    let mut outputs = outputs.map_err(|e| anyhow!("native einsum failed: {e}"))?;
    if outputs.len() != 1 {
        return Err(anyhow!(
            "native einsum returned {} outputs instead of one",
            outputs.len()
        ));
    }
    outputs
        .pop()
        .ok_or_else(|| anyhow!("native einsum returned no output"))
}

/// Build a dense native tensor from column-major data.
/// # Errors
///
/// Returns an error when the shape product would overflow `usize`,
/// when `data.len()` does not match the expected shape, or when the backend
/// rejects the tensor.
pub fn dense_native_tensor_from_col_major<T: TensorElement>(
    data: &[T],
    logical_dims: &[usize],
) -> Result<NativeTensor> {
    T::dense_native_tensor_from_col_major(data, logical_dims)
}

/// Build a dense native tensor by moving an owned column-major payload.
///
/// # Arguments
///
/// * `data` - Owned column-major values whose length equals the product of
///   `logical_dims`.
/// * `logical_dims` - Logical tensor dimensions in column-major axis order.
///
/// # Returns
///
/// A native dense tensor that owns the supplied payload without cloning it.
///
/// # Errors
///
/// Returns an error when the shape product would overflow `usize`,
/// when `data.len()` does not match the expected shape, or when the backend
/// rejects the tensor.
pub fn dense_native_tensor_from_col_major_owned<T: TensorElement>(
    data: Vec<T>,
    logical_dims: &[usize],
) -> Result<NativeTensor> {
    T::dense_native_tensor_from_col_major_owned(data, logical_dims)
}

/// Borrow column-major data as a read-only native operand without copying.
///
/// The returned input owns nothing: it is a [`TensorRead`] view over `data`, so
/// a shared operand (a frame matrix, a weight matrix) reaches an einsum as a
/// borrowed read instead of a full-size copy. Use it with
/// [`einsum_native_tensor_reads`] on the compatibility frontend, or
/// [`NativeTensorReadInput::as_read`] to feed an explicit session's read route.
///
/// # Arguments
///
/// * `data` - Borrowed column-major values whose length equals the product of
///   `logical_dims`.
/// * `logical_dims` - Logical tensor dimensions in column-major axis order.
///
/// # Returns
///
/// A borrowed read input that stays valid for `data`'s lifetime.
///
/// # Errors
///
/// Returns an error when the shape product overflows `usize`, when `data.len()`
/// does not match the expected shape, or when the view is rejected as invalid.
///
/// # Examples
///
/// ```
/// use tensor4all_tensorbackend::{
///     dense_native_tensor_read_input_from_col_major, einsum_native_tensor_reads,
/// };
///
/// let lhs = [1.0_f64, 2.0, 3.0, 4.0];
/// let rhs = [5.0_f64, 6.0];
/// let lhs = dense_native_tensor_read_input_from_col_major(&lhs, &[2, 2])?;
/// let rhs = dense_native_tensor_read_input_from_col_major(&rhs, &[2, 1])?;
/// let product = einsum_native_tensor_reads(
///     &[(&lhs, &[0, 1]), (&rhs, &[1, 2])],
///     &[0, 2],
/// )?;
/// assert_eq!(product.as_slice::<f64>()?, &[23.0, 34.0]);
/// # Ok::<(), Box<dyn std::error::Error>>(())
/// ```
pub fn dense_native_tensor_read_input_from_col_major<'a, T>(
    data: &'a [T],
    logical_dims: &[usize],
) -> Result<NativeTensorReadInput<'a>>
where
    T: TensorElement,
{
    let view = tenferro::TypedTensorView::from_col_major(logical_dims, data)
        .map_err(|error| anyhow!("native tensor view rejected: {error}"))?;
    Ok(NativeTensorReadInput::Borrowed(TensorRead::from_view(
        <T as TensorScalar>::tensor_view(view),
    )))
}

/// Build a dense native tensor whose logical values are diagonal.
/// # Errors
///
/// Returns an error when `logical_rank` is zero, when expanding the diagonal
/// would overflow `usize`, or when the backend rejects the resulting dense
/// tensor.
pub fn diag_native_tensor_from_col_major<T: TensorElement>(
    data: &[T],
    logical_rank: usize,
) -> Result<NativeTensor> {
    T::diag_native_tensor_from_col_major(data, logical_rank)
}

/// Convert storage to a dense native tensor.
///
/// The native tensor keeps the storage's own scalar dtype: a compact `f32` or
/// `Complex32` payload becomes an `f32`/`c32` native tensor instead of being
/// widened to `f64`/`Complex64`.
/// # Errors
///
/// Returns a [`BridgeError`] when materializing `storage` to the requested
/// dense shape fails (for example a shape mismatch) or when the
/// dense-to-native conversion fails.
pub fn storage_to_native_tensor(
    storage: &Storage,
    logical_dims: &[usize],
) -> std::result::Result<NativeTensor, BridgeError> {
    storage_dtype_dispatch!(storage.dtype(), Payload => {
        let values = <Payload as StorageScalar>::dense_values(storage, logical_dims)
            .map_err(|error| anyhow!("dense payload materialization failed: {error}"))?;
        <Payload as TensorElement>::dense_native_tensor_from_col_major_owned(
            values,
            logical_dims,
        )
        .map_err(BridgeError::from)
    },
    else Err(anyhow!("unsupported storage scalar type").into()))
}

/// Copy the compact payload as a native tensor of the stored dtype.
///
/// This visits only referenced payload entries and preserves the stored scalar
/// dtype, so it is the dtype-generic payload bridge for compact storage.
/// # Errors
///
/// Returns a [`BridgeError`] when the payload cannot be copied or the
/// backend rejects the native tensor.
pub fn storage_payload_native(storage: &Storage) -> std::result::Result<NativeTensor, BridgeError> {
    storage_dtype_dispatch!(storage.dtype(), Payload => {
        let values = <Payload as StorageScalar>::payload_vec(storage)
            .map_err(|error| anyhow!("compact payload copy failed: {error}"))?;
        native_tensor_from_vec(storage.payload_dims().to_vec(), values)
    },
    else Err(anyhow!("unsupported storage scalar type").into()))
}

/// Build structured storage from a native payload, preserving its dtype.
///
/// `output_payload_dims` and `output_axis_classes` describe the planned compact
/// layout. The compact payload keeps the native tensor's scalar dtype, so a
/// narrow contraction result stays narrow.
/// # Errors
///
/// Returns a [`BridgeError`] when the payload shape does not match
/// `output_payload_dims`, when its dtype has no compact storage representation,
/// or when the storage metadata or payload read fails.
pub fn storage_from_payload_native(
    payload: NativeTensor,
    output_payload_dims: &[usize],
    output_axis_classes: Vec<usize>,
) -> std::result::Result<Storage, BridgeError> {
    if payload.shape() != output_payload_dims {
        return Err(anyhow!(
            "payload shape {:?} does not match planned payload dims {:?}",
            payload.shape(),
            output_payload_dims
        )
        .into());
    }
    let strides = crate::storage::col_major_strides(output_payload_dims)?;
    storage_dtype_dispatch!(payload.dtype(), Payload => {
        let values = native_slice::<Payload>(&payload, "failed to read structured payload")?.to_vec();
        Storage::new_structured(
            values,
            output_payload_dims.to_vec(),
            strides,
            output_axis_classes,
        )
        .map_err(|error| anyhow!("structured payload storage construction failed: {error}").into())
    },
    else Err(anyhow!(
        "unsupported structured payload dtype {:?}; expected a floating or complex floating dtype",
        payload.dtype()
    )
    .into()))
}

/// Build a read-only native tensor input over the compact storage payload.
///
/// Contiguous payloads are borrowed without copying. Non-contiguous payloads
/// are materialized into an owned native tensor. Both paths preserve the stored
/// scalar dtype, so `f32`/`Complex32` payloads are served without widening.
/// # Errors
///
/// Returns a [`BridgeError`] when the storage dtype is not a floating or
/// complex floating dtype, when the contiguous payload view cannot be built, or
/// when the payload cannot be read into a native tensor.
pub fn storage_payload_native_read_input(
    storage: &Storage,
) -> std::result::Result<NativeTensorReadInput<'_>, BridgeError> {
    storage_dtype_dispatch!(storage.dtype(), Payload => {
        if let Some(view) = <Payload as StorageScalar>::payload_view(storage)
            .map_err(anyhow::Error::msg)?
        {
            let typed = tenferro::TypedTensorView::from_col_major(storage.payload_dims(), view)
                .map_err(|error| BridgeError::from(anyhow::Error::new(error)))?;
            return Ok(NativeTensorReadInput::Borrowed(TensorRead::from_view(
                <Payload as TensorScalar>::tensor_view(typed),
            )));
        }
        let values = <Payload as StorageScalar>::payload_vec(storage)
            .map_err(|error| anyhow!("compact payload copy failed: {error}"))?;
        native_tensor_from_vec(storage.payload_dims().to_vec(), values)
            .map(NativeTensorReadInput::Owned)
    },
    else Err(anyhow!("unsupported storage scalar type").into()))
}

/// Materialize a native tensor into dense storage.
/// # Errors
///
/// Returns a [`BridgeError`] when the tensor has an external scalar kind, when
/// reading its host slice fails, or when the dense storage materialization
/// fails.
pub fn native_tensor_primal_to_storage(
    tensor: &NativeTensor,
) -> std::result::Result<Storage, BridgeError> {
    match tensor.dtype() {
        DType::F32 => Storage::from_dense_col_major(
            native_slice::<f32>(tensor, "failed to read f32 native tensor")?.to_vec(),
            tensor.shape(),
        )
        .map_err(|e| {
            BridgeError::from(anyhow!(
                "native tensor snapshot materialization failed: {e}"
            ))
        }),
        DType::F64 => Storage::from_dense_col_major(
            native_slice::<f64>(tensor, "failed to read f64 native tensor")?.to_vec(),
            tensor.shape(),
        )
        .map_err(|e| {
            BridgeError::from(anyhow!(
                "native tensor snapshot materialization failed: {e}"
            ))
        }),
        DType::I32 => Storage::from_dense_col_major(
            native_slice::<i32>(tensor, "failed to read i32 native tensor")?
                .iter()
                .map(|&value| value as f64)
                .collect::<Vec<_>>(),
            tensor.shape(),
        )
        .map_err(|e| {
            BridgeError::from(anyhow!(
                "native tensor snapshot materialization failed: {e}"
            ))
        }),
        DType::I64 => Storage::from_dense_col_major(
            native_slice::<i64>(tensor, "failed to read i64 native tensor")?
                .iter()
                .map(|&value| value as f64)
                .collect::<Vec<_>>(),
            tensor.shape(),
        )
        .map_err(|e| {
            BridgeError::from(anyhow!(
                "native tensor snapshot materialization failed: {e}"
            ))
        }),
        DType::Bool => Storage::from_dense_col_major(
            native_slice::<bool>(tensor, "failed to read bool native tensor")?
                .iter()
                .map(|&value| if value { 1.0 } else { 0.0 })
                .collect::<Vec<_>>(),
            tensor.shape(),
        )
        .map_err(|e| {
            BridgeError::from(anyhow!(
                "native tensor snapshot materialization failed: {e}"
            ))
        }),
        DType::C32 => Storage::from_dense_col_major(
            native_slice::<Complex32>(tensor, "failed to read c32 native tensor")?.to_vec(),
            tensor.shape(),
        )
        .map_err(|e| {
            BridgeError::from(anyhow!(
                "native tensor snapshot materialization failed: {e}"
            ))
        }),
        DType::C64 => Storage::from_dense_col_major(
            native_slice::<Complex64>(tensor, "failed to read c64 native tensor")?.to_vec(),
            tensor.shape(),
        )
        .map_err(|e| {
            BridgeError::from(anyhow!(
                "native tensor snapshot materialization failed: {e}"
            ))
        }),
        DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
            tenferro_tensor::Error::unsupported(
                "dense storage",
                "external scalar kinds cannot be materialized into dense storage",
            ),
        ))),
    }
}

/// Materialize dense column-major values from a native tensor.
/// # Errors
///
/// Returns a [`BridgeError`] when `tensor`'s dtype does not match `T` or when
/// its host data cannot be read.
pub fn native_tensor_primal_to_dense_col_major<T: TensorElement>(
    tensor: &NativeTensor,
) -> std::result::Result<Vec<T>, BridgeError> {
    let target = <T as TensorScalar>::dtype();
    let tensor_is_real = matches!(
        tensor.dtype(),
        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
    );
    let target_is_real = matches!(
        target,
        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
    );
    if tensor_is_real != target_is_real {
        return Err(anyhow!(
            "expected {} native tensor, got dtype {:?}",
            if target_is_real { "real" } else { "complex" },
            tensor.dtype()
        )
        .into());
    }
    <T as TensorElement>::dense_values_from_native_col_major(tensor).map_err(BridgeError::from)
}

/// Materialize diagonal values from a native tensor, promoting to the
/// matching real (`f64`) or complex (`Complex64`) dtype.
///
/// Real native tensors (`f32`, `f64`, `i32`, `i64`, `bool`) are promoted to
/// `f64`; complex tensors (`c32`, `c64`) are promoted to `Complex64`. The
/// scalar type `T` selects the promoted target.
/// # Errors
///
/// Returns a [`BridgeError`] when the tensor's real/complex kind does not
/// match `T`, when conversion to `T` fails, or when the tensor is not
/// diagonal-compatible (rank 0 or unequal dimensions).
///
/// # Examples
/// ```
/// use tenferro::Tensor as NativeTensor;
/// use tensor4all_tensorbackend::native_tensor_primal_to_diag;
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let native = NativeTensor::from_vec_col_major(vec![3, 3], vec![1.0_f64, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 3.0])?;
/// let diag = native_tensor_primal_to_diag::<f64>(&native)?;
/// assert_eq!(diag, vec![1.0, 2.0, 3.0]);
/// # Ok(())
/// # }
/// ```
pub fn native_tensor_primal_to_diag<T: TensorElement>(
    tensor: &NativeTensor,
) -> std::result::Result<Vec<T>, BridgeError> {
    let promote_to = <T as TensorScalar>::dtype();
    let tensor_is_real = matches!(
        tensor.dtype(),
        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
    );
    let target_is_real = matches!(
        promote_to,
        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
    );
    if tensor_is_real != target_is_real {
        return Err(anyhow!(
            "expected {} native tensor, got dtype {:?}",
            if target_is_real { "real" } else { "complex" },
            tensor.dtype()
        )
        .into());
    }
    let promoted = convert_tensor(tensor, promote_to)?;
    <T as TensorElement>::diag_values_from_native_temp(&promoted).map_err(BridgeError::from)
}

/// Reshape a native tensor without changing its column-major linearization.
/// # Errors
///
/// Fails with a `CpuExecutionContextError` when the default context's engine is
/// already borrowed on this thread, or when the shape `logical_dims` and the
/// element count of `tensor` do not match.
pub fn reshape_col_major_native_tensor(
    tensor: &NativeTensor,
    logical_dims: &[usize],
) -> Result<NativeTensor> {
    with_default_session(|session| tensor.reshape(logical_dims, session))
        .map_err(|e| anyhow!("native reshape failed: {e}"))
}

/// Compute a QR decomposition on a native tensor.
/// # Errors
///
/// Returns a [`BridgeError`] when the context rejects the entry or when
/// the QR factorization fails.
pub fn qr_native_tensor(
    tensor: &NativeTensor,
) -> std::result::Result<(NativeTensor, NativeTensor), BridgeError> {
    let (q, r) = with_default_session(|session| tensor.qr(session))
        .map_err(|e| anyhow!("native QR failed: {e}"))?;
    Ok((q, r))
}

/// Compute an SVD on a native tensor.
/// # Errors
///
/// Fails with a `CpuExecutionContextError` when the default context's engine is
/// already borrowed on this thread, or when the backend SVD reports a
/// non-convergence failure.
pub fn svd_native_tensor(
    tensor: &NativeTensor,
) -> Result<(NativeTensor, NativeTensor, NativeTensor)> {
    let (u, s, vt) = with_default_session(|session| tensor.svd(session))
        .map_err(|e| anyhow!("native SVD failed: {e}"))?;
    Ok((u, s, vt))
}

/// Sum all elements of a native tensor, returning a dynamic scalar.
/// # Errors
///
/// Returns a [`BridgeError`] when the backend reduction fails, when the
/// reduced result cannot read back its host slice, or when the scalar cannot
/// be wrapped as a [`BackendScalar`].
pub fn sum_native_tensor(tensor: &NativeTensor) -> std::result::Result<BackendScalar, BridgeError> {
    let reduced = if tensor.shape().is_empty() {
        tensor
            .duplicate()
            .map_err(|e| anyhow!("native scalar duplication failed: {e}"))?
    } else {
        let axes: Vec<usize> = (0..tensor.shape().len()).collect();
        with_default_session(|session| tensor.reduce_sum(Some(&axes), session))
            .map_err(|e| anyhow!("native sum failed: {e}"))?
    };
    Ok(BackendScalar::from_native(reduced)?)
}

/// Return the tangent tensor when present.
///
/// Plain `Tensor` values do not carry tangent storage, so this bridge returns
/// `None`.
pub fn tangent_native_tensor(_tensor: &NativeTensor) -> Option<NativeTensor> {
    None
}

/// Multiply a native tensor by a dynamic scalar.
/// # Errors
///
/// Returns a [`BridgeError`] when `tensor` or `scalar` cannot be promoted to a
/// common dtype, when the dtype is integer, bool, or external, or when the
/// backend rejects the scaled tensor.
pub fn scale_native_tensor(
    tensor: &NativeTensor,
    scalar: &BackendScalar,
) -> std::result::Result<NativeTensor, BridgeError> {
    let target = common_dtype(&[tensor.dtype(), scalar.as_native().dtype()]);
    let tensor = convert_tensor(tensor, target)?;
    let scalar = promote_scalar_native(scalar.as_native(), target)?;

    match target {
        DType::F32 => {
            let factor = native_slice::<f32>(&scalar, "failed to read promoted f32 scalar")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted f32 scalar"))?;
            let values = native_slice::<f32>(&tensor, "failed to read promoted f32 tensor")?
                .iter()
                .map(|&value| value * factor)
                .collect::<Vec<_>>();
            native_tensor_from_vec(tensor.shape().to_vec(), values)
        }
        DType::F64 => {
            let factor = native_slice::<f64>(&scalar, "failed to read promoted f64 scalar")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted f64 scalar"))?;
            let values = native_slice::<f64>(&tensor, "failed to read promoted f64 tensor")?
                .iter()
                .map(|&value| value * factor)
                .collect::<Vec<_>>();
            native_tensor_from_vec(tensor.shape().to_vec(), values)
        }
        DType::C32 => {
            let factor = native_slice::<Complex32>(&scalar, "failed to read promoted c32 scalar")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted c32 scalar"))?;
            let values = native_slice::<Complex32>(&tensor, "failed to read promoted c32 tensor")?
                .iter()
                .map(|&value| value * factor)
                .collect::<Vec<_>>();
            native_tensor_from_vec(tensor.shape().to_vec(), values)
        }
        DType::C64 => {
            let factor = native_slice::<Complex64>(&scalar, "failed to read promoted c64 scalar")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted c64 scalar"))?;
            let values = native_slice::<Complex64>(&tensor, "failed to read promoted c64 tensor")?
                .iter()
                .map(|&value| value * factor)
                .collect::<Vec<_>>();
            native_tensor_from_vec(tensor.shape().to_vec(), values)
        }
        DType::I32 | DType::I64 | DType::Bool => {
            Err(anyhow!("scale_native_tensor does not support integer/bool tensors").into())
        }
        DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
            tenferro_tensor::Error::unsupported(
                "scaling",
                "external scalar kinds are not scalable dtypes",
            ),
        ))),
    }
}

/// Compute `a * lhs + b * rhs`.
/// # Errors
///
/// Returns a [`BridgeError`] when `lhs` and `rhs` have different shapes, when
/// the operands cannot be promoted to a common dtype, when the dtype is
/// integer, bool, or external, or when the backend rejects the result.
pub fn axpby_native_tensor(
    lhs: &NativeTensor,
    a: &BackendScalar,
    rhs: &NativeTensor,
    b: &BackendScalar,
) -> std::result::Result<NativeTensor, BridgeError> {
    if lhs.shape() != rhs.shape() {
        return Err(BridgeError::from(anyhow!(
            "axpby requires matching tensor shapes, got lhs {:?} and rhs {:?}",
            lhs.shape(),
            rhs.shape()
        )));
    }

    let target = common_dtype(&[
        lhs.dtype(),
        rhs.dtype(),
        a.as_native().dtype(),
        b.as_native().dtype(),
    ]);
    let lhs = convert_tensor(lhs, target)?;
    let rhs = convert_tensor(rhs, target)?;
    let a = promote_scalar_native(a.as_native(), target)?;
    let b = promote_scalar_native(b.as_native(), target)?;

    match target {
        DType::F32 => {
            let a = native_slice::<f32>(&a, "failed to read promoted f32 scalar a")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted f32 scalar a"))?;
            let b = native_slice::<f32>(&b, "failed to read promoted f32 scalar b")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted f32 scalar b"))?;
            let lhs_values = native_slice::<f32>(&lhs, "failed to read promoted f32 lhs")?;
            let rhs_values = native_slice::<f32>(&rhs, "failed to read promoted f32 rhs")?;
            let values = lhs_values
                .iter()
                .zip(rhs_values.iter())
                .map(|(&x, &y)| a * x + b * y)
                .collect::<Vec<_>>();
            native_tensor_from_vec(lhs.shape().to_vec(), values)
        }
        DType::F64 => {
            let a = native_slice::<f64>(&a, "failed to read promoted f64 scalar a")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted f64 scalar a"))?;
            let b = native_slice::<f64>(&b, "failed to read promoted f64 scalar b")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted f64 scalar b"))?;
            let lhs_values = native_slice::<f64>(&lhs, "failed to read promoted f64 lhs")?;
            let rhs_values = native_slice::<f64>(&rhs, "failed to read promoted f64 rhs")?;
            let values = lhs_values
                .iter()
                .zip(rhs_values.iter())
                .map(|(&x, &y)| a * x + b * y)
                .collect::<Vec<_>>();
            native_tensor_from_vec(lhs.shape().to_vec(), values)
        }
        DType::C32 => {
            let a = native_slice::<Complex32>(&a, "failed to read promoted c32 scalar a")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted c32 scalar a"))?;
            let b = native_slice::<Complex32>(&b, "failed to read promoted c32 scalar b")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted c32 scalar b"))?;
            let lhs_values = native_slice::<Complex32>(&lhs, "failed to read promoted c32 lhs")?;
            let rhs_values = native_slice::<Complex32>(&rhs, "failed to read promoted c32 rhs")?;
            let values = lhs_values
                .iter()
                .zip(rhs_values.iter())
                .map(|(&x, &y)| a * x + b * y)
                .collect::<Vec<_>>();
            native_tensor_from_vec(lhs.shape().to_vec(), values)
        }
        DType::C64 => {
            let a = native_slice::<Complex64>(&a, "failed to read promoted c64 scalar a")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted c64 scalar a"))?;
            let b = native_slice::<Complex64>(&b, "failed to read promoted c64 scalar b")?
                .first()
                .copied()
                .ok_or_else(|| anyhow!("failed to read promoted c64 scalar b"))?;
            let lhs_values = native_slice::<Complex64>(&lhs, "failed to read promoted c64 lhs")?;
            let rhs_values = native_slice::<Complex64>(&rhs, "failed to read promoted c64 rhs")?;
            let values = lhs_values
                .iter()
                .zip(rhs_values.iter())
                .map(|(&x, &y)| a * x + b * y)
                .collect::<Vec<_>>();
            native_tensor_from_vec(lhs.shape().to_vec(), values)
        }
        DType::I32 | DType::I64 | DType::Bool => {
            Err(anyhow!("axpby_native_tensor does not support integer/bool tensors").into())
        }
        DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
            tenferro_tensor::Error::unsupported(
                "axpby",
                "external scalar kinds are not axpby dtypes",
            ),
        ))),
    }
}

/// Execute a cached einsum over owned native tensors.
///
/// This is the consuming bridge used by higher-level owned contraction APIs.
/// Inputs are promoted to a common dtype before tenferro evaluates the
/// contraction. Repeated calls with the same equation and shapes reuse
/// tenferro's process-global contraction path cache.
///
/// # Arguments
/// * `operands` - Native tensors paired with numeric einsum labels for each axis.
/// * `output_ids` - Numeric labels to keep in the result, in output axis order.
///
/// # Returns
/// The contracted native tensor in the promoted common dtype.
///
/// # Errors
/// Returns an error if the operand list is empty, any label list length does
/// not match its tensor rank, label generation exceeds the supported range, or
/// the backend contraction fails.
///
/// # Examples
/// ```
/// use tensor4all_tensorbackend::einsum_native_tensors_owned;
/// use tenferro::Tensor as NativeTensor;
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
///
/// let lhs = NativeTensor::from_vec_col_major(vec![2, 3], vec![1.0_f64; 6])?;
/// let rhs = NativeTensor::from_vec_col_major(vec![3, 2], vec![1.0_f64; 6])?;
/// let result = einsum_native_tensors_owned(vec![(lhs, vec![0, 1]), (rhs, vec![1, 2])], &[0, 2])?;
///
/// assert_eq!(result.shape(), &[2, 2]);
/// assert_eq!(result.as_slice::<f64>()?, &[3.0, 3.0, 3.0, 3.0]);
/// # Ok(())
/// # }
/// ```
pub fn einsum_native_tensors_owned(
    operands: Vec<(NativeTensor, Vec<usize>)>,
    output_ids: &[usize],
) -> Result<NativeTensor> {
    ensure!(
        !operands.is_empty(),
        "native einsum requires at least one operand"
    );

    let target = common_dtype(
        &operands
            .iter()
            .map(|(tensor, _)| tensor.dtype())
            .collect::<Vec<_>>(),
    );

    let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
    let mut converted = Vec::with_capacity(operands.len());
    let mut input_ids = Vec::with_capacity(operands.len());
    for (tensor, ids) in operands {
        ensure!(
            tensor.shape().len() == ids.len(),
            "einsum id list {:?} does not match tensor shape {:?}",
            ids,
            tensor.shape()
        );
        let checked_ids = checked_native_einsum_labels(&ids)?;
        let tensor = if tensor.dtype() == target {
            tensor
        } else {
            convert_tensor(&tensor, target)?
        };
        input_ids.push(checked_ids);
        converted.push(tensor);
    }

    let input_slices = input_ids.iter().map(Vec::as_slice).collect::<Vec<_>>();
    let subscripts = EinsumSubscripts::new(&input_slices, &output_ids_u32);

    let input_refs = converted.iter().collect::<Vec<_>>();
    let trace_operands = input_refs
        .iter()
        .zip(input_ids.iter())
        .map(|(tensor, ids)| (*tensor, ids.as_slice()))
        .collect::<Vec<_>>();
    maybe_trace_native_einsum_path(NativeEinsumPath::Owned, &trace_operands, &output_ids_u32);
    let started = Instant::now();
    let result = cached_einsum_native_tensors(&input_refs, &subscripts)?;
    record_native_einsum_profile(
        NativeEinsumPath::Owned,
        &trace_operands,
        &output_ids_u32,
        started.elapsed(),
    );
    Ok(result)
}

/// Execute a cached einsum over borrowed native tensors.
///
/// Inputs are promoted to a common dtype before contraction. Operands that
/// already have the target dtype are passed to the backend by reference;
/// operands with another dtype are converted into temporary native tensors and
/// then borrowed for the contraction. Repeated calls with the same equation
/// and shapes reuse tenferro's process-global contraction path cache.
///
/// # Arguments
/// * `operands` - Native tensors paired with numeric einsum labels for each axis.
///
///   Each label slice must have the same length as the corresponding tensor rank.
/// * `output_ids` - Numeric labels to keep in the result, in output axis order.
///
/// # Returns
/// The contracted native tensor in the promoted common dtype.
///
/// # Errors
/// Returns an error if the operand list is empty, any label list length does
/// not match its tensor rank, label generation exceeds the supported range,
/// dtype conversion fails, or the backend contraction fails.
///
/// # Examples
/// ```
/// use tensor4all_tensorbackend::einsum_native_tensors;
/// use tenferro::Tensor as NativeTensor;
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
///
/// let lhs = NativeTensor::from_vec_col_major(vec![2, 3], vec![1.0_f64; 6])?;
/// let rhs = NativeTensor::from_vec_col_major(vec![3, 2], vec![1.0_f64; 6])?;
/// let result = einsum_native_tensors(&[(&lhs, &[0, 1]), (&rhs, &[1, 2])], &[0, 2])?;
///
/// assert_eq!(result.shape(), &[2, 2]);
/// assert_eq!(result.as_slice::<f64>()?, &[3.0, 3.0, 3.0, 3.0]);
/// # Ok(())
/// # }
/// ```
pub fn einsum_native_tensors(
    operands: &[(&NativeTensor, &[usize])],
    output_ids: &[usize],
) -> Result<NativeTensor> {
    ensure!(
        !operands.is_empty(),
        "native einsum requires at least one operand"
    );

    let target = common_dtype(
        &operands
            .iter()
            .map(|(tensor, _)| tensor.dtype())
            .collect::<Vec<_>>(),
    );
    let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
    let mut converted = Vec::with_capacity(operands.len());
    let mut input_ids = Vec::with_capacity(operands.len());
    let mut has_conversions = false;
    let started = Instant::now();

    for (tensor, ids) in operands {
        ensure!(
            tensor.shape().len() == ids.len(),
            "einsum id list {:?} does not match tensor shape {:?}",
            ids,
            tensor.shape()
        );
        input_ids.push(checked_native_einsum_labels(ids)?);
        if tensor.dtype() == target {
            converted.push(None);
        } else {
            converted.push(Some(convert_tensor(tensor, target)?));
            has_conversions = true;
        }
    }

    let input_slices = input_ids.iter().map(Vec::as_slice).collect::<Vec<_>>();
    let subscripts = EinsumSubscripts::new(&input_slices, &output_ids_u32);
    let input_refs = operands
        .iter()
        .zip(converted.iter())
        .map(|((tensor, _), converted)| converted.as_ref().unwrap_or(*tensor))
        .collect::<Vec<_>>();
    let trace_path = if has_conversions {
        NativeEinsumPath::BorrowedWithConversions
    } else {
        NativeEinsumPath::Borrowed
    };
    let trace_operands = input_refs
        .iter()
        .zip(input_ids.iter())
        .map(|(tensor, ids)| (*tensor, ids.as_slice()))
        .collect::<Vec<_>>();
    maybe_trace_native_einsum_path(trace_path, &trace_operands, &output_ids_u32);
    let result = cached_einsum_native_tensors(&input_refs, &subscripts)?;
    record_native_einsum_profile(
        trace_path,
        &trace_operands,
        &output_ids_u32,
        started.elapsed(),
    );
    Ok(result)
}

/// Execute a cached einsum over borrowed native tensors using string subscripts.
///
/// This is the shared string-notation entry: it parses `subscripts` into the
/// numeric labels that [`einsum_native_tensors`] consumes, checks that the
/// equation describes exactly the supplied operands, and then delegates the
/// contraction. Use it instead of re-implementing the parsing plus label
/// mapping in each crate; use [`einsum_native_tensors`] directly when the
/// labels are already numeric.
///
/// # Arguments
///
/// * `subscripts` - NumPy/PyTorch-style einsum notation such as `"ij,jk->ik"`
///   or `"ii->"` for a trace. Parenthesized contraction order is rejected; see
///   [`tenferro_einsum::Subscripts::parse`].
/// * `operands` - One native tensor per comma-separated input term, in term
///   order. Ranks must match the term lengths and repeated labels must have
///   equal extents.
///
/// # Returns
///
/// The contracted native tensor in the promoted common dtype of the operands.
///
/// # Errors
///
/// Returns an error when the notation parser rejects `subscripts` as invalid
/// notation, when the operand count does not match the number of comma-separated
/// input terms, when an input term's label count does not match its operand
/// rank, when repeated or contracted labels have incompatible shapes, or when
/// dtype promotion or the backend contraction fails.
///
/// # Compatibility frontend
///
/// Like [`einsum_native_tensors`], this runs on the retained process-global
/// (compatibility) frontend of
/// [tensor4all-rs#859](https://github.com/tensor4all/tensor4all-rs/issues/859):
/// it selects the default execution context and pays one graph dispatch per
/// call. It is meant for cold or per-stage paths, not per-element hot loops;
/// use [`CpuExecutionContext`](crate::CpuExecutionContext) sessions for those.
///
/// # Examples
///
/// ```
/// use tensor4all_tensorbackend::einsum_native_subscripts;
/// use tenferro::Tensor as NativeTensor;
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
///
/// // Column-major A = [[1, 2], [3, 4]] and B = [[5, 6], [7, 8]].
/// let lhs = NativeTensor::from_vec_col_major(vec![2, 2], vec![1.0_f64, 3.0, 2.0, 4.0])?;
/// let rhs = NativeTensor::from_vec_col_major(vec![2, 2], vec![5.0_f64, 7.0, 6.0, 8.0])?;
/// let product = einsum_native_subscripts("ij,jk->ik", &[&lhs, &rhs])?;
/// assert_eq!(product.shape(), &[2, 2]);
/// assert_eq!(product.as_slice::<f64>()?, &[19.0, 43.0, 22.0, 50.0]);
///
/// let trace = einsum_native_subscripts("ii->", &[&lhs])?;
/// assert_eq!(trace.as_slice::<f64>()?, &[5.0]);
/// # Ok(())
/// # }
/// ```
pub fn einsum_native_subscripts(
    subscripts: &str,
    operands: &[&NativeTensor],
) -> Result<NativeTensor> {
    let parsed = Subscripts::parse(subscripts)
        .map_err(|error| anyhow!("invalid einsum subscripts '{subscripts}': {error}"))?;
    ensure!(
        parsed.inputs.len() == operands.len(),
        "einsum subscripts '{subscripts}' describe {} input term(s) but {} operand(s) were given",
        parsed.inputs.len(),
        operands.len()
    );

    let input_ids = parsed
        .inputs
        .iter()
        .map(|ids| ids.iter().map(|&id| id as usize).collect::<Vec<_>>())
        .collect::<Vec<_>>();
    let output_ids = parsed
        .output
        .iter()
        .map(|&id| id as usize)
        .collect::<Vec<_>>();
    let operand_refs = operands
        .iter()
        .zip(input_ids.iter())
        .map(|(tensor, ids)| (*tensor, ids.as_slice()))
        .collect::<Vec<_>>();

    einsum_native_tensors(&operand_refs, &output_ids)
}

/// Execute a cached einsum over read-only native tensor inputs.
///
/// Backends consume borrowed host views inside their execution session. Mixed
/// dtypes are promoted by `Convert` nodes in the compiled einsum graph, so
/// non-contiguous operands remain borrowed until runtime execution.
/// # Errors
///
/// Returns an error when `operands` is empty, when a tensor rank does not
/// match its id list length, when a label is out of bounds for the supported
/// range, or when the backend einsum fails.
pub fn einsum_native_tensor_reads(
    operands: &[(&NativeTensorReadInput<'_>, &[usize])],
    output_ids: &[usize],
) -> Result<NativeTensor> {
    ensure!(
        !operands.is_empty(),
        "native einsum requires at least one operand"
    );

    let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
    let mut input_ids = Vec::with_capacity(operands.len());
    let mut read_inputs = Vec::with_capacity(operands.len());

    for (tensor, ids) in operands {
        ensure!(
            tensor.shape().len() == ids.len(),
            "einsum id list {:?} does not match tensor shape {:?}",
            ids,
            tensor.shape()
        );
        input_ids.push(checked_native_einsum_labels(ids)?);
        read_inputs.push(tensor.as_read());
    }

    let subscripts = Subscripts {
        inputs: input_ids,
        output: output_ids_u32,
    };
    cached_einsum_native_reads(&read_inputs, &subscripts)
}

/// Permute axes of a native tensor.
/// # Errors
///
/// Returns a [`BridgeError`] when `perm` is not a permutation of `tensor`'s
/// axes or when the backend transpose fails.
pub fn permute_native_tensor(
    tensor: &NativeTensor,
    perm: &[usize],
) -> std::result::Result<NativeTensor, BridgeError> {
    with_default_session(|session| tensor.transpose(perm, session))
        .map_err(|e| anyhow!("native permute failed: {e}"))
        .map_err(BridgeError::from)
}

/// Contract two native tensors along matching axes.
/// # Errors
///
/// Returns an error when `axes_a`/`axes_b` are out of bounds or have differing
/// lengths, when the contracted dimensions do not match, or when the backend
/// einsum fails.
pub fn contract_native_tensor(
    lhs: &NativeTensor,
    axes_a: &[usize],
    rhs: &NativeTensor,
    axes_b: &[usize],
) -> Result<NativeTensor> {
    let (lhs_ids, rhs_ids, output_ids) =
        build_binary_einsum_ids(lhs.shape().len(), axes_a, rhs.shape().len(), axes_b)?;
    let lhs_ids_usize = lhs_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
    let rhs_ids_usize = rhs_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
    let output_ids_usize = output_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
    let operands = [
        (lhs, lhs_ids_usize.as_slice()),
        (rhs, rhs_ids_usize.as_slice()),
    ];
    einsum_native_tensors(&operands, &output_ids_usize)
}

/// Compute the outer product of two native tensors.
/// # Errors
///
/// Returns a [`BridgeError`] when the underlying einsum contraction fails.
pub fn outer_product_native_tensor(
    lhs: &NativeTensor,
    rhs: &NativeTensor,
) -> std::result::Result<NativeTensor, BridgeError> {
    contract_native_tensor(lhs, &[], rhs, &[]).map_err(BridgeError::from)
}

/// Conjugate a native tensor.
/// # Errors
///
/// Returns a [`BridgeError`] when the dtype is external, when duplicating a
/// real tensor fails, or when the host slice of a complex tensor cannot be
/// read.
pub fn conj_native_tensor(tensor: &NativeTensor) -> std::result::Result<NativeTensor, BridgeError> {
    match tensor.dtype() {
        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool => tensor
            .duplicate()
            .map_err(|e| anyhow!("native tensor duplication failed: {e}"))
            .map_err(BridgeError::from),
        DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
            tenferro_tensor::Error::unsupported(
                "conjugation",
                "external scalar kinds have no conjugation",
            ),
        ))),
        DType::C32 => native_tensor_from_vec(
            tensor.shape().to_vec(),
            native_slice::<Complex32>(tensor, "failed to read c32 native tensor")?
                .iter()
                .map(|&value| value.conj())
                .collect::<Vec<_>>(),
        ),
        DType::C64 => native_tensor_from_vec(
            tensor.shape().to_vec(),
            native_slice::<Complex64>(tensor, "failed to read c64 native tensor")?
                .iter()
                .map(|&value| value.conj())
                .collect::<Vec<_>>(),
        ),
    }
}

/// Permute storage by round-tripping through native tensors.
/// # Errors
///
/// Returns a [`BridgeError`] when the storage-to-native conversion, the
/// permutation, or the native-to-storage conversion fails.
pub fn permute_storage_native(
    storage: &Storage,
    logical_dims: &[usize],
    perm: &[usize],
) -> std::result::Result<Storage, BridgeError> {
    let native = storage_to_native_tensor(storage, logical_dims)?;
    let permuted = permute_native_tensor(&native, perm)?;
    native_tensor_primal_to_storage(&permuted)
}

/// Contract storages via native tensors.
/// # Errors
///
/// Returns a [`BridgeError`] when either storage-to-native conversion, the
/// contraction, or the native-to-storage conversion fails.
pub fn contract_storage_native(
    storage_a: &Storage,
    dims_a: &[usize],
    axes_a: &[usize],
    storage_b: &Storage,
    dims_b: &[usize],
    axes_b: &[usize],
    _result_dims: &[usize],
) -> std::result::Result<Storage, BridgeError> {
    let lhs = storage_to_native_tensor(storage_a, dims_a)?;
    let rhs = storage_to_native_tensor(storage_b, dims_b)?;
    let result = contract_native_tensor(&lhs, axes_a, &rhs, axes_b)?;
    native_tensor_primal_to_storage(&result)
}

/// Outer-product storages via native tensors.
/// # Errors
///
/// Returns a [`BridgeError`] when either storage-to-native conversion, the
/// outer product, or the native-to-storage conversion fails.
pub fn outer_product_storage_native(
    lhs: &Storage,
    lhs_dims: &[usize],
    rhs: &Storage,
    rhs_dims: &[usize],
    _result_dims: &[usize],
) -> std::result::Result<Storage, BridgeError> {
    let lhs = storage_to_native_tensor(lhs, lhs_dims)?;
    let rhs = storage_to_native_tensor(rhs, rhs_dims)?;
    let result = outer_product_native_tensor(&lhs, &rhs)?;
    native_tensor_primal_to_storage(&result)
}

/// Scale storage by a scalar via native tensors.
/// # Errors
///
/// Returns a [`BridgeError`] when the storage-to-native conversion, the
/// scaling, or the native-to-storage conversion fails.
pub fn scale_storage_native(
    storage: &Storage,
    logical_dims: &[usize],
    scalar: &BackendScalar,
) -> std::result::Result<Storage, BridgeError> {
    let native = storage_to_native_tensor(storage, logical_dims)?;
    let scaled = scale_native_tensor(&native, scalar)?;
    native_tensor_primal_to_storage(&scaled)
}

/// Compute `a * lhs + b * rhs` over storages via native tensors.
/// # Errors
///
/// Returns a [`BridgeError`] when either storage-to-native conversion, the
/// axpby, or the native-to-storage conversion fails.
pub fn axpby_storage_native(
    lhs: &Storage,
    lhs_dims: &[usize],
    a: &BackendScalar,
    rhs: &Storage,
    rhs_dims: &[usize],
    b: &BackendScalar,
) -> std::result::Result<Storage, BridgeError> {
    let lhs = storage_to_native_tensor(lhs, lhs_dims)?;
    let rhs = storage_to_native_tensor(rhs, rhs_dims)?;
    let combined = axpby_native_tensor(&lhs, a, &rhs, b)?;
    native_tensor_primal_to_storage(&combined)
}

#[cfg(test)]
mod tests;