Skip to main content

tensor4all_tensorbackend/
tenferro_bridge.rs

1//! Bridge helpers between tensor4all storage snapshots and tenferro tensors.
2
3use std::cell::RefCell;
4use std::cmp::Reverse;
5use std::collections::{HashMap, HashSet};
6use std::env;
7use std::sync::Arc;
8use std::time::{Duration, Instant};
9
10use anyhow::{anyhow, ensure, Result};
11use num_complex::{Complex32, Complex64};
12use omeco::ScoreFunction;
13use tenferro::{
14    program::CoreSemanticOp, CompiledGraph, DType, GraphCompiler, Tensor as NativeTensor,
15    TensorRead, TensorScalar, TensorSessionOpsExt, TraceContext,
16};
17use tenferro_einsum::{
18    ConcreteEinsumPlan, ContractionOptimizerOptions, ContractionTree, EinsumSubscripts, Subscripts,
19    TraceContextEinsumExt,
20};
21use tenferro_linalg::TensorLinalgExt;
22
23use crate::any_scalar::promote_scalar_native;
24pub(crate) use crate::einsum_ids::{
25    build_binary_einsum_ids, checked_native_einsum_labels, common_dtype, convert_native_tensor_in,
26};
27/// Error returned by the storage/tensor bridge helpers.
28///
29/// Wraps the underlying tensor-element or backend diagnostic, preserving its
30/// source chain.
31///
32/// # Remedies
33/// - Dtype mismatch: extract with the element type matching the native tensor
34///   dtype, or promote explicitly before the call.
35/// - Shape mismatch: validate the native tensor shape against the payload
36///   contract.
37/// - Backend failure: the wrapped source chain identifies the failing stage.
38#[derive(Debug, thiserror::Error)]
39#[error("native tensor bridge operation failed: {source}")]
40pub struct BridgeError {
41    /// Original tensor-element or backend diagnostic.
42    #[source]
43    pub source: anyhow::Error,
44}
45
46impl From<anyhow::Error> for BridgeError {
47    fn from(source: anyhow::Error) -> Self {
48        Self { source }
49    }
50}
51
52fn native_tensor_from_vec<T: TensorScalar>(
53    shape: Vec<usize>,
54    values: Vec<T>,
55) -> std::result::Result<NativeTensor, BridgeError> {
56    NativeTensor::from_vec_col_major(shape, values)
57        .map_err(|source| BridgeError::from(anyhow::Error::new(source)))
58}
59
60use crate::context::{
61    default_context, default_engine_buffer_pool_stats, reset_default_engine,
62    reset_default_engine_buffer_pool, with_default_graph_compiler, with_default_session,
63};
64use crate::memory::release_process_allocator_cached_memory;
65use crate::storage::{storage_dtype_dispatch, Storage, StorageScalar};
66use crate::tensor_element::TensorElement;
67use crate::BackendScalar;
68
69/// Read-only native tensor input that can either borrow external payload data
70/// or own a temporary materialized tensor.
71// Both tenferro handle variants are large; boxing the slightly larger one would
72// add an allocation to the eager einsum path for little enum-size reduction.
73#[allow(clippy::large_enum_variant)]
74pub enum NativeTensorReadInput<'a> {
75    /// Borrowed read-only tensor input.
76    Borrowed(TensorRead<'a>),
77    /// Owned temporary tensor input.
78    Owned(NativeTensor),
79}
80
81impl<'a> NativeTensorReadInput<'a> {
82    /// Return this input as a read-only tenferro tensor input.
83    pub fn as_read(&'a self) -> TensorRead<'a> {
84        match self {
85            Self::Borrowed(read) => read.clone(),
86            Self::Owned(tensor) => TensorRead::from_tensor(tensor),
87        }
88    }
89
90    /// Return the scalar dtype of this input.
91    pub fn dtype(&self) -> DType {
92        match self {
93            Self::Borrowed(read) => read.dtype(),
94            Self::Owned(tensor) => tensor.dtype(),
95        }
96    }
97
98    /// Return the tensor shape of this input.
99    pub fn shape(&self) -> &[usize] {
100        match self {
101            Self::Borrowed(read) => read.shape(),
102            Self::Owned(tensor) => tensor.shape(),
103        }
104    }
105}
106
107#[cfg(test)]
108use std::cell::Cell;
109
110#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
111enum NativeEinsumPath {
112    Owned,
113    Borrowed,
114    BorrowedWithConversions,
115}
116
117#[derive(Debug, Clone, Hash, PartialEq, Eq)]
118struct NativeOperandSignature {
119    shape: Vec<usize>,
120    ids: Vec<u32>,
121    dtype: DType,
122}
123
124#[derive(Debug, Clone, Hash, PartialEq, Eq)]
125struct NativeEinsumSignature {
126    path: NativeEinsumPath,
127    operands: Vec<NativeOperandSignature>,
128    output_ids: Vec<u32>,
129}
130
131#[derive(Debug, Default, Clone)]
132struct NativeEinsumProfileEntry {
133    calls: usize,
134    total_time: Duration,
135}
136
137/// Entries kept before the cache is cleared. Real workloads cycle through
138/// hundreds to a few thousand distinct contraction signatures (#891: a
139/// 400-shape QTT cycle re-prepared every plan at 256), so the bound sits above
140/// that working set while still capping the per-thread cache.
141const CONCRETE_EINSUM_PLAN_CACHE_CAPACITY: usize = 4096;
142
143/// One parsed einsum expression, shared by every plan built from the same string.
144#[derive(Debug)]
145struct NativeEinsumSpec {
146    subscripts: Subscripts,
147    einsum_subscripts: EinsumSubscripts,
148}
149
150/// Distinct expression strings kept before the cache is cleared. The string form is used with
151/// literal expressions, so the working set is a few dozen entries.
152const NATIVE_EINSUM_SPEC_CACHE_CAPACITY: usize = 1024;
153
154thread_local! {
155    static NATIVE_EINSUM_SPEC_CACHE: RefCell<HashMap<String, Arc<NativeEinsumSpec>>> =
156        RefCell::new(HashMap::new());
157    static NATIVE_EINSUM_PROFILE_STATE: RefCell<HashMap<NativeEinsumSignature, NativeEinsumProfileEntry>> =
158        RefCell::new(HashMap::new());
159    static NATIVE_EINSUM_TRACE_STATE: RefCell<HashSet<NativeEinsumSignature>> =
160        RefCell::new(HashSet::new());
161    static CONCRETE_EINSUM_PLAN_CACHE: RefCell<HashMap<NativeEinsumSignature, Arc<ConcreteEinsumPlan>>> =
162        RefCell::new(HashMap::new());
163}
164
165#[cfg(test)]
166thread_local! {
167    static FORCE_NATIVE_EINSUM_PROFILE: Cell<bool> = const { Cell::new(false) };
168}
169
170fn native_einsum_profile_enabled() -> bool {
171    #[cfg(test)]
172    if FORCE_NATIVE_EINSUM_PROFILE.with(Cell::get) {
173        return true;
174    }
175    env::var("T4A_PROFILE_NATIVE_EINSUM").is_ok()
176}
177
178fn native_einsum_path_trace_enabled() -> bool {
179    env::var("T4A_TRACE_NATIVE_EINSUM_PATHS").is_ok()
180}
181
182fn native_einsum_path_trace_min_bytes() -> usize {
183    env::var("T4A_TRACE_NATIVE_EINSUM_MIN_BYTES")
184        .ok()
185        .and_then(|value| value.parse().ok())
186        .unwrap_or(0)
187}
188
189fn native_einsum_path_trace_max_signatures() -> usize {
190    env::var("T4A_TRACE_NATIVE_EINSUM_MAX_SIGNATURES")
191        .ok()
192        .and_then(|value| value.parse().ok())
193        .unwrap_or(64)
194}
195
196fn native_einsum_pool_trace_enabled() -> bool {
197    env::var("T4A_TRACE_NATIVE_EINSUM_POOL").is_ok()
198}
199
200fn native_einsum_pool_trace_min_output_bytes() -> usize {
201    env::var("T4A_TRACE_NATIVE_EINSUM_POOL_MIN_OUTPUT_BYTES")
202        .ok()
203        .and_then(|value| value.parse().ok())
204        .unwrap_or(0)
205}
206
207fn native_einsum_pool_trace_min_retained_bytes() -> usize {
208    env::var("T4A_TRACE_NATIVE_EINSUM_POOL_MIN_RETAINED_BYTES")
209        .ok()
210        .and_then(|value| value.parse().ok())
211        .unwrap_or(0)
212}
213
214fn reset_native_einsum_engine_after_call() -> bool {
215    env::var("T4A_RESET_NATIVE_EINSUM_ENGINE_AFTER_CALL").is_ok()
216}
217
218fn reset_native_einsum_buffer_pool_after_call() -> bool {
219    env::var("T4A_RESET_NATIVE_EINSUM_BUFFER_POOL_AFTER_CALL").is_ok()
220}
221
222fn release_allocator_after_native_einsum_call() -> bool {
223    env::var("T4A_RELEASE_ALLOCATOR_AFTER_NATIVE_EINSUM_CALL").is_ok()
224}
225
226#[cfg(test)]
227pub(crate) fn set_native_einsum_profile_enabled_for_tests(enabled: bool) {
228    FORCE_NATIVE_EINSUM_PROFILE.with(|slot| slot.set(enabled));
229}
230
231fn native_einsum_signature(
232    path: NativeEinsumPath,
233    operands: &[(&NativeTensor, &[u32])],
234    output_ids: &[u32],
235) -> NativeEinsumSignature {
236    NativeEinsumSignature {
237        path,
238        operands: operands
239            .iter()
240            .map(|(tensor, ids)| NativeOperandSignature {
241                shape: tensor.shape().to_vec(),
242                ids: ids.to_vec(),
243                dtype: tensor.dtype(),
244            })
245            .collect(),
246        output_ids: output_ids.to_vec(),
247    }
248}
249
250fn concrete_einsum_dtype_supported(dtype: DType) -> bool {
251    matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64)
252}
253
254fn concrete_einsum_signature(
255    inputs: &[TensorRead<'_>],
256    subscripts: &Subscripts,
257) -> NativeEinsumSignature {
258    // ConcreteEinsumPlan owns only dtype, shape, labels, and its contraction
259    // tree. Per-call TensorRead strides are validated and consumed at execute.
260    NativeEinsumSignature {
261        path: NativeEinsumPath::Borrowed,
262        operands: inputs
263            .iter()
264            .zip(subscripts.inputs.iter())
265            .map(|(tensor, ids)| NativeOperandSignature {
266                shape: tensor.shape().to_vec(),
267                ids: ids.clone(),
268                dtype: tensor.dtype(),
269            })
270            .collect(),
271        output_ids: subscripts.output.clone(),
272    }
273}
274
275/// Parse an einsum expression once per distinct string on this thread.
276fn cached_native_einsum_spec(subscripts: &str) -> Result<Arc<NativeEinsumSpec>> {
277    if let Some(spec) =
278        NATIVE_EINSUM_SPEC_CACHE.with(|cache| cache.borrow().get(subscripts).cloned())
279    {
280        return Ok(spec);
281    }
282    let parsed = Subscripts::parse(subscripts)
283        .map_err(|error| anyhow!("invalid einsum subscripts '{subscripts}': {error}"))?;
284    let spec = Arc::new(NativeEinsumSpec {
285        einsum_subscripts: EinsumSubscripts::from(&parsed),
286        subscripts: parsed,
287    });
288    NATIVE_EINSUM_SPEC_CACHE.with(|cache| {
289        let mut cache = cache.borrow_mut();
290        if cache.len() >= NATIVE_EINSUM_SPEC_CACHE_CAPACITY {
291            cache.clear();
292        }
293        cache.insert(subscripts.to_string(), Arc::clone(&spec));
294    });
295    Ok(spec)
296}
297
298fn cached_concrete_einsum_plan(
299    inputs: &[TensorRead<'_>],
300    subscripts: &Subscripts,
301    einsum_subscripts: &EinsumSubscripts,
302) -> Result<Arc<ConcreteEinsumPlan>> {
303    let signature = concrete_einsum_signature(inputs, subscripts);
304    if let Some(plan) =
305        CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow().get(&signature).cloned())
306    {
307        return Ok(plan);
308    }
309
310    let plan = Arc::new(
311        ConcreteEinsumPlan::prepare_read_subscripts(inputs, einsum_subscripts)
312            .map_err(|error| anyhow!("native einsum session plan preparation failed: {error}"))?,
313    );
314    CONCRETE_EINSUM_PLAN_CACHE.with(|cache| {
315        let mut cache = cache.borrow_mut();
316        // ponytail: bounded clear-on-full cache; use an LRU only if shape churn is measured.
317        if cache.len() >= CONCRETE_EINSUM_PLAN_CACHE_CAPACITY {
318            cache.clear();
319        }
320        cache.insert(signature, Arc::clone(&plan));
321    });
322    Ok(plan)
323}
324
325#[cfg(test)]
326fn reset_concrete_einsum_plan_cache() {
327    CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow_mut().clear());
328}
329
330#[cfg(test)]
331fn native_einsum_spec_cache_len() -> usize {
332    NATIVE_EINSUM_SPEC_CACHE.with(|cache| cache.borrow().len())
333}
334
335#[cfg(test)]
336fn concrete_einsum_plan_cache_len() -> usize {
337    CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow().len())
338}
339
340fn record_native_einsum_profile(
341    path: NativeEinsumPath,
342    operands: &[(&NativeTensor, &[u32])],
343    output_ids: &[u32],
344    elapsed: Duration,
345) {
346    if !native_einsum_profile_enabled() {
347        return;
348    }
349    let signature = native_einsum_signature(path, operands, output_ids);
350    NATIVE_EINSUM_PROFILE_STATE.with(|state| {
351        let mut state = state.borrow_mut();
352        let entry = state.entry(signature).or_default();
353        entry.calls += 1;
354        entry.total_time += elapsed;
355    });
356}
357
358fn native_slice<'a, T: TensorScalar>(
359    tensor: &'a NativeTensor,
360    label: &'static str,
361) -> Result<&'a [T]> {
362    tensor
363        .as_slice::<T>()
364        .map_err(|error| anyhow!("{label}: {error}"))
365}
366
367fn dtype_size_bytes(dtype: DType) -> usize {
368    match dtype {
369        DType::F32 => 4,
370        DType::F64 => 8,
371        DType::C32 => 8,
372        DType::C64 => 16,
373        DType::I32 => 4,
374        DType::I64 => 8,
375        DType::Bool => 1,
376        // External scalar kinds have no element width this crate can know. They
377        // are rejected at every entry point that can materialize storage, so this
378        // accounting arm is unreachable for a tensor that passed those checks.
379        DType::External(_) => 0,
380    }
381}
382
383fn native_tensor_bytes(tensor: &NativeTensor) -> usize {
384    tensor
385        .shape()
386        .iter()
387        .copied()
388        .fold(1usize, usize::saturating_mul)
389        .saturating_mul(dtype_size_bytes(tensor.dtype()))
390}
391
392fn format_label(label: u32) -> String {
393    char::from_u32(label).map_or_else(|| label.to_string(), |label| label.to_string())
394}
395
396fn format_labels(labels: &[u32]) -> String {
397    if labels.is_empty() {
398        "scalar".to_string()
399    } else {
400        labels
401            .iter()
402            .map(|&label| format_label(label))
403            .collect::<Vec<_>>()
404            .join("")
405    }
406}
407
408fn label_dims(subscripts: &Subscripts, shapes: &[Vec<usize>]) -> Result<HashMap<u32, usize>> {
409    let mut dims = HashMap::new();
410    for (labels, shape) in subscripts.inputs.iter().zip(shapes.iter()) {
411        ensure!(
412            labels.len() == shape.len(),
413            "einsum labels {:?} do not match shape {:?}",
414            labels,
415            shape
416        );
417        for (&label, &dim) in labels.iter().zip(shape.iter()) {
418            if let Some(previous) = dims.insert(label, dim) {
419                ensure!(
420                    previous == dim,
421                    "inconsistent dimension for einsum label {}: {} vs {}",
422                    format_label(label),
423                    previous,
424                    dim
425                );
426            }
427        }
428    }
429    Ok(dims)
430}
431
432fn labels_size(labels: &[u32], dims: &HashMap<u32, usize>) -> usize {
433    labels.iter().fold(1usize, |size, label| {
434        size.saturating_mul(dims.get(label).copied().unwrap_or(1))
435    })
436}
437
438fn union_labels(lhs: &[u32], rhs: &[u32]) -> Vec<u32> {
439    let mut seen = HashSet::new();
440    let mut labels = Vec::new();
441    for &label in lhs.iter().chain(rhs.iter()) {
442        if seen.insert(label) {
443            labels.push(label);
444        }
445    }
446    labels
447}
448
449#[derive(Debug)]
450struct NativeEinsumPlanReport {
451    lines: Vec<String>,
452    peak_intermediate_bytes: usize,
453}
454
455fn time_optimized_contraction_options() -> ContractionOptimizerOptions {
456    ContractionOptimizerOptions {
457        score: ScoreFunction::time_optimized(),
458        ..ContractionOptimizerOptions::default()
459    }
460}
461
462fn native_einsum_plan_report_with_options(
463    signature: &NativeEinsumSignature,
464    optimizer_name: &'static str,
465    options: &ContractionOptimizerOptions,
466) -> Result<NativeEinsumPlanReport> {
467    let input_ids = signature
468        .operands
469        .iter()
470        .map(|operand| operand.ids.as_slice())
471        .collect::<Vec<_>>();
472    let subscripts_string = build_einsum_subscripts(&input_ids, &signature.output_ids)?;
473    let subscripts = Subscripts {
474        inputs: input_ids.iter().map(|ids| ids.to_vec()).collect(),
475        output: signature.output_ids.clone(),
476    };
477    let shapes = signature
478        .operands
479        .iter()
480        .map(|operand| operand.shape.clone())
481        .collect::<Vec<_>>();
482    let shape_refs = shapes.iter().map(Vec::as_slice).collect::<Vec<_>>();
483    let tree = ContractionTree::optimize_with_options(&subscripts, &shape_refs, options)
484        .map_err(|e| anyhow!("failed to optimize native einsum path: {e}"))?;
485    let dims = label_dims(&subscripts, &shapes)?;
486    let dtype = signature
487        .operands
488        .first()
489        .map(|operand| operand.dtype)
490        .unwrap_or(DType::F64);
491    let dtype_size = dtype_size_bytes(dtype);
492
493    let mut lines = Vec::new();
494    lines.push(format!(
495        "optimizer={optimizer_name} subscripts={subscripts_string} dtype={dtype:?} steps={}",
496        tree.step_count()
497    ));
498    let mut peak_intermediate_elems = 1usize;
499    for step in 0..tree.step_count() {
500        let Some((left, right)) = tree.step_pair(step) else {
501            continue;
502        };
503        let Some((lhs, rhs, out)) = tree.step_subscripts(step) else {
504            continue;
505        };
506        let lhs_elems = labels_size(lhs, &dims);
507        let rhs_elems = labels_size(rhs, &dims);
508        let out_elems = labels_size(out, &dims);
509        let flop_index_elems = labels_size(&union_labels(lhs, rhs), &dims);
510        peak_intermediate_elems = peak_intermediate_elems.max(out_elems);
511        lines.push(format!(
512            "  step {step:02}: pair=({left},{right}) {}[{}] x {}[{}] -> {}[{}]  flop_index={}  intermediate={} elems ({:.3} MiB)",
513            format_labels(lhs),
514            lhs_elems,
515            format_labels(rhs),
516            rhs_elems,
517            format_labels(out),
518            out_elems,
519            flop_index_elems,
520            out_elems,
521            out_elems as f64 * dtype_size as f64 / (1024.0 * 1024.0),
522        ));
523    }
524    let peak_intermediate_bytes = peak_intermediate_elems.saturating_mul(dtype_size);
525    lines.push(format!(
526        "  peak_intermediate={} elems ({:.3} MiB)",
527        peak_intermediate_elems,
528        peak_intermediate_bytes as f64 / (1024.0 * 1024.0)
529    ));
530
531    Ok(NativeEinsumPlanReport {
532        lines,
533        peak_intermediate_bytes,
534    })
535}
536
537fn native_einsum_time_optimized_plan_report(
538    signature: &NativeEinsumSignature,
539) -> Result<NativeEinsumPlanReport> {
540    native_einsum_plan_report_with_options(
541        signature,
542        "time_optimized",
543        &time_optimized_contraction_options(),
544    )
545}
546
547fn native_einsum_balanced_plan_report(
548    signature: &NativeEinsumSignature,
549) -> Result<NativeEinsumPlanReport> {
550    native_einsum_plan_report_with_options(
551        signature,
552        "balanced_default",
553        &ContractionOptimizerOptions::default(),
554    )
555}
556
557fn maybe_trace_native_einsum_path(
558    path: NativeEinsumPath,
559    operands: &[(&NativeTensor, &[u32])],
560    output_ids: &[u32],
561) {
562    if !native_einsum_path_trace_enabled() {
563        return;
564    }
565    let signature = native_einsum_signature(path, operands, output_ids);
566    let report = match native_einsum_time_optimized_plan_report(&signature) {
567        Ok(report) if report.peak_intermediate_bytes >= native_einsum_path_trace_min_bytes() => {
568            report
569        }
570        Ok(_) => return,
571        Err(err) => {
572            eprintln!("native_einsum path trace failed: {err:#}");
573            return;
574        }
575    };
576
577    let max_signatures = native_einsum_path_trace_max_signatures();
578    let should_trace = NATIVE_EINSUM_TRACE_STATE.with(|state| {
579        let mut state = state.borrow_mut();
580        if state.len() >= max_signatures || state.contains(&signature) {
581            false
582        } else {
583            state.insert(signature.clone());
584            true
585        }
586    });
587    if !should_trace {
588        return;
589    }
590
591    eprintln!("=== native_einsum Path Trace ===");
592    eprintln!(
593        "path={:?} output_ids={:?}",
594        signature.path, signature.output_ids
595    );
596    for operand in &signature.operands {
597        eprintln!(
598            "  operand shape={:?} ids={:?} dtype={:?}",
599            operand.shape, operand.ids, operand.dtype
600        );
601    }
602    for line in report.lines {
603        eprintln!("{line}");
604    }
605    if env::var("T4A_TRACE_NATIVE_EINSUM_COMPARE_BALANCED").is_ok() {
606        match native_einsum_balanced_plan_report(&signature) {
607            Ok(balanced) => {
608                for line in balanced.lines {
609                    eprintln!("{line}");
610                }
611            }
612            Err(err) => eprintln!("balanced native_einsum path trace failed: {err:#}"),
613        }
614    }
615}
616
617/// Reset the aggregated native einsum profile.
618pub fn reset_native_einsum_profile() {
619    NATIVE_EINSUM_PROFILE_STATE.with(|state| state.borrow_mut().clear());
620    NATIVE_EINSUM_TRACE_STATE.with(|state| state.borrow_mut().clear());
621}
622
623/// Print and clear the aggregated native einsum profile.
624pub fn print_and_reset_native_einsum_profile() {
625    if !native_einsum_profile_enabled() {
626        return;
627    }
628    NATIVE_EINSUM_PROFILE_STATE.with(|state| {
629        let mut entries: Vec<_> = state
630            .borrow()
631            .iter()
632            .map(|(k, v)| (k.clone(), v.clone()))
633            .collect();
634        state.borrow_mut().clear();
635        entries.sort_by_key(|(_, entry)| Reverse(entry.total_time));
636
637        eprintln!("=== native_einsum Profile ===");
638        for (idx, (signature, entry)) in entries.into_iter().take(20).enumerate() {
639            eprintln!(
640                "#{idx:02} path={:?} calls={} total={:.3}s per_call={:.3}us output_ids={:?}",
641                signature.path,
642                entry.calls,
643                entry.total_time.as_secs_f64(),
644                entry.total_time.as_secs_f64() * 1e6 / entry.calls as f64,
645                signature.output_ids,
646            );
647            for operand in &signature.operands {
648                eprintln!(
649                    "     shape={:?} ids={:?} dtype={:?}",
650                    operand.shape, operand.ids, operand.dtype
651                );
652            }
653            match native_einsum_time_optimized_plan_report(&signature) {
654                Ok(report) => {
655                    for line in report.lines {
656                        eprintln!("     {line}");
657                    }
658                }
659                Err(err) => eprintln!("     path report failed: {err:#}"),
660            }
661        }
662    });
663}
664
665fn convert_tensor(tensor: &NativeTensor, to: DType) -> Result<NativeTensor> {
666    with_default_session(|session| convert_native_tensor_in(session, tensor, to))
667        .map_err(|e| anyhow::Error::new(e).context(format!("tensor conversion to {to:?} failed")))
668}
669
670fn ids_to_subscript(ids: &[u32]) -> Result<String> {
671    const LETTERS: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
672    let mut out = String::with_capacity(ids.len());
673    for &id in ids {
674        let idx = usize::try_from(id).unwrap_or(usize::MAX);
675        let letter = LETTERS
676            .get(idx)
677            .ok_or_else(|| anyhow!("einsum label {id} exceeds supported label range"))?;
678        out.push(char::from(*letter));
679    }
680    Ok(out)
681}
682
683fn build_einsum_subscripts(operands: &[&[u32]], output_ids: &[u32]) -> Result<String> {
684    let inputs = operands
685        .iter()
686        .map(|ids| ids_to_subscript(ids))
687        .collect::<Result<Vec<_>>>()?;
688    Ok(format!(
689        "{}->{}",
690        inputs.join(","),
691        ids_to_subscript(output_ids)?
692    ))
693}
694
695fn compile_native_einsum_program(
696    compiler: &mut GraphCompiler,
697    input_specs: impl IntoIterator<Item = tenferro::program::ProgramInputSpec>,
698    subscripts: &EinsumSubscripts,
699) -> Result<tenferro::CompiledGraph> {
700    let input_specs = input_specs.into_iter().collect::<Vec<_>>();
701    let target = common_dtype(
702        input_specs
703            .iter()
704            .map(|spec| spec.metadata().dtype())
705            .collect::<Vec<_>>()
706            .as_slice(),
707    );
708    let mut trace = TraceContext::new();
709    let trace_inputs = input_specs
710        .into_iter()
711        .map(|spec| {
712            let dtype = spec.metadata().dtype();
713            let input = trace
714                .input(spec)
715                .map_err(|e| anyhow!("native einsum input tracing failed: {e}"))?;
716            if dtype == target {
717                return Ok(input);
718            }
719            trace
720                .add_op(
721                    CoreSemanticOp::Convert {
722                        from: dtype,
723                        to: target,
724                    },
725                    &[input],
726                )
727                .map_err(|e| anyhow!("native einsum promotion tracing failed: {e}"))?
728                .first()
729                .copied()
730                .ok_or_else(|| anyhow!("native einsum promotion returned no output"))
731        })
732        .collect::<Result<Vec<_>>>()?;
733    let output = trace
734        .einsum_subscripts(&trace_inputs, subscripts)
735        .map_err(|e| anyhow!("native einsum tracing failed: {e}"))?;
736    let graph = trace
737        .finish(&[output])
738        .map_err(|e| anyhow!("native einsum graph finalization failed: {e}"))?;
739    compiler
740        .compile_traced_graph(&graph)
741        .map_err(|e| anyhow!("native einsum graph compilation failed: {e}"))
742}
743
744fn run_cached_native_einsum(
745    subscripts: &EinsumSubscripts,
746    build: impl FnOnce(&mut GraphCompiler) -> Result<CompiledGraph>,
747    execute: impl FnOnce(&crate::context::CpuExecutionContext, &CompiledGraph) -> Result<NativeTensor>,
748) -> Result<NativeTensor> {
749    let trace_pool = native_einsum_pool_trace_enabled();
750    let pool_before = trace_pool
751        .then(default_engine_buffer_pool_stats)
752        .transpose()?;
753    // Compile under the compiler lock only; the graph then runs under the
754    // context's engine lock. Holding both would make a compile wait for a graph
755    // that is running.
756    let program = with_default_graph_compiler(build)?;
757    let result = execute(default_context(), &program)?;
758    if trace_pool {
759        let pool_after = default_engine_buffer_pool_stats()?;
760        let output_bytes = native_tensor_bytes(&result);
761        let retained_threshold = native_einsum_pool_trace_min_retained_bytes();
762        if pool_after != pool_before.unwrap_or_default()
763            && pool_after.capacity_bytes >= retained_threshold
764            || output_bytes >= native_einsum_pool_trace_min_output_bytes()
765        {
766            let before = pool_before.unwrap_or_default();
767            eprintln!(
768                "native_einsum pool subscripts={subscripts:?} before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB output_shape={:?} output_bytes={:.3} MiB",
769                before.buffers,
770                before.capacity_bytes as f64 / (1024.0 * 1024.0),
771                pool_after.buffers,
772                pool_after.capacity_bytes as f64 / (1024.0 * 1024.0),
773                result.shape(),
774                output_bytes as f64 / (1024.0 * 1024.0),
775            );
776        }
777    }
778    if reset_native_einsum_engine_after_call() {
779        let before_reset = trace_pool
780            .then(default_engine_buffer_pool_stats)
781            .transpose()?;
782        reset_default_engine()?;
783        if trace_pool
784            && before_reset.unwrap_or_default().capacity_bytes
785                >= native_einsum_pool_trace_min_retained_bytes()
786        {
787            let before = before_reset.unwrap_or_default();
788            let after = default_engine_buffer_pool_stats()?;
789            eprintln!(
790                "native_einsum engine_reset before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB",
791                before.buffers,
792                before.capacity_bytes as f64 / (1024.0 * 1024.0),
793                after.buffers,
794                after.capacity_bytes as f64 / (1024.0 * 1024.0),
795            );
796        }
797    } else if reset_native_einsum_buffer_pool_after_call() {
798        let before_clear = trace_pool
799            .then(default_engine_buffer_pool_stats)
800            .transpose()?;
801        reset_default_engine_buffer_pool()?;
802        if trace_pool
803            && before_clear.unwrap_or_default().capacity_bytes
804                >= native_einsum_pool_trace_min_retained_bytes()
805        {
806            let before = before_clear.unwrap_or_default();
807            let after = default_engine_buffer_pool_stats()?;
808            eprintln!(
809                "native_einsum pool_reset before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB",
810                before.buffers,
811                before.capacity_bytes as f64 / (1024.0 * 1024.0),
812                after.buffers,
813                after.capacity_bytes as f64 / (1024.0 * 1024.0),
814            );
815        }
816    }
817    if release_allocator_after_native_einsum_call() {
818        let report = release_process_allocator_cached_memory();
819        if trace_pool && (report.released_bytes.unwrap_or(0) > 0 || report.success == Some(true)) {
820            eprintln!(
821                "native_einsum allocator_pressure_relief supported={} released_bytes={:?} success={:?}",
822                report.supported,
823                report.released_bytes,
824                report.success,
825            );
826        }
827    }
828    Ok(result)
829}
830
831fn cached_einsum_native_tensors(
832    inputs: &[&NativeTensor],
833    subscripts: &EinsumSubscripts,
834) -> Result<NativeTensor> {
835    run_cached_native_einsum(
836        subscripts,
837        |compiler| {
838            compile_native_einsum_program(
839                compiler,
840                inputs.iter().map(|tensor| {
841                    tenferro::program::ProgramInputSpec::new(
842                        tensor.dtype(),
843                        tensor.shape().iter().copied().map(Into::into),
844                    )
845                }),
846                subscripts,
847            )
848        },
849        |context, program| single_output(context.run_graph(program, inputs)),
850    )
851}
852
853fn cached_einsum_native_reads(
854    inputs: &[TensorRead<'_>],
855    subscripts: &Subscripts,
856) -> Result<NativeTensor> {
857    let einsum_subscripts = EinsumSubscripts::from(subscripts);
858    execute_native_einsum_reads(inputs, subscripts, &einsum_subscripts)
859}
860
861fn execute_native_einsum_reads(
862    inputs: &[TensorRead<'_>],
863    subscripts: &Subscripts,
864    einsum_subscripts: &EinsumSubscripts,
865) -> Result<NativeTensor> {
866    if inputs.first().is_some_and(|first| {
867        concrete_einsum_dtype_supported(first.dtype())
868            && inputs.iter().all(|input| input.dtype() == first.dtype())
869    }) {
870        let plan = cached_concrete_einsum_plan(inputs, subscripts, einsum_subscripts)?;
871        // The plan reports `tenferro_einsum::Error`, so the session-entry
872        // rejection is mapped here rather than through the tenferro-tensor-typed
873        // session helper.
874        return default_context()
875            .with_session(|engine| plan.execute_read(inputs, engine))
876            .map_err(|entry| {
877                anyhow::Error::new(entry).context("native einsum session entry failed")
878            })?
879            .map_err(|error| anyhow!("native einsum session execution failed: {error}"));
880    }
881
882    run_cached_native_einsum(
883        einsum_subscripts,
884        |compiler| {
885            compile_native_einsum_program(
886                compiler,
887                inputs.iter().map(|tensor| {
888                    tenferro::program::ProgramInputSpec::new(
889                        tensor.dtype(),
890                        tensor.shape().iter().copied().map(Into::into),
891                    )
892                }),
893                einsum_subscripts,
894            )
895        },
896        |context, program| single_output(context.run_graph_reads(program, inputs)),
897    )
898    .map_err(|e| anyhow!("native read einsum failed: {e}"))
899}
900
901/// Reduce a graph's output list to the single tensor an einsum produces.
902fn single_output(
903    outputs: std::result::Result<Vec<NativeTensor>, crate::context::CpuExecutionContextError>,
904) -> Result<NativeTensor> {
905    let mut outputs = outputs.map_err(|e| anyhow!("native einsum failed: {e}"))?;
906    if outputs.len() != 1 {
907        return Err(anyhow!(
908            "native einsum returned {} outputs instead of one",
909            outputs.len()
910        ));
911    }
912    outputs
913        .pop()
914        .ok_or_else(|| anyhow!("native einsum returned no output"))
915}
916
917/// Build a dense native tensor from column-major data.
918/// # Errors
919///
920/// Returns an error when the shape product would overflow `usize`,
921/// when `data.len()` does not match the expected shape, or when the backend
922/// rejects the tensor.
923pub fn dense_native_tensor_from_col_major<T: TensorElement>(
924    data: &[T],
925    logical_dims: &[usize],
926) -> Result<NativeTensor> {
927    T::dense_native_tensor_from_col_major(data, logical_dims)
928}
929
930/// Build a dense native tensor by moving an owned column-major payload.
931///
932/// # Arguments
933///
934/// * `data` - Owned column-major values whose length equals the product of
935///   `logical_dims`.
936/// * `logical_dims` - Logical tensor dimensions in column-major axis order.
937///
938/// # Returns
939///
940/// A native dense tensor that owns the supplied payload without cloning it.
941///
942/// # Errors
943///
944/// Returns an error when the shape product would overflow `usize`,
945/// when `data.len()` does not match the expected shape, or when the backend
946/// rejects the tensor.
947pub fn dense_native_tensor_from_col_major_owned<T: TensorElement>(
948    data: Vec<T>,
949    logical_dims: &[usize],
950) -> Result<NativeTensor> {
951    T::dense_native_tensor_from_col_major_owned(data, logical_dims)
952}
953
954/// Borrow column-major data as a read-only native operand without copying.
955///
956/// The returned input owns nothing: it is a [`TensorRead`] view over `data`, so
957/// a shared operand (a frame matrix, a weight matrix) reaches an einsum as a
958/// borrowed read instead of a full-size copy. Use it with
959/// [`einsum_native_tensor_reads`] on the compatibility frontend, or
960/// [`NativeTensorReadInput::as_read`] to feed an explicit session's read route.
961///
962/// # Arguments
963///
964/// * `data` - Borrowed column-major values whose length equals the product of
965///   `logical_dims`.
966/// * `logical_dims` - Logical tensor dimensions in column-major axis order.
967///
968/// # Returns
969///
970/// A borrowed read input that stays valid for `data`'s lifetime.
971///
972/// # Errors
973///
974/// Returns an error when the shape product overflows `usize`, when `data.len()`
975/// does not match the expected shape, or when the view is rejected as invalid.
976///
977/// # Examples
978///
979/// ```
980/// use tensor4all_tensorbackend::{
981///     dense_native_tensor_read_input_from_col_major, einsum_native_tensor_reads,
982/// };
983///
984/// let lhs = [1.0_f64, 2.0, 3.0, 4.0];
985/// let rhs = [5.0_f64, 6.0];
986/// let lhs = dense_native_tensor_read_input_from_col_major(&lhs, &[2, 2])?;
987/// let rhs = dense_native_tensor_read_input_from_col_major(&rhs, &[2, 1])?;
988/// let product = einsum_native_tensor_reads(
989///     &[(&lhs, &[0, 1]), (&rhs, &[1, 2])],
990///     &[0, 2],
991/// )?;
992/// assert_eq!(product.as_slice::<f64>()?, &[23.0, 34.0]);
993/// # Ok::<(), Box<dyn std::error::Error>>(())
994/// ```
995pub fn dense_native_tensor_read_input_from_col_major<'a, T>(
996    data: &'a [T],
997    logical_dims: &[usize],
998) -> Result<NativeTensorReadInput<'a>>
999where
1000    T: TensorElement,
1001{
1002    let view = tenferro::TypedTensorView::from_col_major(logical_dims, data)
1003        .map_err(|error| anyhow!("native tensor view rejected: {error}"))?;
1004    Ok(NativeTensorReadInput::Borrowed(TensorRead::from_view(
1005        <T as TensorScalar>::tensor_view(view),
1006    )))
1007}
1008
1009/// Build a dense native tensor whose logical values are diagonal.
1010/// # Errors
1011///
1012/// Returns an error when `logical_rank` is zero, when expanding the diagonal
1013/// would overflow `usize`, or when the backend rejects the resulting dense
1014/// tensor.
1015pub fn diag_native_tensor_from_col_major<T: TensorElement>(
1016    data: &[T],
1017    logical_rank: usize,
1018) -> Result<NativeTensor> {
1019    T::diag_native_tensor_from_col_major(data, logical_rank)
1020}
1021
1022/// Convert storage to a dense native tensor.
1023///
1024/// The native tensor keeps the storage's own scalar dtype: a compact `f32` or
1025/// `Complex32` payload becomes an `f32`/`c32` native tensor instead of being
1026/// widened to `f64`/`Complex64`.
1027/// # Errors
1028///
1029/// Returns a [`BridgeError`] when materializing `storage` to the requested
1030/// dense shape fails (for example a shape mismatch) or when the
1031/// dense-to-native conversion fails.
1032pub fn storage_to_native_tensor(
1033    storage: &Storage,
1034    logical_dims: &[usize],
1035) -> std::result::Result<NativeTensor, BridgeError> {
1036    storage_dtype_dispatch!(storage.dtype(), Payload => {
1037        let values = <Payload as StorageScalar>::dense_values(storage, logical_dims)
1038            .map_err(|error| anyhow!("dense payload materialization failed: {error}"))?;
1039        <Payload as TensorElement>::dense_native_tensor_from_col_major_owned(
1040            values,
1041            logical_dims,
1042        )
1043        .map_err(BridgeError::from)
1044    },
1045    else Err(anyhow!("unsupported storage scalar type").into()))
1046}
1047
1048/// Copy the compact payload as a native tensor of the stored dtype.
1049///
1050/// This visits only referenced payload entries and preserves the stored scalar
1051/// dtype, so it is the dtype-generic payload bridge for compact storage.
1052/// # Errors
1053///
1054/// Returns a [`BridgeError`] when the payload cannot be copied or the
1055/// backend rejects the native tensor.
1056pub fn storage_payload_native(storage: &Storage) -> std::result::Result<NativeTensor, BridgeError> {
1057    storage_dtype_dispatch!(storage.dtype(), Payload => {
1058        let values = <Payload as StorageScalar>::payload_vec(storage)
1059            .map_err(|error| anyhow!("compact payload copy failed: {error}"))?;
1060        native_tensor_from_vec(storage.payload_dims().to_vec(), values)
1061    },
1062    else Err(anyhow!("unsupported storage scalar type").into()))
1063}
1064
1065/// Build structured storage from a native payload, preserving its dtype.
1066///
1067/// `output_payload_dims` and `output_axis_classes` describe the planned compact
1068/// layout. The compact payload keeps the native tensor's scalar dtype, so a
1069/// narrow contraction result stays narrow.
1070/// # Errors
1071///
1072/// Returns a [`BridgeError`] when the payload shape does not match
1073/// `output_payload_dims`, when its dtype has no compact storage representation,
1074/// or when the storage metadata or payload read fails.
1075pub fn storage_from_payload_native(
1076    payload: NativeTensor,
1077    output_payload_dims: &[usize],
1078    output_axis_classes: Vec<usize>,
1079) -> std::result::Result<Storage, BridgeError> {
1080    if payload.shape() != output_payload_dims {
1081        return Err(anyhow!(
1082            "payload shape {:?} does not match planned payload dims {:?}",
1083            payload.shape(),
1084            output_payload_dims
1085        )
1086        .into());
1087    }
1088    let strides = crate::storage::col_major_strides(output_payload_dims)?;
1089    storage_dtype_dispatch!(payload.dtype(), Payload => {
1090        let values = native_slice::<Payload>(&payload, "failed to read structured payload")?.to_vec();
1091        Storage::new_structured(
1092            values,
1093            output_payload_dims.to_vec(),
1094            strides,
1095            output_axis_classes,
1096        )
1097        .map_err(|error| anyhow!("structured payload storage construction failed: {error}").into())
1098    },
1099    else Err(anyhow!(
1100        "unsupported structured payload dtype {:?}; expected a floating or complex floating dtype",
1101        payload.dtype()
1102    )
1103    .into()))
1104}
1105
1106/// Build a read-only native tensor input over the compact storage payload.
1107///
1108/// Contiguous payloads are borrowed without copying. Non-contiguous payloads
1109/// are materialized into an owned native tensor. Both paths preserve the stored
1110/// scalar dtype, so `f32`/`Complex32` payloads are served without widening.
1111/// # Errors
1112///
1113/// Returns a [`BridgeError`] when the storage dtype is not a floating or
1114/// complex floating dtype, when the contiguous payload view cannot be built, or
1115/// when the payload cannot be read into a native tensor.
1116pub fn storage_payload_native_read_input(
1117    storage: &Storage,
1118) -> std::result::Result<NativeTensorReadInput<'_>, BridgeError> {
1119    storage_dtype_dispatch!(storage.dtype(), Payload => {
1120        if let Some(view) = <Payload as StorageScalar>::payload_view(storage)
1121            .map_err(anyhow::Error::msg)?
1122        {
1123            let typed = tenferro::TypedTensorView::from_col_major(storage.payload_dims(), view)
1124                .map_err(|error| BridgeError::from(anyhow::Error::new(error)))?;
1125            return Ok(NativeTensorReadInput::Borrowed(TensorRead::from_view(
1126                <Payload as TensorScalar>::tensor_view(typed),
1127            )));
1128        }
1129        let values = <Payload as StorageScalar>::payload_vec(storage)
1130            .map_err(|error| anyhow!("compact payload copy failed: {error}"))?;
1131        native_tensor_from_vec(storage.payload_dims().to_vec(), values)
1132            .map(NativeTensorReadInput::Owned)
1133    },
1134    else Err(anyhow!("unsupported storage scalar type").into()))
1135}
1136
1137/// Materialize a native tensor into dense storage.
1138/// # Errors
1139///
1140/// Returns a [`BridgeError`] when the tensor has an external scalar kind, when
1141/// reading its host slice fails, or when the dense storage materialization
1142/// fails.
1143pub fn native_tensor_primal_to_storage(
1144    tensor: &NativeTensor,
1145) -> std::result::Result<Storage, BridgeError> {
1146    match tensor.dtype() {
1147        DType::F32 => Storage::from_dense_col_major(
1148            native_slice::<f32>(tensor, "failed to read f32 native tensor")?.to_vec(),
1149            tensor.shape(),
1150        )
1151        .map_err(|e| {
1152            BridgeError::from(anyhow!(
1153                "native tensor snapshot materialization failed: {e}"
1154            ))
1155        }),
1156        DType::F64 => Storage::from_dense_col_major(
1157            native_slice::<f64>(tensor, "failed to read f64 native tensor")?.to_vec(),
1158            tensor.shape(),
1159        )
1160        .map_err(|e| {
1161            BridgeError::from(anyhow!(
1162                "native tensor snapshot materialization failed: {e}"
1163            ))
1164        }),
1165        DType::I32 => Storage::from_dense_col_major(
1166            native_slice::<i32>(tensor, "failed to read i32 native tensor")?
1167                .iter()
1168                .map(|&value| value as f64)
1169                .collect::<Vec<_>>(),
1170            tensor.shape(),
1171        )
1172        .map_err(|e| {
1173            BridgeError::from(anyhow!(
1174                "native tensor snapshot materialization failed: {e}"
1175            ))
1176        }),
1177        DType::I64 => Storage::from_dense_col_major(
1178            native_slice::<i64>(tensor, "failed to read i64 native tensor")?
1179                .iter()
1180                .map(|&value| value as f64)
1181                .collect::<Vec<_>>(),
1182            tensor.shape(),
1183        )
1184        .map_err(|e| {
1185            BridgeError::from(anyhow!(
1186                "native tensor snapshot materialization failed: {e}"
1187            ))
1188        }),
1189        DType::Bool => Storage::from_dense_col_major(
1190            native_slice::<bool>(tensor, "failed to read bool native tensor")?
1191                .iter()
1192                .map(|&value| if value { 1.0 } else { 0.0 })
1193                .collect::<Vec<_>>(),
1194            tensor.shape(),
1195        )
1196        .map_err(|e| {
1197            BridgeError::from(anyhow!(
1198                "native tensor snapshot materialization failed: {e}"
1199            ))
1200        }),
1201        DType::C32 => Storage::from_dense_col_major(
1202            native_slice::<Complex32>(tensor, "failed to read c32 native tensor")?.to_vec(),
1203            tensor.shape(),
1204        )
1205        .map_err(|e| {
1206            BridgeError::from(anyhow!(
1207                "native tensor snapshot materialization failed: {e}"
1208            ))
1209        }),
1210        DType::C64 => Storage::from_dense_col_major(
1211            native_slice::<Complex64>(tensor, "failed to read c64 native tensor")?.to_vec(),
1212            tensor.shape(),
1213        )
1214        .map_err(|e| {
1215            BridgeError::from(anyhow!(
1216                "native tensor snapshot materialization failed: {e}"
1217            ))
1218        }),
1219        DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
1220            tenferro_tensor::Error::unsupported(
1221                "dense storage",
1222                "external scalar kinds cannot be materialized into dense storage",
1223            ),
1224        ))),
1225    }
1226}
1227
1228/// Materialize dense column-major values from a native tensor.
1229/// # Errors
1230///
1231/// Returns a [`BridgeError`] when `tensor`'s dtype does not match `T` or when
1232/// its host data cannot be read.
1233pub fn native_tensor_primal_to_dense_col_major<T: TensorElement>(
1234    tensor: &NativeTensor,
1235) -> std::result::Result<Vec<T>, BridgeError> {
1236    let target = <T as TensorScalar>::dtype();
1237    let tensor_is_real = matches!(
1238        tensor.dtype(),
1239        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
1240    );
1241    let target_is_real = matches!(
1242        target,
1243        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
1244    );
1245    if tensor_is_real != target_is_real {
1246        return Err(anyhow!(
1247            "expected {} native tensor, got dtype {:?}",
1248            if target_is_real { "real" } else { "complex" },
1249            tensor.dtype()
1250        )
1251        .into());
1252    }
1253    <T as TensorElement>::dense_values_from_native_col_major(tensor).map_err(BridgeError::from)
1254}
1255
1256/// Materialize diagonal values from a native tensor, promoting to the
1257/// matching real (`f64`) or complex (`Complex64`) dtype.
1258///
1259/// Real native tensors (`f32`, `f64`, `i32`, `i64`, `bool`) are promoted to
1260/// `f64`; complex tensors (`c32`, `c64`) are promoted to `Complex64`. The
1261/// scalar type `T` selects the promoted target.
1262/// # Errors
1263///
1264/// Returns a [`BridgeError`] when the tensor's real/complex kind does not
1265/// match `T`, when conversion to `T` fails, or when the tensor is not
1266/// diagonal-compatible (rank 0 or unequal dimensions).
1267///
1268/// # Examples
1269/// ```
1270/// use tenferro::Tensor as NativeTensor;
1271/// use tensor4all_tensorbackend::native_tensor_primal_to_diag;
1272/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1273/// let native = NativeTensor::from_vec_col_major(vec![3, 3], vec![1.0_f64, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 3.0])?;
1274/// let diag = native_tensor_primal_to_diag::<f64>(&native)?;
1275/// assert_eq!(diag, vec![1.0, 2.0, 3.0]);
1276/// # Ok(())
1277/// # }
1278/// ```
1279pub fn native_tensor_primal_to_diag<T: TensorElement>(
1280    tensor: &NativeTensor,
1281) -> std::result::Result<Vec<T>, BridgeError> {
1282    let promote_to = <T as TensorScalar>::dtype();
1283    let tensor_is_real = matches!(
1284        tensor.dtype(),
1285        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
1286    );
1287    let target_is_real = matches!(
1288        promote_to,
1289        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
1290    );
1291    if tensor_is_real != target_is_real {
1292        return Err(anyhow!(
1293            "expected {} native tensor, got dtype {:?}",
1294            if target_is_real { "real" } else { "complex" },
1295            tensor.dtype()
1296        )
1297        .into());
1298    }
1299    let promoted = convert_tensor(tensor, promote_to)?;
1300    <T as TensorElement>::diag_values_from_native_temp(&promoted).map_err(BridgeError::from)
1301}
1302
1303/// Reshape a native tensor without changing its column-major linearization.
1304/// # Errors
1305///
1306/// Fails with a `CpuExecutionContextError` when the default context's engine is
1307/// already borrowed on this thread, or when the shape `logical_dims` and the
1308/// element count of `tensor` do not match.
1309pub fn reshape_col_major_native_tensor(
1310    tensor: &NativeTensor,
1311    logical_dims: &[usize],
1312) -> Result<NativeTensor> {
1313    with_default_session(|session| tensor.reshape(logical_dims, session))
1314        .map_err(|e| anyhow!("native reshape failed: {e}"))
1315}
1316
1317/// Compute a QR decomposition on a native tensor.
1318/// # Errors
1319///
1320/// Returns a [`BridgeError`] when the context rejects the entry or when
1321/// the QR factorization fails.
1322pub fn qr_native_tensor(
1323    tensor: &NativeTensor,
1324) -> std::result::Result<(NativeTensor, NativeTensor), BridgeError> {
1325    let (q, r) = with_default_session(|session| tensor.qr(session))
1326        .map_err(|e| anyhow!("native QR failed: {e}"))?;
1327    Ok((q, r))
1328}
1329
1330/// Compute an SVD on a native tensor.
1331/// # Errors
1332///
1333/// Fails with a `CpuExecutionContextError` when the default context's engine is
1334/// already borrowed on this thread, or when the backend SVD reports a
1335/// non-convergence failure.
1336pub fn svd_native_tensor(
1337    tensor: &NativeTensor,
1338) -> Result<(NativeTensor, NativeTensor, NativeTensor)> {
1339    let (u, s, vt) = with_default_session(|session| tensor.svd(session))
1340        .map_err(|e| anyhow!("native SVD failed: {e}"))?;
1341    Ok((u, s, vt))
1342}
1343
1344/// Sum all elements of a native tensor, returning a dynamic scalar.
1345/// # Errors
1346///
1347/// Returns a [`BridgeError`] when the backend reduction fails, when the
1348/// reduced result cannot read back its host slice, or when the scalar cannot
1349/// be wrapped as a [`BackendScalar`].
1350pub fn sum_native_tensor(tensor: &NativeTensor) -> std::result::Result<BackendScalar, BridgeError> {
1351    let reduced = if tensor.shape().is_empty() {
1352        tensor
1353            .duplicate()
1354            .map_err(|e| anyhow!("native scalar duplication failed: {e}"))?
1355    } else {
1356        let axes: Vec<usize> = (0..tensor.shape().len()).collect();
1357        with_default_session(|session| tensor.reduce_sum(Some(&axes), session))
1358            .map_err(|e| anyhow!("native sum failed: {e}"))?
1359    };
1360    Ok(BackendScalar::from_native(reduced)?)
1361}
1362
1363/// Return the tangent tensor when present.
1364///
1365/// Plain `Tensor` values do not carry tangent storage, so this bridge returns
1366/// `None`.
1367pub fn tangent_native_tensor(_tensor: &NativeTensor) -> Option<NativeTensor> {
1368    None
1369}
1370
1371/// Multiply a native tensor by a dynamic scalar.
1372/// # Errors
1373///
1374/// Returns a [`BridgeError`] when `tensor` or `scalar` cannot be promoted to a
1375/// common dtype, when the dtype is integer, bool, or external, or when the
1376/// backend rejects the scaled tensor.
1377pub fn scale_native_tensor(
1378    tensor: &NativeTensor,
1379    scalar: &BackendScalar,
1380) -> std::result::Result<NativeTensor, BridgeError> {
1381    let target = common_dtype(&[tensor.dtype(), scalar.as_native().dtype()]);
1382    let tensor = convert_tensor(tensor, target)?;
1383    let scalar = promote_scalar_native(scalar.as_native(), target)?;
1384
1385    match target {
1386        DType::F32 => {
1387            let factor = native_slice::<f32>(&scalar, "failed to read promoted f32 scalar")?
1388                .first()
1389                .copied()
1390                .ok_or_else(|| anyhow!("failed to read promoted f32 scalar"))?;
1391            let values = native_slice::<f32>(&tensor, "failed to read promoted f32 tensor")?
1392                .iter()
1393                .map(|&value| value * factor)
1394                .collect::<Vec<_>>();
1395            native_tensor_from_vec(tensor.shape().to_vec(), values)
1396        }
1397        DType::F64 => {
1398            let factor = native_slice::<f64>(&scalar, "failed to read promoted f64 scalar")?
1399                .first()
1400                .copied()
1401                .ok_or_else(|| anyhow!("failed to read promoted f64 scalar"))?;
1402            let values = native_slice::<f64>(&tensor, "failed to read promoted f64 tensor")?
1403                .iter()
1404                .map(|&value| value * factor)
1405                .collect::<Vec<_>>();
1406            native_tensor_from_vec(tensor.shape().to_vec(), values)
1407        }
1408        DType::C32 => {
1409            let factor = native_slice::<Complex32>(&scalar, "failed to read promoted c32 scalar")?
1410                .first()
1411                .copied()
1412                .ok_or_else(|| anyhow!("failed to read promoted c32 scalar"))?;
1413            let values = native_slice::<Complex32>(&tensor, "failed to read promoted c32 tensor")?
1414                .iter()
1415                .map(|&value| value * factor)
1416                .collect::<Vec<_>>();
1417            native_tensor_from_vec(tensor.shape().to_vec(), values)
1418        }
1419        DType::C64 => {
1420            let factor = native_slice::<Complex64>(&scalar, "failed to read promoted c64 scalar")?
1421                .first()
1422                .copied()
1423                .ok_or_else(|| anyhow!("failed to read promoted c64 scalar"))?;
1424            let values = native_slice::<Complex64>(&tensor, "failed to read promoted c64 tensor")?
1425                .iter()
1426                .map(|&value| value * factor)
1427                .collect::<Vec<_>>();
1428            native_tensor_from_vec(tensor.shape().to_vec(), values)
1429        }
1430        DType::I32 | DType::I64 | DType::Bool => {
1431            Err(anyhow!("scale_native_tensor does not support integer/bool tensors").into())
1432        }
1433        DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
1434            tenferro_tensor::Error::unsupported(
1435                "scaling",
1436                "external scalar kinds are not scalable dtypes",
1437            ),
1438        ))),
1439    }
1440}
1441
1442/// Compute `a * lhs + b * rhs`.
1443/// # Errors
1444///
1445/// Returns a [`BridgeError`] when `lhs` and `rhs` have different shapes, when
1446/// the operands cannot be promoted to a common dtype, when the dtype is
1447/// integer, bool, or external, or when the backend rejects the result.
1448pub fn axpby_native_tensor(
1449    lhs: &NativeTensor,
1450    a: &BackendScalar,
1451    rhs: &NativeTensor,
1452    b: &BackendScalar,
1453) -> std::result::Result<NativeTensor, BridgeError> {
1454    if lhs.shape() != rhs.shape() {
1455        return Err(BridgeError::from(anyhow!(
1456            "axpby requires matching tensor shapes, got lhs {:?} and rhs {:?}",
1457            lhs.shape(),
1458            rhs.shape()
1459        )));
1460    }
1461
1462    let target = common_dtype(&[
1463        lhs.dtype(),
1464        rhs.dtype(),
1465        a.as_native().dtype(),
1466        b.as_native().dtype(),
1467    ]);
1468    let lhs = convert_tensor(lhs, target)?;
1469    let rhs = convert_tensor(rhs, target)?;
1470    let a = promote_scalar_native(a.as_native(), target)?;
1471    let b = promote_scalar_native(b.as_native(), target)?;
1472
1473    match target {
1474        DType::F32 => {
1475            let a = native_slice::<f32>(&a, "failed to read promoted f32 scalar a")?
1476                .first()
1477                .copied()
1478                .ok_or_else(|| anyhow!("failed to read promoted f32 scalar a"))?;
1479            let b = native_slice::<f32>(&b, "failed to read promoted f32 scalar b")?
1480                .first()
1481                .copied()
1482                .ok_or_else(|| anyhow!("failed to read promoted f32 scalar b"))?;
1483            let lhs_values = native_slice::<f32>(&lhs, "failed to read promoted f32 lhs")?;
1484            let rhs_values = native_slice::<f32>(&rhs, "failed to read promoted f32 rhs")?;
1485            let values = lhs_values
1486                .iter()
1487                .zip(rhs_values.iter())
1488                .map(|(&x, &y)| a * x + b * y)
1489                .collect::<Vec<_>>();
1490            native_tensor_from_vec(lhs.shape().to_vec(), values)
1491        }
1492        DType::F64 => {
1493            let a = native_slice::<f64>(&a, "failed to read promoted f64 scalar a")?
1494                .first()
1495                .copied()
1496                .ok_or_else(|| anyhow!("failed to read promoted f64 scalar a"))?;
1497            let b = native_slice::<f64>(&b, "failed to read promoted f64 scalar b")?
1498                .first()
1499                .copied()
1500                .ok_or_else(|| anyhow!("failed to read promoted f64 scalar b"))?;
1501            let lhs_values = native_slice::<f64>(&lhs, "failed to read promoted f64 lhs")?;
1502            let rhs_values = native_slice::<f64>(&rhs, "failed to read promoted f64 rhs")?;
1503            let values = lhs_values
1504                .iter()
1505                .zip(rhs_values.iter())
1506                .map(|(&x, &y)| a * x + b * y)
1507                .collect::<Vec<_>>();
1508            native_tensor_from_vec(lhs.shape().to_vec(), values)
1509        }
1510        DType::C32 => {
1511            let a = native_slice::<Complex32>(&a, "failed to read promoted c32 scalar a")?
1512                .first()
1513                .copied()
1514                .ok_or_else(|| anyhow!("failed to read promoted c32 scalar a"))?;
1515            let b = native_slice::<Complex32>(&b, "failed to read promoted c32 scalar b")?
1516                .first()
1517                .copied()
1518                .ok_or_else(|| anyhow!("failed to read promoted c32 scalar b"))?;
1519            let lhs_values = native_slice::<Complex32>(&lhs, "failed to read promoted c32 lhs")?;
1520            let rhs_values = native_slice::<Complex32>(&rhs, "failed to read promoted c32 rhs")?;
1521            let values = lhs_values
1522                .iter()
1523                .zip(rhs_values.iter())
1524                .map(|(&x, &y)| a * x + b * y)
1525                .collect::<Vec<_>>();
1526            native_tensor_from_vec(lhs.shape().to_vec(), values)
1527        }
1528        DType::C64 => {
1529            let a = native_slice::<Complex64>(&a, "failed to read promoted c64 scalar a")?
1530                .first()
1531                .copied()
1532                .ok_or_else(|| anyhow!("failed to read promoted c64 scalar a"))?;
1533            let b = native_slice::<Complex64>(&b, "failed to read promoted c64 scalar b")?
1534                .first()
1535                .copied()
1536                .ok_or_else(|| anyhow!("failed to read promoted c64 scalar b"))?;
1537            let lhs_values = native_slice::<Complex64>(&lhs, "failed to read promoted c64 lhs")?;
1538            let rhs_values = native_slice::<Complex64>(&rhs, "failed to read promoted c64 rhs")?;
1539            let values = lhs_values
1540                .iter()
1541                .zip(rhs_values.iter())
1542                .map(|(&x, &y)| a * x + b * y)
1543                .collect::<Vec<_>>();
1544            native_tensor_from_vec(lhs.shape().to_vec(), values)
1545        }
1546        DType::I32 | DType::I64 | DType::Bool => {
1547            Err(anyhow!("axpby_native_tensor does not support integer/bool tensors").into())
1548        }
1549        DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
1550            tenferro_tensor::Error::unsupported(
1551                "axpby",
1552                "external scalar kinds are not axpby dtypes",
1553            ),
1554        ))),
1555    }
1556}
1557
1558/// Execute a cached einsum over owned native tensors.
1559///
1560/// This is the consuming bridge used by higher-level owned contraction APIs.
1561/// Inputs are promoted to a common dtype before tenferro evaluates the
1562/// contraction. Repeated calls with the same equation and shapes reuse
1563/// tenferro's process-global contraction path cache.
1564///
1565/// # Arguments
1566/// * `operands` - Native tensors paired with numeric einsum labels for each axis.
1567/// * `output_ids` - Numeric labels to keep in the result, in output axis order.
1568///
1569/// # Returns
1570/// The contracted native tensor in the promoted common dtype.
1571///
1572/// # Errors
1573/// Returns an error if the operand list is empty, any label list length does
1574/// not match its tensor rank, label generation exceeds the supported range, or
1575/// the backend contraction fails.
1576///
1577/// # Examples
1578/// ```
1579/// use tensor4all_tensorbackend::einsum_native_tensors_owned;
1580/// use tenferro::Tensor as NativeTensor;
1581/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1582///
1583/// let lhs = NativeTensor::from_vec_col_major(vec![2, 3], vec![1.0_f64; 6])?;
1584/// let rhs = NativeTensor::from_vec_col_major(vec![3, 2], vec![1.0_f64; 6])?;
1585/// let result = einsum_native_tensors_owned(vec![(lhs, vec![0, 1]), (rhs, vec![1, 2])], &[0, 2])?;
1586///
1587/// assert_eq!(result.shape(), &[2, 2]);
1588/// assert_eq!(result.as_slice::<f64>()?, &[3.0, 3.0, 3.0, 3.0]);
1589/// # Ok(())
1590/// # }
1591/// ```
1592pub fn einsum_native_tensors_owned(
1593    operands: Vec<(NativeTensor, Vec<usize>)>,
1594    output_ids: &[usize],
1595) -> Result<NativeTensor> {
1596    ensure!(
1597        !operands.is_empty(),
1598        "native einsum requires at least one operand"
1599    );
1600
1601    let target = common_dtype(
1602        &operands
1603            .iter()
1604            .map(|(tensor, _)| tensor.dtype())
1605            .collect::<Vec<_>>(),
1606    );
1607
1608    let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
1609    let mut converted = Vec::with_capacity(operands.len());
1610    let mut input_ids = Vec::with_capacity(operands.len());
1611    for (tensor, ids) in operands {
1612        ensure!(
1613            tensor.shape().len() == ids.len(),
1614            "einsum id list {:?} does not match tensor shape {:?}",
1615            ids,
1616            tensor.shape()
1617        );
1618        let checked_ids = checked_native_einsum_labels(&ids)?;
1619        let tensor = if tensor.dtype() == target {
1620            tensor
1621        } else {
1622            convert_tensor(&tensor, target)?
1623        };
1624        input_ids.push(checked_ids);
1625        converted.push(tensor);
1626    }
1627
1628    let input_slices = input_ids.iter().map(Vec::as_slice).collect::<Vec<_>>();
1629    let subscripts = EinsumSubscripts::new(&input_slices, &output_ids_u32);
1630
1631    let input_refs = converted.iter().collect::<Vec<_>>();
1632    let trace_operands = input_refs
1633        .iter()
1634        .zip(input_ids.iter())
1635        .map(|(tensor, ids)| (*tensor, ids.as_slice()))
1636        .collect::<Vec<_>>();
1637    maybe_trace_native_einsum_path(NativeEinsumPath::Owned, &trace_operands, &output_ids_u32);
1638    let started = Instant::now();
1639    let result = cached_einsum_native_tensors(&input_refs, &subscripts)?;
1640    record_native_einsum_profile(
1641        NativeEinsumPath::Owned,
1642        &trace_operands,
1643        &output_ids_u32,
1644        started.elapsed(),
1645    );
1646    Ok(result)
1647}
1648
1649/// Execute a cached einsum over borrowed native tensors.
1650///
1651/// Inputs are promoted to a common dtype before contraction. Operands that
1652/// already have the target dtype are passed to the backend by reference;
1653/// operands with another dtype are converted into temporary native tensors and
1654/// then borrowed for the contraction. Repeated calls with the same equation
1655/// and shapes reuse tenferro's process-global contraction path cache.
1656///
1657/// # Arguments
1658/// * `operands` - Native tensors paired with numeric einsum labels for each axis.
1659///
1660///   Each label slice must have the same length as the corresponding tensor rank.
1661/// * `output_ids` - Numeric labels to keep in the result, in output axis order.
1662///
1663/// # Returns
1664/// The contracted native tensor in the promoted common dtype.
1665///
1666/// # Errors
1667/// Returns an error if the operand list is empty, any label list length does
1668/// not match its tensor rank, label generation exceeds the supported range,
1669/// dtype conversion fails, or the backend contraction fails.
1670///
1671/// # Examples
1672/// ```
1673/// use tensor4all_tensorbackend::einsum_native_tensors;
1674/// use tenferro::Tensor as NativeTensor;
1675/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1676///
1677/// let lhs = NativeTensor::from_vec_col_major(vec![2, 3], vec![1.0_f64; 6])?;
1678/// let rhs = NativeTensor::from_vec_col_major(vec![3, 2], vec![1.0_f64; 6])?;
1679/// let result = einsum_native_tensors(&[(&lhs, &[0, 1]), (&rhs, &[1, 2])], &[0, 2])?;
1680///
1681/// assert_eq!(result.shape(), &[2, 2]);
1682/// assert_eq!(result.as_slice::<f64>()?, &[3.0, 3.0, 3.0, 3.0]);
1683/// # Ok(())
1684/// # }
1685/// ```
1686pub fn einsum_native_tensors(
1687    operands: &[(&NativeTensor, &[usize])],
1688    output_ids: &[usize],
1689) -> Result<NativeTensor> {
1690    ensure!(
1691        !operands.is_empty(),
1692        "native einsum requires at least one operand"
1693    );
1694
1695    let target = common_dtype(
1696        &operands
1697            .iter()
1698            .map(|(tensor, _)| tensor.dtype())
1699            .collect::<Vec<_>>(),
1700    );
1701    let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
1702    let mut converted = Vec::with_capacity(operands.len());
1703    let mut input_ids = Vec::with_capacity(operands.len());
1704    let mut has_conversions = false;
1705    let started = Instant::now();
1706
1707    for (tensor, ids) in operands {
1708        ensure!(
1709            tensor.shape().len() == ids.len(),
1710            "einsum id list {:?} does not match tensor shape {:?}",
1711            ids,
1712            tensor.shape()
1713        );
1714        input_ids.push(checked_native_einsum_labels(ids)?);
1715        if tensor.dtype() == target {
1716            converted.push(None);
1717        } else {
1718            converted.push(Some(convert_tensor(tensor, target)?));
1719            has_conversions = true;
1720        }
1721    }
1722
1723    let input_slices = input_ids.iter().map(Vec::as_slice).collect::<Vec<_>>();
1724    let subscripts = EinsumSubscripts::new(&input_slices, &output_ids_u32);
1725    let input_refs = operands
1726        .iter()
1727        .zip(converted.iter())
1728        .map(|((tensor, _), converted)| converted.as_ref().unwrap_or(*tensor))
1729        .collect::<Vec<_>>();
1730    let trace_path = if has_conversions {
1731        NativeEinsumPath::BorrowedWithConversions
1732    } else {
1733        NativeEinsumPath::Borrowed
1734    };
1735    let trace_operands = input_refs
1736        .iter()
1737        .zip(input_ids.iter())
1738        .map(|(tensor, ids)| (*tensor, ids.as_slice()))
1739        .collect::<Vec<_>>();
1740    maybe_trace_native_einsum_path(trace_path, &trace_operands, &output_ids_u32);
1741    let result = cached_einsum_native_tensors(&input_refs, &subscripts)?;
1742    record_native_einsum_profile(
1743        trace_path,
1744        &trace_operands,
1745        &output_ids_u32,
1746        started.elapsed(),
1747    );
1748    Ok(result)
1749}
1750
1751/// Execute a cached einsum over borrowed native tensors using string subscripts.
1752///
1753/// This is the shared string-notation entry: it parses `subscripts` into the
1754/// numeric labels that [`einsum_native_tensors`] consumes, checks that the
1755/// equation describes exactly the supplied operands, and then delegates the
1756/// contraction. Use it instead of re-implementing the parsing plus label
1757/// mapping in each crate; use [`einsum_native_tensors`] directly when the
1758/// labels are already numeric.
1759///
1760/// # Arguments
1761///
1762/// * `subscripts` - NumPy/PyTorch-style einsum notation such as `"ij,jk->ik"`
1763///   or `"ii->"` for a trace. Parenthesized contraction order is rejected; see
1764///   [`tenferro_einsum::Subscripts::parse`].
1765/// * `operands` - One native tensor per comma-separated input term, in term
1766///   order. Ranks must match the term lengths and repeated labels must have
1767///   equal extents.
1768///
1769/// # Returns
1770///
1771/// The contracted native tensor in the promoted common dtype of the operands.
1772///
1773/// # Errors
1774///
1775/// Returns an error when the notation parser rejects `subscripts` as invalid
1776/// notation, when the operand count does not match the number of comma-separated
1777/// input terms, when an input term's label count does not match its operand
1778/// rank, when repeated or contracted labels have incompatible shapes, or when
1779/// dtype promotion or the backend contraction fails.
1780///
1781/// # Compatibility frontend
1782///
1783/// Like [`einsum_native_tensors`], this runs on the retained process-global
1784/// (compatibility) frontend of
1785/// [tensor4all-rs#859](https://github.com/tensor4all/tensor4all-rs/issues/859):
1786/// it selects the default execution context and pays one graph dispatch per
1787/// call. It is meant for cold or per-stage paths, not per-element hot loops;
1788/// use [`CpuExecutionContext`](crate::CpuExecutionContext) sessions for those.
1789///
1790/// # Examples
1791///
1792/// ```
1793/// use tensor4all_tensorbackend::einsum_native_subscripts;
1794/// use tenferro::Tensor as NativeTensor;
1795/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1796///
1797/// // Column-major A = [[1, 2], [3, 4]] and B = [[5, 6], [7, 8]].
1798/// let lhs = NativeTensor::from_vec_col_major(vec![2, 2], vec![1.0_f64, 3.0, 2.0, 4.0])?;
1799/// let rhs = NativeTensor::from_vec_col_major(vec![2, 2], vec![5.0_f64, 7.0, 6.0, 8.0])?;
1800/// let product = einsum_native_subscripts("ij,jk->ik", &[&lhs, &rhs])?;
1801/// assert_eq!(product.shape(), &[2, 2]);
1802/// assert_eq!(product.as_slice::<f64>()?, &[19.0, 43.0, 22.0, 50.0]);
1803///
1804/// let trace = einsum_native_subscripts("ii->", &[&lhs])?;
1805/// assert_eq!(trace.as_slice::<f64>()?, &[5.0]);
1806/// # Ok(())
1807/// # }
1808/// ```
1809pub fn einsum_native_subscripts(
1810    subscripts: &str,
1811    operands: &[&NativeTensor],
1812) -> Result<NativeTensor> {
1813    let plan = NativeEinsumPlan::new(subscripts)?;
1814    let reads = operands
1815        .iter()
1816        .map(|tensor| TensorRead::from_tensor(tensor))
1817        .collect::<Vec<_>>();
1818    plan.execute_reads(&reads)
1819}
1820
1821/// A parsed einsum specification reusable across operand shapes and borrowed buffers.
1822///
1823/// The specification is parsed once; shape-specific owning execution plans are
1824/// prepared and cached by the backend. This is the minimal parsed-plan bridge
1825/// for <https://github.com/tensor4all/tensor4all-rs/issues/1065>.
1826/// Unlike an operand-bound concrete plan, this specification accepts new shapes.
1827///
1828/// # Examples
1829///
1830/// ```
1831/// use tensor4all_tensorbackend::NativeEinsumPlan;
1832/// use tenferro_tensor::{Tensor, TensorRead};
1833/// # fn main() -> anyhow::Result<()> {
1834/// let plan = NativeEinsumPlan::new("i->")?;
1835/// for (data, expected) in [(vec![1.0_f64, 2.0], 3.0), (vec![1.0, 2.0, 3.0], 6.0)] {
1836///     let operand = Tensor::from_vec_col_major([data.len()], data)?;
1837///     let sum = plan.execute_reads(&[TensorRead::from_tensor(&operand)])?;
1838///     assert_eq!(sum.as_slice::<f64>()?, &[expected]);
1839/// }
1840/// # Ok(())
1841/// # }
1842/// ```
1843#[derive(Debug)]
1844pub struct NativeEinsumPlan {
1845    spec: Arc<NativeEinsumSpec>,
1846}
1847
1848impl NativeEinsumPlan {
1849    /// Parse a fixed einsum expression without binding operand shapes or storage.
1850    ///
1851    /// The parse is shared per thread: a repeated call with the same expression string returns a
1852    /// plan over the already parsed specification instead of re-parsing it, so the string form can
1853    /// be called inside a loop.
1854    ///
1855    /// # Errors
1856    ///
1857    /// Returns an error for invalid subscripts, such as a missing explicit output term.
1858    ///
1859    /// # Examples
1860    ///
1861    /// ```
1862    /// use tensor4all_tensorbackend::NativeEinsumPlan;
1863    /// let error = NativeEinsumPlan::new("ij,jk").unwrap_err();
1864    /// assert!(error.to_string().contains("invalid einsum subscripts"));
1865    /// ```
1866    pub fn new(subscripts: &str) -> Result<Self> {
1867        Ok(Self {
1868            spec: cached_native_einsum_spec(subscripts)?,
1869        })
1870    }
1871
1872    /// Execute the parsed expression on borrowed inputs without copying them.
1873    ///
1874    /// # Errors
1875    ///
1876    /// Returns an error for empty inputs, operand-count or rank mismatches,
1877    /// incompatible label extents, or a backend execution failure. Mixed dtypes
1878    /// retain the promotion performed by the native einsum graph.
1879    ///
1880    /// # Examples
1881    ///
1882    /// ```
1883    /// use tensor4all_tensorbackend::NativeEinsumPlan;
1884    /// use tenferro_tensor::{Tensor, TensorRead};
1885    /// # fn main() -> anyhow::Result<()> {
1886    /// let plan = NativeEinsumPlan::new("i,i->")?;
1887    /// let lhs = Tensor::from_vec_col_major([2], vec![1.0_f64, 2.0])?;
1888    /// let rhs = Tensor::from_vec_col_major([2], vec![3.0_f64, 4.0])?;
1889    /// let dot = plan.execute_reads(&[TensorRead::from_tensor(&lhs), TensorRead::from_tensor(&rhs)])?;
1890    /// assert_eq!(dot.as_slice::<f64>()?, &[11.0]);
1891    /// # Ok(())
1892    /// # }
1893    /// ```
1894    pub fn execute_reads(&self, inputs: &[TensorRead<'_>]) -> Result<NativeTensor> {
1895        ensure!(
1896            self.spec.subscripts.inputs.len() == inputs.len(),
1897            "einsum subscripts describe {} input term(s) but {} operand(s) were given",
1898            self.spec.subscripts.inputs.len(),
1899            inputs.len()
1900        );
1901        ensure!(
1902            !inputs.is_empty(),
1903            "native einsum requires at least one operand"
1904        );
1905        for (tensor, ids) in inputs.iter().zip(&self.spec.subscripts.inputs) {
1906            ensure!(
1907                tensor.shape().len() == ids.len(),
1908                "einsum id list {:?} does not match tensor shape {:?}",
1909                ids,
1910                tensor.shape()
1911            );
1912        }
1913        execute_native_einsum_reads(inputs, &self.spec.subscripts, &self.spec.einsum_subscripts)
1914    }
1915}
1916
1917/// Execute a cached einsum over read-only native tensor inputs.
1918///
1919/// Backends consume borrowed host views inside their execution session. Mixed
1920/// dtypes are promoted by `Convert` nodes in the compiled einsum graph, so
1921/// non-contiguous operands remain borrowed until runtime execution.
1922/// # Errors
1923///
1924/// Returns an error when `operands` is empty, when a tensor rank does not
1925/// match its id list length, when a label is out of bounds for the supported
1926/// range, or when the backend einsum fails.
1927pub fn einsum_native_tensor_reads(
1928    operands: &[(&NativeTensorReadInput<'_>, &[usize])],
1929    output_ids: &[usize],
1930) -> Result<NativeTensor> {
1931    ensure!(
1932        !operands.is_empty(),
1933        "native einsum requires at least one operand"
1934    );
1935
1936    let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
1937    let mut input_ids = Vec::with_capacity(operands.len());
1938    let mut read_inputs = Vec::with_capacity(operands.len());
1939
1940    for (tensor, ids) in operands {
1941        ensure!(
1942            tensor.shape().len() == ids.len(),
1943            "einsum id list {:?} does not match tensor shape {:?}",
1944            ids,
1945            tensor.shape()
1946        );
1947        input_ids.push(checked_native_einsum_labels(ids)?);
1948        read_inputs.push(tensor.as_read());
1949    }
1950
1951    let subscripts = Subscripts {
1952        inputs: input_ids,
1953        output: output_ids_u32,
1954    };
1955    cached_einsum_native_reads(&read_inputs, &subscripts)
1956}
1957
1958/// Permute axes of a native tensor.
1959/// # Errors
1960///
1961/// Returns a [`BridgeError`] when `perm` is not a permutation of `tensor`'s
1962/// axes or when the backend transpose fails.
1963pub fn permute_native_tensor(
1964    tensor: &NativeTensor,
1965    perm: &[usize],
1966) -> std::result::Result<NativeTensor, BridgeError> {
1967    with_default_session(|session| tensor.transpose(perm, session))
1968        .map_err(|e| anyhow!("native permute failed: {e}"))
1969        .map_err(BridgeError::from)
1970}
1971
1972/// Contract two native tensors along matching axes.
1973/// # Errors
1974///
1975/// Returns an error when `axes_a`/`axes_b` are out of bounds or have differing
1976/// lengths, when the contracted dimensions do not match, or when the backend
1977/// einsum fails.
1978pub fn contract_native_tensor(
1979    lhs: &NativeTensor,
1980    axes_a: &[usize],
1981    rhs: &NativeTensor,
1982    axes_b: &[usize],
1983) -> Result<NativeTensor> {
1984    let (lhs_ids, rhs_ids, output_ids) =
1985        build_binary_einsum_ids(lhs.shape().len(), axes_a, rhs.shape().len(), axes_b)?;
1986    let lhs_ids_usize = lhs_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
1987    let rhs_ids_usize = rhs_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
1988    let output_ids_usize = output_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
1989    let operands = [
1990        (lhs, lhs_ids_usize.as_slice()),
1991        (rhs, rhs_ids_usize.as_slice()),
1992    ];
1993    einsum_native_tensors(&operands, &output_ids_usize)
1994}
1995
1996/// Compute the outer product of two native tensors.
1997/// # Errors
1998///
1999/// Returns a [`BridgeError`] when the underlying einsum contraction fails.
2000pub fn outer_product_native_tensor(
2001    lhs: &NativeTensor,
2002    rhs: &NativeTensor,
2003) -> std::result::Result<NativeTensor, BridgeError> {
2004    contract_native_tensor(lhs, &[], rhs, &[]).map_err(BridgeError::from)
2005}
2006
2007/// Conjugate a native tensor.
2008/// # Errors
2009///
2010/// Returns a [`BridgeError`] when the dtype is external, when duplicating a
2011/// real tensor fails, or when the host slice of a complex tensor cannot be
2012/// read.
2013pub fn conj_native_tensor(tensor: &NativeTensor) -> std::result::Result<NativeTensor, BridgeError> {
2014    match tensor.dtype() {
2015        DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool => tensor
2016            .duplicate()
2017            .map_err(|e| anyhow!("native tensor duplication failed: {e}"))
2018            .map_err(BridgeError::from),
2019        DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
2020            tenferro_tensor::Error::unsupported(
2021                "conjugation",
2022                "external scalar kinds have no conjugation",
2023            ),
2024        ))),
2025        DType::C32 => native_tensor_from_vec(
2026            tensor.shape().to_vec(),
2027            native_slice::<Complex32>(tensor, "failed to read c32 native tensor")?
2028                .iter()
2029                .map(|&value| value.conj())
2030                .collect::<Vec<_>>(),
2031        ),
2032        DType::C64 => native_tensor_from_vec(
2033            tensor.shape().to_vec(),
2034            native_slice::<Complex64>(tensor, "failed to read c64 native tensor")?
2035                .iter()
2036                .map(|&value| value.conj())
2037                .collect::<Vec<_>>(),
2038        ),
2039    }
2040}
2041
2042/// Permute storage by round-tripping through native tensors.
2043/// # Errors
2044///
2045/// Returns a [`BridgeError`] when the storage-to-native conversion, the
2046/// permutation, or the native-to-storage conversion fails.
2047pub fn permute_storage_native(
2048    storage: &Storage,
2049    logical_dims: &[usize],
2050    perm: &[usize],
2051) -> std::result::Result<Storage, BridgeError> {
2052    let native = storage_to_native_tensor(storage, logical_dims)?;
2053    let permuted = permute_native_tensor(&native, perm)?;
2054    native_tensor_primal_to_storage(&permuted)
2055}
2056
2057/// Contract storages via native tensors.
2058/// # Errors
2059///
2060/// Returns a [`BridgeError`] when either storage-to-native conversion, the
2061/// contraction, or the native-to-storage conversion fails.
2062pub fn contract_storage_native(
2063    storage_a: &Storage,
2064    dims_a: &[usize],
2065    axes_a: &[usize],
2066    storage_b: &Storage,
2067    dims_b: &[usize],
2068    axes_b: &[usize],
2069    _result_dims: &[usize],
2070) -> std::result::Result<Storage, BridgeError> {
2071    let lhs = storage_to_native_tensor(storage_a, dims_a)?;
2072    let rhs = storage_to_native_tensor(storage_b, dims_b)?;
2073    let result = contract_native_tensor(&lhs, axes_a, &rhs, axes_b)?;
2074    native_tensor_primal_to_storage(&result)
2075}
2076
2077/// Outer-product storages via native tensors.
2078/// # Errors
2079///
2080/// Returns a [`BridgeError`] when either storage-to-native conversion, the
2081/// outer product, or the native-to-storage conversion fails.
2082pub fn outer_product_storage_native(
2083    lhs: &Storage,
2084    lhs_dims: &[usize],
2085    rhs: &Storage,
2086    rhs_dims: &[usize],
2087    _result_dims: &[usize],
2088) -> std::result::Result<Storage, BridgeError> {
2089    let lhs = storage_to_native_tensor(lhs, lhs_dims)?;
2090    let rhs = storage_to_native_tensor(rhs, rhs_dims)?;
2091    let result = outer_product_native_tensor(&lhs, &rhs)?;
2092    native_tensor_primal_to_storage(&result)
2093}
2094
2095/// Scale storage by a scalar via native tensors.
2096/// # Errors
2097///
2098/// Returns a [`BridgeError`] when the storage-to-native conversion, the
2099/// scaling, or the native-to-storage conversion fails.
2100pub fn scale_storage_native(
2101    storage: &Storage,
2102    logical_dims: &[usize],
2103    scalar: &BackendScalar,
2104) -> std::result::Result<Storage, BridgeError> {
2105    let native = storage_to_native_tensor(storage, logical_dims)?;
2106    let scaled = scale_native_tensor(&native, scalar)?;
2107    native_tensor_primal_to_storage(&scaled)
2108}
2109
2110/// Compute `a * lhs + b * rhs` over storages via native tensors.
2111/// # Errors
2112///
2113/// Returns a [`BridgeError`] when either storage-to-native conversion, the
2114/// axpby, or the native-to-storage conversion fails.
2115pub fn axpby_storage_native(
2116    lhs: &Storage,
2117    lhs_dims: &[usize],
2118    a: &BackendScalar,
2119    rhs: &Storage,
2120    rhs_dims: &[usize],
2121    b: &BackendScalar,
2122) -> std::result::Result<Storage, BridgeError> {
2123    let lhs = storage_to_native_tensor(lhs, lhs_dims)?;
2124    let rhs = storage_to_native_tensor(rhs, rhs_dims)?;
2125    let combined = axpby_native_tensor(&lhs, a, &rhs, b)?;
2126    native_tensor_primal_to_storage(&combined)
2127}
2128
2129#[cfg(test)]
2130mod tests;