1use std::cell::RefCell;
4use std::cmp::Reverse;
5use std::collections::{HashMap, HashSet};
6use std::env;
7use std::sync::Arc;
8use std::time::{Duration, Instant};
9
10use anyhow::{anyhow, ensure, Result};
11use num_complex::{Complex32, Complex64};
12use omeco::ScoreFunction;
13use tenferro::{
14 program::CoreSemanticOp, CompiledGraph, DType, GraphCompiler, Tensor as NativeTensor,
15 TensorRead, TensorScalar, TensorSessionOpsExt, TraceContext,
16};
17use tenferro_einsum::{
18 ConcreteEinsumPlan, ContractionOptimizerOptions, ContractionTree, EinsumSubscripts, Subscripts,
19 TraceContextEinsumExt,
20};
21use tenferro_linalg::TensorLinalgExt;
22
23use crate::any_scalar::promote_scalar_native;
24pub(crate) use crate::einsum_ids::{
25 build_binary_einsum_ids, checked_native_einsum_labels, common_dtype, convert_native_tensor_in,
26};
27#[derive(Debug, thiserror::Error)]
39#[error("native tensor bridge operation failed: {source}")]
40pub struct BridgeError {
41 #[source]
43 pub source: anyhow::Error,
44}
45
46impl From<anyhow::Error> for BridgeError {
47 fn from(source: anyhow::Error) -> Self {
48 Self { source }
49 }
50}
51
52fn native_tensor_from_vec<T: TensorScalar>(
53 shape: Vec<usize>,
54 values: Vec<T>,
55) -> std::result::Result<NativeTensor, BridgeError> {
56 NativeTensor::from_vec_col_major(shape, values)
57 .map_err(|source| BridgeError::from(anyhow::Error::new(source)))
58}
59
60use crate::context::{
61 default_context, default_engine_buffer_pool_stats, reset_default_engine,
62 reset_default_engine_buffer_pool, with_default_graph_compiler, with_default_session,
63};
64use crate::memory::release_process_allocator_cached_memory;
65use crate::storage::{storage_dtype_dispatch, Storage, StorageScalar};
66use crate::tensor_element::TensorElement;
67use crate::BackendScalar;
68
69#[allow(clippy::large_enum_variant)]
74pub enum NativeTensorReadInput<'a> {
75 Borrowed(TensorRead<'a>),
77 Owned(NativeTensor),
79}
80
81impl<'a> NativeTensorReadInput<'a> {
82 pub fn as_read(&'a self) -> TensorRead<'a> {
84 match self {
85 Self::Borrowed(read) => read.clone(),
86 Self::Owned(tensor) => TensorRead::from_tensor(tensor),
87 }
88 }
89
90 pub fn dtype(&self) -> DType {
92 match self {
93 Self::Borrowed(read) => read.dtype(),
94 Self::Owned(tensor) => tensor.dtype(),
95 }
96 }
97
98 pub fn shape(&self) -> &[usize] {
100 match self {
101 Self::Borrowed(read) => read.shape(),
102 Self::Owned(tensor) => tensor.shape(),
103 }
104 }
105}
106
107#[cfg(test)]
108use std::cell::Cell;
109
110#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
111enum NativeEinsumPath {
112 Owned,
113 Borrowed,
114 BorrowedWithConversions,
115}
116
117#[derive(Debug, Clone, Hash, PartialEq, Eq)]
118struct NativeOperandSignature {
119 shape: Vec<usize>,
120 ids: Vec<u32>,
121 dtype: DType,
122}
123
124#[derive(Debug, Clone, Hash, PartialEq, Eq)]
125struct NativeEinsumSignature {
126 path: NativeEinsumPath,
127 operands: Vec<NativeOperandSignature>,
128 output_ids: Vec<u32>,
129}
130
131#[derive(Debug, Default, Clone)]
132struct NativeEinsumProfileEntry {
133 calls: usize,
134 total_time: Duration,
135}
136
137const CONCRETE_EINSUM_PLAN_CACHE_CAPACITY: usize = 4096;
142
143#[derive(Debug)]
145struct NativeEinsumSpec {
146 subscripts: Subscripts,
147 einsum_subscripts: EinsumSubscripts,
148}
149
150const NATIVE_EINSUM_SPEC_CACHE_CAPACITY: usize = 1024;
153
154thread_local! {
155 static NATIVE_EINSUM_SPEC_CACHE: RefCell<HashMap<String, Arc<NativeEinsumSpec>>> =
156 RefCell::new(HashMap::new());
157 static NATIVE_EINSUM_PROFILE_STATE: RefCell<HashMap<NativeEinsumSignature, NativeEinsumProfileEntry>> =
158 RefCell::new(HashMap::new());
159 static NATIVE_EINSUM_TRACE_STATE: RefCell<HashSet<NativeEinsumSignature>> =
160 RefCell::new(HashSet::new());
161 static CONCRETE_EINSUM_PLAN_CACHE: RefCell<HashMap<NativeEinsumSignature, Arc<ConcreteEinsumPlan>>> =
162 RefCell::new(HashMap::new());
163}
164
165#[cfg(test)]
166thread_local! {
167 static FORCE_NATIVE_EINSUM_PROFILE: Cell<bool> = const { Cell::new(false) };
168}
169
170fn native_einsum_profile_enabled() -> bool {
171 #[cfg(test)]
172 if FORCE_NATIVE_EINSUM_PROFILE.with(Cell::get) {
173 return true;
174 }
175 env::var("T4A_PROFILE_NATIVE_EINSUM").is_ok()
176}
177
178fn native_einsum_path_trace_enabled() -> bool {
179 env::var("T4A_TRACE_NATIVE_EINSUM_PATHS").is_ok()
180}
181
182fn native_einsum_path_trace_min_bytes() -> usize {
183 env::var("T4A_TRACE_NATIVE_EINSUM_MIN_BYTES")
184 .ok()
185 .and_then(|value| value.parse().ok())
186 .unwrap_or(0)
187}
188
189fn native_einsum_path_trace_max_signatures() -> usize {
190 env::var("T4A_TRACE_NATIVE_EINSUM_MAX_SIGNATURES")
191 .ok()
192 .and_then(|value| value.parse().ok())
193 .unwrap_or(64)
194}
195
196fn native_einsum_pool_trace_enabled() -> bool {
197 env::var("T4A_TRACE_NATIVE_EINSUM_POOL").is_ok()
198}
199
200fn native_einsum_pool_trace_min_output_bytes() -> usize {
201 env::var("T4A_TRACE_NATIVE_EINSUM_POOL_MIN_OUTPUT_BYTES")
202 .ok()
203 .and_then(|value| value.parse().ok())
204 .unwrap_or(0)
205}
206
207fn native_einsum_pool_trace_min_retained_bytes() -> usize {
208 env::var("T4A_TRACE_NATIVE_EINSUM_POOL_MIN_RETAINED_BYTES")
209 .ok()
210 .and_then(|value| value.parse().ok())
211 .unwrap_or(0)
212}
213
214fn reset_native_einsum_engine_after_call() -> bool {
215 env::var("T4A_RESET_NATIVE_EINSUM_ENGINE_AFTER_CALL").is_ok()
216}
217
218fn reset_native_einsum_buffer_pool_after_call() -> bool {
219 env::var("T4A_RESET_NATIVE_EINSUM_BUFFER_POOL_AFTER_CALL").is_ok()
220}
221
222fn release_allocator_after_native_einsum_call() -> bool {
223 env::var("T4A_RELEASE_ALLOCATOR_AFTER_NATIVE_EINSUM_CALL").is_ok()
224}
225
226#[cfg(test)]
227pub(crate) fn set_native_einsum_profile_enabled_for_tests(enabled: bool) {
228 FORCE_NATIVE_EINSUM_PROFILE.with(|slot| slot.set(enabled));
229}
230
231fn native_einsum_signature(
232 path: NativeEinsumPath,
233 operands: &[(&NativeTensor, &[u32])],
234 output_ids: &[u32],
235) -> NativeEinsumSignature {
236 NativeEinsumSignature {
237 path,
238 operands: operands
239 .iter()
240 .map(|(tensor, ids)| NativeOperandSignature {
241 shape: tensor.shape().to_vec(),
242 ids: ids.to_vec(),
243 dtype: tensor.dtype(),
244 })
245 .collect(),
246 output_ids: output_ids.to_vec(),
247 }
248}
249
250fn concrete_einsum_dtype_supported(dtype: DType) -> bool {
251 matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64)
252}
253
254fn concrete_einsum_signature(
255 inputs: &[TensorRead<'_>],
256 subscripts: &Subscripts,
257) -> NativeEinsumSignature {
258 NativeEinsumSignature {
261 path: NativeEinsumPath::Borrowed,
262 operands: inputs
263 .iter()
264 .zip(subscripts.inputs.iter())
265 .map(|(tensor, ids)| NativeOperandSignature {
266 shape: tensor.shape().to_vec(),
267 ids: ids.clone(),
268 dtype: tensor.dtype(),
269 })
270 .collect(),
271 output_ids: subscripts.output.clone(),
272 }
273}
274
275fn cached_native_einsum_spec(subscripts: &str) -> Result<Arc<NativeEinsumSpec>> {
277 if let Some(spec) =
278 NATIVE_EINSUM_SPEC_CACHE.with(|cache| cache.borrow().get(subscripts).cloned())
279 {
280 return Ok(spec);
281 }
282 let parsed = Subscripts::parse(subscripts)
283 .map_err(|error| anyhow!("invalid einsum subscripts '{subscripts}': {error}"))?;
284 let spec = Arc::new(NativeEinsumSpec {
285 einsum_subscripts: EinsumSubscripts::from(&parsed),
286 subscripts: parsed,
287 });
288 NATIVE_EINSUM_SPEC_CACHE.with(|cache| {
289 let mut cache = cache.borrow_mut();
290 if cache.len() >= NATIVE_EINSUM_SPEC_CACHE_CAPACITY {
291 cache.clear();
292 }
293 cache.insert(subscripts.to_string(), Arc::clone(&spec));
294 });
295 Ok(spec)
296}
297
298fn cached_concrete_einsum_plan(
299 inputs: &[TensorRead<'_>],
300 subscripts: &Subscripts,
301 einsum_subscripts: &EinsumSubscripts,
302) -> Result<Arc<ConcreteEinsumPlan>> {
303 let signature = concrete_einsum_signature(inputs, subscripts);
304 if let Some(plan) =
305 CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow().get(&signature).cloned())
306 {
307 return Ok(plan);
308 }
309
310 let plan = Arc::new(
311 ConcreteEinsumPlan::prepare_read_subscripts(inputs, einsum_subscripts)
312 .map_err(|error| anyhow!("native einsum session plan preparation failed: {error}"))?,
313 );
314 CONCRETE_EINSUM_PLAN_CACHE.with(|cache| {
315 let mut cache = cache.borrow_mut();
316 if cache.len() >= CONCRETE_EINSUM_PLAN_CACHE_CAPACITY {
318 cache.clear();
319 }
320 cache.insert(signature, Arc::clone(&plan));
321 });
322 Ok(plan)
323}
324
325#[cfg(test)]
326fn reset_concrete_einsum_plan_cache() {
327 CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow_mut().clear());
328}
329
330#[cfg(test)]
331fn native_einsum_spec_cache_len() -> usize {
332 NATIVE_EINSUM_SPEC_CACHE.with(|cache| cache.borrow().len())
333}
334
335#[cfg(test)]
336fn concrete_einsum_plan_cache_len() -> usize {
337 CONCRETE_EINSUM_PLAN_CACHE.with(|cache| cache.borrow().len())
338}
339
340fn record_native_einsum_profile(
341 path: NativeEinsumPath,
342 operands: &[(&NativeTensor, &[u32])],
343 output_ids: &[u32],
344 elapsed: Duration,
345) {
346 if !native_einsum_profile_enabled() {
347 return;
348 }
349 let signature = native_einsum_signature(path, operands, output_ids);
350 NATIVE_EINSUM_PROFILE_STATE.with(|state| {
351 let mut state = state.borrow_mut();
352 let entry = state.entry(signature).or_default();
353 entry.calls += 1;
354 entry.total_time += elapsed;
355 });
356}
357
358fn native_slice<'a, T: TensorScalar>(
359 tensor: &'a NativeTensor,
360 label: &'static str,
361) -> Result<&'a [T]> {
362 tensor
363 .as_slice::<T>()
364 .map_err(|error| anyhow!("{label}: {error}"))
365}
366
367fn dtype_size_bytes(dtype: DType) -> usize {
368 match dtype {
369 DType::F32 => 4,
370 DType::F64 => 8,
371 DType::C32 => 8,
372 DType::C64 => 16,
373 DType::I32 => 4,
374 DType::I64 => 8,
375 DType::Bool => 1,
376 DType::External(_) => 0,
380 }
381}
382
383fn native_tensor_bytes(tensor: &NativeTensor) -> usize {
384 tensor
385 .shape()
386 .iter()
387 .copied()
388 .fold(1usize, usize::saturating_mul)
389 .saturating_mul(dtype_size_bytes(tensor.dtype()))
390}
391
392fn format_label(label: u32) -> String {
393 char::from_u32(label).map_or_else(|| label.to_string(), |label| label.to_string())
394}
395
396fn format_labels(labels: &[u32]) -> String {
397 if labels.is_empty() {
398 "scalar".to_string()
399 } else {
400 labels
401 .iter()
402 .map(|&label| format_label(label))
403 .collect::<Vec<_>>()
404 .join("")
405 }
406}
407
408fn label_dims(subscripts: &Subscripts, shapes: &[Vec<usize>]) -> Result<HashMap<u32, usize>> {
409 let mut dims = HashMap::new();
410 for (labels, shape) in subscripts.inputs.iter().zip(shapes.iter()) {
411 ensure!(
412 labels.len() == shape.len(),
413 "einsum labels {:?} do not match shape {:?}",
414 labels,
415 shape
416 );
417 for (&label, &dim) in labels.iter().zip(shape.iter()) {
418 if let Some(previous) = dims.insert(label, dim) {
419 ensure!(
420 previous == dim,
421 "inconsistent dimension for einsum label {}: {} vs {}",
422 format_label(label),
423 previous,
424 dim
425 );
426 }
427 }
428 }
429 Ok(dims)
430}
431
432fn labels_size(labels: &[u32], dims: &HashMap<u32, usize>) -> usize {
433 labels.iter().fold(1usize, |size, label| {
434 size.saturating_mul(dims.get(label).copied().unwrap_or(1))
435 })
436}
437
438fn union_labels(lhs: &[u32], rhs: &[u32]) -> Vec<u32> {
439 let mut seen = HashSet::new();
440 let mut labels = Vec::new();
441 for &label in lhs.iter().chain(rhs.iter()) {
442 if seen.insert(label) {
443 labels.push(label);
444 }
445 }
446 labels
447}
448
449#[derive(Debug)]
450struct NativeEinsumPlanReport {
451 lines: Vec<String>,
452 peak_intermediate_bytes: usize,
453}
454
455fn time_optimized_contraction_options() -> ContractionOptimizerOptions {
456 ContractionOptimizerOptions {
457 score: ScoreFunction::time_optimized(),
458 ..ContractionOptimizerOptions::default()
459 }
460}
461
462fn native_einsum_plan_report_with_options(
463 signature: &NativeEinsumSignature,
464 optimizer_name: &'static str,
465 options: &ContractionOptimizerOptions,
466) -> Result<NativeEinsumPlanReport> {
467 let input_ids = signature
468 .operands
469 .iter()
470 .map(|operand| operand.ids.as_slice())
471 .collect::<Vec<_>>();
472 let subscripts_string = build_einsum_subscripts(&input_ids, &signature.output_ids)?;
473 let subscripts = Subscripts {
474 inputs: input_ids.iter().map(|ids| ids.to_vec()).collect(),
475 output: signature.output_ids.clone(),
476 };
477 let shapes = signature
478 .operands
479 .iter()
480 .map(|operand| operand.shape.clone())
481 .collect::<Vec<_>>();
482 let shape_refs = shapes.iter().map(Vec::as_slice).collect::<Vec<_>>();
483 let tree = ContractionTree::optimize_with_options(&subscripts, &shape_refs, options)
484 .map_err(|e| anyhow!("failed to optimize native einsum path: {e}"))?;
485 let dims = label_dims(&subscripts, &shapes)?;
486 let dtype = signature
487 .operands
488 .first()
489 .map(|operand| operand.dtype)
490 .unwrap_or(DType::F64);
491 let dtype_size = dtype_size_bytes(dtype);
492
493 let mut lines = Vec::new();
494 lines.push(format!(
495 "optimizer={optimizer_name} subscripts={subscripts_string} dtype={dtype:?} steps={}",
496 tree.step_count()
497 ));
498 let mut peak_intermediate_elems = 1usize;
499 for step in 0..tree.step_count() {
500 let Some((left, right)) = tree.step_pair(step) else {
501 continue;
502 };
503 let Some((lhs, rhs, out)) = tree.step_subscripts(step) else {
504 continue;
505 };
506 let lhs_elems = labels_size(lhs, &dims);
507 let rhs_elems = labels_size(rhs, &dims);
508 let out_elems = labels_size(out, &dims);
509 let flop_index_elems = labels_size(&union_labels(lhs, rhs), &dims);
510 peak_intermediate_elems = peak_intermediate_elems.max(out_elems);
511 lines.push(format!(
512 " step {step:02}: pair=({left},{right}) {}[{}] x {}[{}] -> {}[{}] flop_index={} intermediate={} elems ({:.3} MiB)",
513 format_labels(lhs),
514 lhs_elems,
515 format_labels(rhs),
516 rhs_elems,
517 format_labels(out),
518 out_elems,
519 flop_index_elems,
520 out_elems,
521 out_elems as f64 * dtype_size as f64 / (1024.0 * 1024.0),
522 ));
523 }
524 let peak_intermediate_bytes = peak_intermediate_elems.saturating_mul(dtype_size);
525 lines.push(format!(
526 " peak_intermediate={} elems ({:.3} MiB)",
527 peak_intermediate_elems,
528 peak_intermediate_bytes as f64 / (1024.0 * 1024.0)
529 ));
530
531 Ok(NativeEinsumPlanReport {
532 lines,
533 peak_intermediate_bytes,
534 })
535}
536
537fn native_einsum_time_optimized_plan_report(
538 signature: &NativeEinsumSignature,
539) -> Result<NativeEinsumPlanReport> {
540 native_einsum_plan_report_with_options(
541 signature,
542 "time_optimized",
543 &time_optimized_contraction_options(),
544 )
545}
546
547fn native_einsum_balanced_plan_report(
548 signature: &NativeEinsumSignature,
549) -> Result<NativeEinsumPlanReport> {
550 native_einsum_plan_report_with_options(
551 signature,
552 "balanced_default",
553 &ContractionOptimizerOptions::default(),
554 )
555}
556
557fn maybe_trace_native_einsum_path(
558 path: NativeEinsumPath,
559 operands: &[(&NativeTensor, &[u32])],
560 output_ids: &[u32],
561) {
562 if !native_einsum_path_trace_enabled() {
563 return;
564 }
565 let signature = native_einsum_signature(path, operands, output_ids);
566 let report = match native_einsum_time_optimized_plan_report(&signature) {
567 Ok(report) if report.peak_intermediate_bytes >= native_einsum_path_trace_min_bytes() => {
568 report
569 }
570 Ok(_) => return,
571 Err(err) => {
572 eprintln!("native_einsum path trace failed: {err:#}");
573 return;
574 }
575 };
576
577 let max_signatures = native_einsum_path_trace_max_signatures();
578 let should_trace = NATIVE_EINSUM_TRACE_STATE.with(|state| {
579 let mut state = state.borrow_mut();
580 if state.len() >= max_signatures || state.contains(&signature) {
581 false
582 } else {
583 state.insert(signature.clone());
584 true
585 }
586 });
587 if !should_trace {
588 return;
589 }
590
591 eprintln!("=== native_einsum Path Trace ===");
592 eprintln!(
593 "path={:?} output_ids={:?}",
594 signature.path, signature.output_ids
595 );
596 for operand in &signature.operands {
597 eprintln!(
598 " operand shape={:?} ids={:?} dtype={:?}",
599 operand.shape, operand.ids, operand.dtype
600 );
601 }
602 for line in report.lines {
603 eprintln!("{line}");
604 }
605 if env::var("T4A_TRACE_NATIVE_EINSUM_COMPARE_BALANCED").is_ok() {
606 match native_einsum_balanced_plan_report(&signature) {
607 Ok(balanced) => {
608 for line in balanced.lines {
609 eprintln!("{line}");
610 }
611 }
612 Err(err) => eprintln!("balanced native_einsum path trace failed: {err:#}"),
613 }
614 }
615}
616
617pub fn reset_native_einsum_profile() {
619 NATIVE_EINSUM_PROFILE_STATE.with(|state| state.borrow_mut().clear());
620 NATIVE_EINSUM_TRACE_STATE.with(|state| state.borrow_mut().clear());
621}
622
623pub fn print_and_reset_native_einsum_profile() {
625 if !native_einsum_profile_enabled() {
626 return;
627 }
628 NATIVE_EINSUM_PROFILE_STATE.with(|state| {
629 let mut entries: Vec<_> = state
630 .borrow()
631 .iter()
632 .map(|(k, v)| (k.clone(), v.clone()))
633 .collect();
634 state.borrow_mut().clear();
635 entries.sort_by_key(|(_, entry)| Reverse(entry.total_time));
636
637 eprintln!("=== native_einsum Profile ===");
638 for (idx, (signature, entry)) in entries.into_iter().take(20).enumerate() {
639 eprintln!(
640 "#{idx:02} path={:?} calls={} total={:.3}s per_call={:.3}us output_ids={:?}",
641 signature.path,
642 entry.calls,
643 entry.total_time.as_secs_f64(),
644 entry.total_time.as_secs_f64() * 1e6 / entry.calls as f64,
645 signature.output_ids,
646 );
647 for operand in &signature.operands {
648 eprintln!(
649 " shape={:?} ids={:?} dtype={:?}",
650 operand.shape, operand.ids, operand.dtype
651 );
652 }
653 match native_einsum_time_optimized_plan_report(&signature) {
654 Ok(report) => {
655 for line in report.lines {
656 eprintln!(" {line}");
657 }
658 }
659 Err(err) => eprintln!(" path report failed: {err:#}"),
660 }
661 }
662 });
663}
664
665fn convert_tensor(tensor: &NativeTensor, to: DType) -> Result<NativeTensor> {
666 with_default_session(|session| convert_native_tensor_in(session, tensor, to))
667 .map_err(|e| anyhow::Error::new(e).context(format!("tensor conversion to {to:?} failed")))
668}
669
670fn ids_to_subscript(ids: &[u32]) -> Result<String> {
671 const LETTERS: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
672 let mut out = String::with_capacity(ids.len());
673 for &id in ids {
674 let idx = usize::try_from(id).unwrap_or(usize::MAX);
675 let letter = LETTERS
676 .get(idx)
677 .ok_or_else(|| anyhow!("einsum label {id} exceeds supported label range"))?;
678 out.push(char::from(*letter));
679 }
680 Ok(out)
681}
682
683fn build_einsum_subscripts(operands: &[&[u32]], output_ids: &[u32]) -> Result<String> {
684 let inputs = operands
685 .iter()
686 .map(|ids| ids_to_subscript(ids))
687 .collect::<Result<Vec<_>>>()?;
688 Ok(format!(
689 "{}->{}",
690 inputs.join(","),
691 ids_to_subscript(output_ids)?
692 ))
693}
694
695fn compile_native_einsum_program(
696 compiler: &mut GraphCompiler,
697 input_specs: impl IntoIterator<Item = tenferro::program::ProgramInputSpec>,
698 subscripts: &EinsumSubscripts,
699) -> Result<tenferro::CompiledGraph> {
700 let input_specs = input_specs.into_iter().collect::<Vec<_>>();
701 let target = common_dtype(
702 input_specs
703 .iter()
704 .map(|spec| spec.metadata().dtype())
705 .collect::<Vec<_>>()
706 .as_slice(),
707 );
708 let mut trace = TraceContext::new();
709 let trace_inputs = input_specs
710 .into_iter()
711 .map(|spec| {
712 let dtype = spec.metadata().dtype();
713 let input = trace
714 .input(spec)
715 .map_err(|e| anyhow!("native einsum input tracing failed: {e}"))?;
716 if dtype == target {
717 return Ok(input);
718 }
719 trace
720 .add_op(
721 CoreSemanticOp::Convert {
722 from: dtype,
723 to: target,
724 },
725 &[input],
726 )
727 .map_err(|e| anyhow!("native einsum promotion tracing failed: {e}"))?
728 .first()
729 .copied()
730 .ok_or_else(|| anyhow!("native einsum promotion returned no output"))
731 })
732 .collect::<Result<Vec<_>>>()?;
733 let output = trace
734 .einsum_subscripts(&trace_inputs, subscripts)
735 .map_err(|e| anyhow!("native einsum tracing failed: {e}"))?;
736 let graph = trace
737 .finish(&[output])
738 .map_err(|e| anyhow!("native einsum graph finalization failed: {e}"))?;
739 compiler
740 .compile_traced_graph(&graph)
741 .map_err(|e| anyhow!("native einsum graph compilation failed: {e}"))
742}
743
744fn run_cached_native_einsum(
745 subscripts: &EinsumSubscripts,
746 build: impl FnOnce(&mut GraphCompiler) -> Result<CompiledGraph>,
747 execute: impl FnOnce(&crate::context::CpuExecutionContext, &CompiledGraph) -> Result<NativeTensor>,
748) -> Result<NativeTensor> {
749 let trace_pool = native_einsum_pool_trace_enabled();
750 let pool_before = trace_pool
751 .then(default_engine_buffer_pool_stats)
752 .transpose()?;
753 let program = with_default_graph_compiler(build)?;
757 let result = execute(default_context(), &program)?;
758 if trace_pool {
759 let pool_after = default_engine_buffer_pool_stats()?;
760 let output_bytes = native_tensor_bytes(&result);
761 let retained_threshold = native_einsum_pool_trace_min_retained_bytes();
762 if pool_after != pool_before.unwrap_or_default()
763 && pool_after.capacity_bytes >= retained_threshold
764 || output_bytes >= native_einsum_pool_trace_min_output_bytes()
765 {
766 let before = pool_before.unwrap_or_default();
767 eprintln!(
768 "native_einsum pool subscripts={subscripts:?} before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB output_shape={:?} output_bytes={:.3} MiB",
769 before.buffers,
770 before.capacity_bytes as f64 / (1024.0 * 1024.0),
771 pool_after.buffers,
772 pool_after.capacity_bytes as f64 / (1024.0 * 1024.0),
773 result.shape(),
774 output_bytes as f64 / (1024.0 * 1024.0),
775 );
776 }
777 }
778 if reset_native_einsum_engine_after_call() {
779 let before_reset = trace_pool
780 .then(default_engine_buffer_pool_stats)
781 .transpose()?;
782 reset_default_engine()?;
783 if trace_pool
784 && before_reset.unwrap_or_default().capacity_bytes
785 >= native_einsum_pool_trace_min_retained_bytes()
786 {
787 let before = before_reset.unwrap_or_default();
788 let after = default_engine_buffer_pool_stats()?;
789 eprintln!(
790 "native_einsum engine_reset before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB",
791 before.buffers,
792 before.capacity_bytes as f64 / (1024.0 * 1024.0),
793 after.buffers,
794 after.capacity_bytes as f64 / (1024.0 * 1024.0),
795 );
796 }
797 } else if reset_native_einsum_buffer_pool_after_call() {
798 let before_clear = trace_pool
799 .then(default_engine_buffer_pool_stats)
800 .transpose()?;
801 reset_default_engine_buffer_pool()?;
802 if trace_pool
803 && before_clear.unwrap_or_default().capacity_bytes
804 >= native_einsum_pool_trace_min_retained_bytes()
805 {
806 let before = before_clear.unwrap_or_default();
807 let after = default_engine_buffer_pool_stats()?;
808 eprintln!(
809 "native_einsum pool_reset before_buffers={} before_capacity={:.3} MiB after_buffers={} after_capacity={:.3} MiB",
810 before.buffers,
811 before.capacity_bytes as f64 / (1024.0 * 1024.0),
812 after.buffers,
813 after.capacity_bytes as f64 / (1024.0 * 1024.0),
814 );
815 }
816 }
817 if release_allocator_after_native_einsum_call() {
818 let report = release_process_allocator_cached_memory();
819 if trace_pool && (report.released_bytes.unwrap_or(0) > 0 || report.success == Some(true)) {
820 eprintln!(
821 "native_einsum allocator_pressure_relief supported={} released_bytes={:?} success={:?}",
822 report.supported,
823 report.released_bytes,
824 report.success,
825 );
826 }
827 }
828 Ok(result)
829}
830
831fn cached_einsum_native_tensors(
832 inputs: &[&NativeTensor],
833 subscripts: &EinsumSubscripts,
834) -> Result<NativeTensor> {
835 run_cached_native_einsum(
836 subscripts,
837 |compiler| {
838 compile_native_einsum_program(
839 compiler,
840 inputs.iter().map(|tensor| {
841 tenferro::program::ProgramInputSpec::new(
842 tensor.dtype(),
843 tensor.shape().iter().copied().map(Into::into),
844 )
845 }),
846 subscripts,
847 )
848 },
849 |context, program| single_output(context.run_graph(program, inputs)),
850 )
851}
852
853fn cached_einsum_native_reads(
854 inputs: &[TensorRead<'_>],
855 subscripts: &Subscripts,
856) -> Result<NativeTensor> {
857 let einsum_subscripts = EinsumSubscripts::from(subscripts);
858 execute_native_einsum_reads(inputs, subscripts, &einsum_subscripts)
859}
860
861fn execute_native_einsum_reads(
862 inputs: &[TensorRead<'_>],
863 subscripts: &Subscripts,
864 einsum_subscripts: &EinsumSubscripts,
865) -> Result<NativeTensor> {
866 if inputs.first().is_some_and(|first| {
867 concrete_einsum_dtype_supported(first.dtype())
868 && inputs.iter().all(|input| input.dtype() == first.dtype())
869 }) {
870 let plan = cached_concrete_einsum_plan(inputs, subscripts, einsum_subscripts)?;
871 return default_context()
875 .with_session(|engine| plan.execute_read(inputs, engine))
876 .map_err(|entry| {
877 anyhow::Error::new(entry).context("native einsum session entry failed")
878 })?
879 .map_err(|error| anyhow!("native einsum session execution failed: {error}"));
880 }
881
882 run_cached_native_einsum(
883 einsum_subscripts,
884 |compiler| {
885 compile_native_einsum_program(
886 compiler,
887 inputs.iter().map(|tensor| {
888 tenferro::program::ProgramInputSpec::new(
889 tensor.dtype(),
890 tensor.shape().iter().copied().map(Into::into),
891 )
892 }),
893 einsum_subscripts,
894 )
895 },
896 |context, program| single_output(context.run_graph_reads(program, inputs)),
897 )
898 .map_err(|e| anyhow!("native read einsum failed: {e}"))
899}
900
901fn single_output(
903 outputs: std::result::Result<Vec<NativeTensor>, crate::context::CpuExecutionContextError>,
904) -> Result<NativeTensor> {
905 let mut outputs = outputs.map_err(|e| anyhow!("native einsum failed: {e}"))?;
906 if outputs.len() != 1 {
907 return Err(anyhow!(
908 "native einsum returned {} outputs instead of one",
909 outputs.len()
910 ));
911 }
912 outputs
913 .pop()
914 .ok_or_else(|| anyhow!("native einsum returned no output"))
915}
916
917pub fn dense_native_tensor_from_col_major<T: TensorElement>(
924 data: &[T],
925 logical_dims: &[usize],
926) -> Result<NativeTensor> {
927 T::dense_native_tensor_from_col_major(data, logical_dims)
928}
929
930pub fn dense_native_tensor_from_col_major_owned<T: TensorElement>(
948 data: Vec<T>,
949 logical_dims: &[usize],
950) -> Result<NativeTensor> {
951 T::dense_native_tensor_from_col_major_owned(data, logical_dims)
952}
953
954pub fn dense_native_tensor_read_input_from_col_major<'a, T>(
996 data: &'a [T],
997 logical_dims: &[usize],
998) -> Result<NativeTensorReadInput<'a>>
999where
1000 T: TensorElement,
1001{
1002 let view = tenferro::TypedTensorView::from_col_major(logical_dims, data)
1003 .map_err(|error| anyhow!("native tensor view rejected: {error}"))?;
1004 Ok(NativeTensorReadInput::Borrowed(TensorRead::from_view(
1005 <T as TensorScalar>::tensor_view(view),
1006 )))
1007}
1008
1009pub fn diag_native_tensor_from_col_major<T: TensorElement>(
1016 data: &[T],
1017 logical_rank: usize,
1018) -> Result<NativeTensor> {
1019 T::diag_native_tensor_from_col_major(data, logical_rank)
1020}
1021
1022pub fn storage_to_native_tensor(
1033 storage: &Storage,
1034 logical_dims: &[usize],
1035) -> std::result::Result<NativeTensor, BridgeError> {
1036 storage_dtype_dispatch!(storage.dtype(), Payload => {
1037 let values = <Payload as StorageScalar>::dense_values(storage, logical_dims)
1038 .map_err(|error| anyhow!("dense payload materialization failed: {error}"))?;
1039 <Payload as TensorElement>::dense_native_tensor_from_col_major_owned(
1040 values,
1041 logical_dims,
1042 )
1043 .map_err(BridgeError::from)
1044 },
1045 else Err(anyhow!("unsupported storage scalar type").into()))
1046}
1047
1048pub fn storage_payload_native(storage: &Storage) -> std::result::Result<NativeTensor, BridgeError> {
1057 storage_dtype_dispatch!(storage.dtype(), Payload => {
1058 let values = <Payload as StorageScalar>::payload_vec(storage)
1059 .map_err(|error| anyhow!("compact payload copy failed: {error}"))?;
1060 native_tensor_from_vec(storage.payload_dims().to_vec(), values)
1061 },
1062 else Err(anyhow!("unsupported storage scalar type").into()))
1063}
1064
1065pub fn storage_from_payload_native(
1076 payload: NativeTensor,
1077 output_payload_dims: &[usize],
1078 output_axis_classes: Vec<usize>,
1079) -> std::result::Result<Storage, BridgeError> {
1080 if payload.shape() != output_payload_dims {
1081 return Err(anyhow!(
1082 "payload shape {:?} does not match planned payload dims {:?}",
1083 payload.shape(),
1084 output_payload_dims
1085 )
1086 .into());
1087 }
1088 let strides = crate::storage::col_major_strides(output_payload_dims)?;
1089 storage_dtype_dispatch!(payload.dtype(), Payload => {
1090 let values = native_slice::<Payload>(&payload, "failed to read structured payload")?.to_vec();
1091 Storage::new_structured(
1092 values,
1093 output_payload_dims.to_vec(),
1094 strides,
1095 output_axis_classes,
1096 )
1097 .map_err(|error| anyhow!("structured payload storage construction failed: {error}").into())
1098 },
1099 else Err(anyhow!(
1100 "unsupported structured payload dtype {:?}; expected a floating or complex floating dtype",
1101 payload.dtype()
1102 )
1103 .into()))
1104}
1105
1106pub fn storage_payload_native_read_input(
1117 storage: &Storage,
1118) -> std::result::Result<NativeTensorReadInput<'_>, BridgeError> {
1119 storage_dtype_dispatch!(storage.dtype(), Payload => {
1120 if let Some(view) = <Payload as StorageScalar>::payload_view(storage)
1121 .map_err(anyhow::Error::msg)?
1122 {
1123 let typed = tenferro::TypedTensorView::from_col_major(storage.payload_dims(), view)
1124 .map_err(|error| BridgeError::from(anyhow::Error::new(error)))?;
1125 return Ok(NativeTensorReadInput::Borrowed(TensorRead::from_view(
1126 <Payload as TensorScalar>::tensor_view(typed),
1127 )));
1128 }
1129 let values = <Payload as StorageScalar>::payload_vec(storage)
1130 .map_err(|error| anyhow!("compact payload copy failed: {error}"))?;
1131 native_tensor_from_vec(storage.payload_dims().to_vec(), values)
1132 .map(NativeTensorReadInput::Owned)
1133 },
1134 else Err(anyhow!("unsupported storage scalar type").into()))
1135}
1136
1137pub fn native_tensor_primal_to_storage(
1144 tensor: &NativeTensor,
1145) -> std::result::Result<Storage, BridgeError> {
1146 match tensor.dtype() {
1147 DType::F32 => Storage::from_dense_col_major(
1148 native_slice::<f32>(tensor, "failed to read f32 native tensor")?.to_vec(),
1149 tensor.shape(),
1150 )
1151 .map_err(|e| {
1152 BridgeError::from(anyhow!(
1153 "native tensor snapshot materialization failed: {e}"
1154 ))
1155 }),
1156 DType::F64 => Storage::from_dense_col_major(
1157 native_slice::<f64>(tensor, "failed to read f64 native tensor")?.to_vec(),
1158 tensor.shape(),
1159 )
1160 .map_err(|e| {
1161 BridgeError::from(anyhow!(
1162 "native tensor snapshot materialization failed: {e}"
1163 ))
1164 }),
1165 DType::I32 => Storage::from_dense_col_major(
1166 native_slice::<i32>(tensor, "failed to read i32 native tensor")?
1167 .iter()
1168 .map(|&value| value as f64)
1169 .collect::<Vec<_>>(),
1170 tensor.shape(),
1171 )
1172 .map_err(|e| {
1173 BridgeError::from(anyhow!(
1174 "native tensor snapshot materialization failed: {e}"
1175 ))
1176 }),
1177 DType::I64 => Storage::from_dense_col_major(
1178 native_slice::<i64>(tensor, "failed to read i64 native tensor")?
1179 .iter()
1180 .map(|&value| value as f64)
1181 .collect::<Vec<_>>(),
1182 tensor.shape(),
1183 )
1184 .map_err(|e| {
1185 BridgeError::from(anyhow!(
1186 "native tensor snapshot materialization failed: {e}"
1187 ))
1188 }),
1189 DType::Bool => Storage::from_dense_col_major(
1190 native_slice::<bool>(tensor, "failed to read bool native tensor")?
1191 .iter()
1192 .map(|&value| if value { 1.0 } else { 0.0 })
1193 .collect::<Vec<_>>(),
1194 tensor.shape(),
1195 )
1196 .map_err(|e| {
1197 BridgeError::from(anyhow!(
1198 "native tensor snapshot materialization failed: {e}"
1199 ))
1200 }),
1201 DType::C32 => Storage::from_dense_col_major(
1202 native_slice::<Complex32>(tensor, "failed to read c32 native tensor")?.to_vec(),
1203 tensor.shape(),
1204 )
1205 .map_err(|e| {
1206 BridgeError::from(anyhow!(
1207 "native tensor snapshot materialization failed: {e}"
1208 ))
1209 }),
1210 DType::C64 => Storage::from_dense_col_major(
1211 native_slice::<Complex64>(tensor, "failed to read c64 native tensor")?.to_vec(),
1212 tensor.shape(),
1213 )
1214 .map_err(|e| {
1215 BridgeError::from(anyhow!(
1216 "native tensor snapshot materialization failed: {e}"
1217 ))
1218 }),
1219 DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
1220 tenferro_tensor::Error::unsupported(
1221 "dense storage",
1222 "external scalar kinds cannot be materialized into dense storage",
1223 ),
1224 ))),
1225 }
1226}
1227
1228pub fn native_tensor_primal_to_dense_col_major<T: TensorElement>(
1234 tensor: &NativeTensor,
1235) -> std::result::Result<Vec<T>, BridgeError> {
1236 let target = <T as TensorScalar>::dtype();
1237 let tensor_is_real = matches!(
1238 tensor.dtype(),
1239 DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
1240 );
1241 let target_is_real = matches!(
1242 target,
1243 DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
1244 );
1245 if tensor_is_real != target_is_real {
1246 return Err(anyhow!(
1247 "expected {} native tensor, got dtype {:?}",
1248 if target_is_real { "real" } else { "complex" },
1249 tensor.dtype()
1250 )
1251 .into());
1252 }
1253 <T as TensorElement>::dense_values_from_native_col_major(tensor).map_err(BridgeError::from)
1254}
1255
1256pub fn native_tensor_primal_to_diag<T: TensorElement>(
1280 tensor: &NativeTensor,
1281) -> std::result::Result<Vec<T>, BridgeError> {
1282 let promote_to = <T as TensorScalar>::dtype();
1283 let tensor_is_real = matches!(
1284 tensor.dtype(),
1285 DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
1286 );
1287 let target_is_real = matches!(
1288 promote_to,
1289 DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool
1290 );
1291 if tensor_is_real != target_is_real {
1292 return Err(anyhow!(
1293 "expected {} native tensor, got dtype {:?}",
1294 if target_is_real { "real" } else { "complex" },
1295 tensor.dtype()
1296 )
1297 .into());
1298 }
1299 let promoted = convert_tensor(tensor, promote_to)?;
1300 <T as TensorElement>::diag_values_from_native_temp(&promoted).map_err(BridgeError::from)
1301}
1302
1303pub fn reshape_col_major_native_tensor(
1310 tensor: &NativeTensor,
1311 logical_dims: &[usize],
1312) -> Result<NativeTensor> {
1313 with_default_session(|session| tensor.reshape(logical_dims, session))
1314 .map_err(|e| anyhow!("native reshape failed: {e}"))
1315}
1316
1317pub fn qr_native_tensor(
1323 tensor: &NativeTensor,
1324) -> std::result::Result<(NativeTensor, NativeTensor), BridgeError> {
1325 let (q, r) = with_default_session(|session| tensor.qr(session))
1326 .map_err(|e| anyhow!("native QR failed: {e}"))?;
1327 Ok((q, r))
1328}
1329
1330pub fn svd_native_tensor(
1337 tensor: &NativeTensor,
1338) -> Result<(NativeTensor, NativeTensor, NativeTensor)> {
1339 let (u, s, vt) = with_default_session(|session| tensor.svd(session))
1340 .map_err(|e| anyhow!("native SVD failed: {e}"))?;
1341 Ok((u, s, vt))
1342}
1343
1344pub fn sum_native_tensor(tensor: &NativeTensor) -> std::result::Result<BackendScalar, BridgeError> {
1351 let reduced = if tensor.shape().is_empty() {
1352 tensor
1353 .duplicate()
1354 .map_err(|e| anyhow!("native scalar duplication failed: {e}"))?
1355 } else {
1356 let axes: Vec<usize> = (0..tensor.shape().len()).collect();
1357 with_default_session(|session| tensor.reduce_sum(Some(&axes), session))
1358 .map_err(|e| anyhow!("native sum failed: {e}"))?
1359 };
1360 Ok(BackendScalar::from_native(reduced)?)
1361}
1362
1363pub fn tangent_native_tensor(_tensor: &NativeTensor) -> Option<NativeTensor> {
1368 None
1369}
1370
1371pub fn scale_native_tensor(
1378 tensor: &NativeTensor,
1379 scalar: &BackendScalar,
1380) -> std::result::Result<NativeTensor, BridgeError> {
1381 let target = common_dtype(&[tensor.dtype(), scalar.as_native().dtype()]);
1382 let tensor = convert_tensor(tensor, target)?;
1383 let scalar = promote_scalar_native(scalar.as_native(), target)?;
1384
1385 match target {
1386 DType::F32 => {
1387 let factor = native_slice::<f32>(&scalar, "failed to read promoted f32 scalar")?
1388 .first()
1389 .copied()
1390 .ok_or_else(|| anyhow!("failed to read promoted f32 scalar"))?;
1391 let values = native_slice::<f32>(&tensor, "failed to read promoted f32 tensor")?
1392 .iter()
1393 .map(|&value| value * factor)
1394 .collect::<Vec<_>>();
1395 native_tensor_from_vec(tensor.shape().to_vec(), values)
1396 }
1397 DType::F64 => {
1398 let factor = native_slice::<f64>(&scalar, "failed to read promoted f64 scalar")?
1399 .first()
1400 .copied()
1401 .ok_or_else(|| anyhow!("failed to read promoted f64 scalar"))?;
1402 let values = native_slice::<f64>(&tensor, "failed to read promoted f64 tensor")?
1403 .iter()
1404 .map(|&value| value * factor)
1405 .collect::<Vec<_>>();
1406 native_tensor_from_vec(tensor.shape().to_vec(), values)
1407 }
1408 DType::C32 => {
1409 let factor = native_slice::<Complex32>(&scalar, "failed to read promoted c32 scalar")?
1410 .first()
1411 .copied()
1412 .ok_or_else(|| anyhow!("failed to read promoted c32 scalar"))?;
1413 let values = native_slice::<Complex32>(&tensor, "failed to read promoted c32 tensor")?
1414 .iter()
1415 .map(|&value| value * factor)
1416 .collect::<Vec<_>>();
1417 native_tensor_from_vec(tensor.shape().to_vec(), values)
1418 }
1419 DType::C64 => {
1420 let factor = native_slice::<Complex64>(&scalar, "failed to read promoted c64 scalar")?
1421 .first()
1422 .copied()
1423 .ok_or_else(|| anyhow!("failed to read promoted c64 scalar"))?;
1424 let values = native_slice::<Complex64>(&tensor, "failed to read promoted c64 tensor")?
1425 .iter()
1426 .map(|&value| value * factor)
1427 .collect::<Vec<_>>();
1428 native_tensor_from_vec(tensor.shape().to_vec(), values)
1429 }
1430 DType::I32 | DType::I64 | DType::Bool => {
1431 Err(anyhow!("scale_native_tensor does not support integer/bool tensors").into())
1432 }
1433 DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
1434 tenferro_tensor::Error::unsupported(
1435 "scaling",
1436 "external scalar kinds are not scalable dtypes",
1437 ),
1438 ))),
1439 }
1440}
1441
1442pub fn axpby_native_tensor(
1449 lhs: &NativeTensor,
1450 a: &BackendScalar,
1451 rhs: &NativeTensor,
1452 b: &BackendScalar,
1453) -> std::result::Result<NativeTensor, BridgeError> {
1454 if lhs.shape() != rhs.shape() {
1455 return Err(BridgeError::from(anyhow!(
1456 "axpby requires matching tensor shapes, got lhs {:?} and rhs {:?}",
1457 lhs.shape(),
1458 rhs.shape()
1459 )));
1460 }
1461
1462 let target = common_dtype(&[
1463 lhs.dtype(),
1464 rhs.dtype(),
1465 a.as_native().dtype(),
1466 b.as_native().dtype(),
1467 ]);
1468 let lhs = convert_tensor(lhs, target)?;
1469 let rhs = convert_tensor(rhs, target)?;
1470 let a = promote_scalar_native(a.as_native(), target)?;
1471 let b = promote_scalar_native(b.as_native(), target)?;
1472
1473 match target {
1474 DType::F32 => {
1475 let a = native_slice::<f32>(&a, "failed to read promoted f32 scalar a")?
1476 .first()
1477 .copied()
1478 .ok_or_else(|| anyhow!("failed to read promoted f32 scalar a"))?;
1479 let b = native_slice::<f32>(&b, "failed to read promoted f32 scalar b")?
1480 .first()
1481 .copied()
1482 .ok_or_else(|| anyhow!("failed to read promoted f32 scalar b"))?;
1483 let lhs_values = native_slice::<f32>(&lhs, "failed to read promoted f32 lhs")?;
1484 let rhs_values = native_slice::<f32>(&rhs, "failed to read promoted f32 rhs")?;
1485 let values = lhs_values
1486 .iter()
1487 .zip(rhs_values.iter())
1488 .map(|(&x, &y)| a * x + b * y)
1489 .collect::<Vec<_>>();
1490 native_tensor_from_vec(lhs.shape().to_vec(), values)
1491 }
1492 DType::F64 => {
1493 let a = native_slice::<f64>(&a, "failed to read promoted f64 scalar a")?
1494 .first()
1495 .copied()
1496 .ok_or_else(|| anyhow!("failed to read promoted f64 scalar a"))?;
1497 let b = native_slice::<f64>(&b, "failed to read promoted f64 scalar b")?
1498 .first()
1499 .copied()
1500 .ok_or_else(|| anyhow!("failed to read promoted f64 scalar b"))?;
1501 let lhs_values = native_slice::<f64>(&lhs, "failed to read promoted f64 lhs")?;
1502 let rhs_values = native_slice::<f64>(&rhs, "failed to read promoted f64 rhs")?;
1503 let values = lhs_values
1504 .iter()
1505 .zip(rhs_values.iter())
1506 .map(|(&x, &y)| a * x + b * y)
1507 .collect::<Vec<_>>();
1508 native_tensor_from_vec(lhs.shape().to_vec(), values)
1509 }
1510 DType::C32 => {
1511 let a = native_slice::<Complex32>(&a, "failed to read promoted c32 scalar a")?
1512 .first()
1513 .copied()
1514 .ok_or_else(|| anyhow!("failed to read promoted c32 scalar a"))?;
1515 let b = native_slice::<Complex32>(&b, "failed to read promoted c32 scalar b")?
1516 .first()
1517 .copied()
1518 .ok_or_else(|| anyhow!("failed to read promoted c32 scalar b"))?;
1519 let lhs_values = native_slice::<Complex32>(&lhs, "failed to read promoted c32 lhs")?;
1520 let rhs_values = native_slice::<Complex32>(&rhs, "failed to read promoted c32 rhs")?;
1521 let values = lhs_values
1522 .iter()
1523 .zip(rhs_values.iter())
1524 .map(|(&x, &y)| a * x + b * y)
1525 .collect::<Vec<_>>();
1526 native_tensor_from_vec(lhs.shape().to_vec(), values)
1527 }
1528 DType::C64 => {
1529 let a = native_slice::<Complex64>(&a, "failed to read promoted c64 scalar a")?
1530 .first()
1531 .copied()
1532 .ok_or_else(|| anyhow!("failed to read promoted c64 scalar a"))?;
1533 let b = native_slice::<Complex64>(&b, "failed to read promoted c64 scalar b")?
1534 .first()
1535 .copied()
1536 .ok_or_else(|| anyhow!("failed to read promoted c64 scalar b"))?;
1537 let lhs_values = native_slice::<Complex64>(&lhs, "failed to read promoted c64 lhs")?;
1538 let rhs_values = native_slice::<Complex64>(&rhs, "failed to read promoted c64 rhs")?;
1539 let values = lhs_values
1540 .iter()
1541 .zip(rhs_values.iter())
1542 .map(|(&x, &y)| a * x + b * y)
1543 .collect::<Vec<_>>();
1544 native_tensor_from_vec(lhs.shape().to_vec(), values)
1545 }
1546 DType::I32 | DType::I64 | DType::Bool => {
1547 Err(anyhow!("axpby_native_tensor does not support integer/bool tensors").into())
1548 }
1549 DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
1550 tenferro_tensor::Error::unsupported(
1551 "axpby",
1552 "external scalar kinds are not axpby dtypes",
1553 ),
1554 ))),
1555 }
1556}
1557
1558pub fn einsum_native_tensors_owned(
1593 operands: Vec<(NativeTensor, Vec<usize>)>,
1594 output_ids: &[usize],
1595) -> Result<NativeTensor> {
1596 ensure!(
1597 !operands.is_empty(),
1598 "native einsum requires at least one operand"
1599 );
1600
1601 let target = common_dtype(
1602 &operands
1603 .iter()
1604 .map(|(tensor, _)| tensor.dtype())
1605 .collect::<Vec<_>>(),
1606 );
1607
1608 let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
1609 let mut converted = Vec::with_capacity(operands.len());
1610 let mut input_ids = Vec::with_capacity(operands.len());
1611 for (tensor, ids) in operands {
1612 ensure!(
1613 tensor.shape().len() == ids.len(),
1614 "einsum id list {:?} does not match tensor shape {:?}",
1615 ids,
1616 tensor.shape()
1617 );
1618 let checked_ids = checked_native_einsum_labels(&ids)?;
1619 let tensor = if tensor.dtype() == target {
1620 tensor
1621 } else {
1622 convert_tensor(&tensor, target)?
1623 };
1624 input_ids.push(checked_ids);
1625 converted.push(tensor);
1626 }
1627
1628 let input_slices = input_ids.iter().map(Vec::as_slice).collect::<Vec<_>>();
1629 let subscripts = EinsumSubscripts::new(&input_slices, &output_ids_u32);
1630
1631 let input_refs = converted.iter().collect::<Vec<_>>();
1632 let trace_operands = input_refs
1633 .iter()
1634 .zip(input_ids.iter())
1635 .map(|(tensor, ids)| (*tensor, ids.as_slice()))
1636 .collect::<Vec<_>>();
1637 maybe_trace_native_einsum_path(NativeEinsumPath::Owned, &trace_operands, &output_ids_u32);
1638 let started = Instant::now();
1639 let result = cached_einsum_native_tensors(&input_refs, &subscripts)?;
1640 record_native_einsum_profile(
1641 NativeEinsumPath::Owned,
1642 &trace_operands,
1643 &output_ids_u32,
1644 started.elapsed(),
1645 );
1646 Ok(result)
1647}
1648
1649pub fn einsum_native_tensors(
1687 operands: &[(&NativeTensor, &[usize])],
1688 output_ids: &[usize],
1689) -> Result<NativeTensor> {
1690 ensure!(
1691 !operands.is_empty(),
1692 "native einsum requires at least one operand"
1693 );
1694
1695 let target = common_dtype(
1696 &operands
1697 .iter()
1698 .map(|(tensor, _)| tensor.dtype())
1699 .collect::<Vec<_>>(),
1700 );
1701 let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
1702 let mut converted = Vec::with_capacity(operands.len());
1703 let mut input_ids = Vec::with_capacity(operands.len());
1704 let mut has_conversions = false;
1705 let started = Instant::now();
1706
1707 for (tensor, ids) in operands {
1708 ensure!(
1709 tensor.shape().len() == ids.len(),
1710 "einsum id list {:?} does not match tensor shape {:?}",
1711 ids,
1712 tensor.shape()
1713 );
1714 input_ids.push(checked_native_einsum_labels(ids)?);
1715 if tensor.dtype() == target {
1716 converted.push(None);
1717 } else {
1718 converted.push(Some(convert_tensor(tensor, target)?));
1719 has_conversions = true;
1720 }
1721 }
1722
1723 let input_slices = input_ids.iter().map(Vec::as_slice).collect::<Vec<_>>();
1724 let subscripts = EinsumSubscripts::new(&input_slices, &output_ids_u32);
1725 let input_refs = operands
1726 .iter()
1727 .zip(converted.iter())
1728 .map(|((tensor, _), converted)| converted.as_ref().unwrap_or(*tensor))
1729 .collect::<Vec<_>>();
1730 let trace_path = if has_conversions {
1731 NativeEinsumPath::BorrowedWithConversions
1732 } else {
1733 NativeEinsumPath::Borrowed
1734 };
1735 let trace_operands = input_refs
1736 .iter()
1737 .zip(input_ids.iter())
1738 .map(|(tensor, ids)| (*tensor, ids.as_slice()))
1739 .collect::<Vec<_>>();
1740 maybe_trace_native_einsum_path(trace_path, &trace_operands, &output_ids_u32);
1741 let result = cached_einsum_native_tensors(&input_refs, &subscripts)?;
1742 record_native_einsum_profile(
1743 trace_path,
1744 &trace_operands,
1745 &output_ids_u32,
1746 started.elapsed(),
1747 );
1748 Ok(result)
1749}
1750
1751pub fn einsum_native_subscripts(
1810 subscripts: &str,
1811 operands: &[&NativeTensor],
1812) -> Result<NativeTensor> {
1813 let plan = NativeEinsumPlan::new(subscripts)?;
1814 let reads = operands
1815 .iter()
1816 .map(|tensor| TensorRead::from_tensor(tensor))
1817 .collect::<Vec<_>>();
1818 plan.execute_reads(&reads)
1819}
1820
1821#[derive(Debug)]
1844pub struct NativeEinsumPlan {
1845 spec: Arc<NativeEinsumSpec>,
1846}
1847
1848impl NativeEinsumPlan {
1849 pub fn new(subscripts: &str) -> Result<Self> {
1867 Ok(Self {
1868 spec: cached_native_einsum_spec(subscripts)?,
1869 })
1870 }
1871
1872 pub fn execute_reads(&self, inputs: &[TensorRead<'_>]) -> Result<NativeTensor> {
1895 ensure!(
1896 self.spec.subscripts.inputs.len() == inputs.len(),
1897 "einsum subscripts describe {} input term(s) but {} operand(s) were given",
1898 self.spec.subscripts.inputs.len(),
1899 inputs.len()
1900 );
1901 ensure!(
1902 !inputs.is_empty(),
1903 "native einsum requires at least one operand"
1904 );
1905 for (tensor, ids) in inputs.iter().zip(&self.spec.subscripts.inputs) {
1906 ensure!(
1907 tensor.shape().len() == ids.len(),
1908 "einsum id list {:?} does not match tensor shape {:?}",
1909 ids,
1910 tensor.shape()
1911 );
1912 }
1913 execute_native_einsum_reads(inputs, &self.spec.subscripts, &self.spec.einsum_subscripts)
1914 }
1915}
1916
1917pub fn einsum_native_tensor_reads(
1928 operands: &[(&NativeTensorReadInput<'_>, &[usize])],
1929 output_ids: &[usize],
1930) -> Result<NativeTensor> {
1931 ensure!(
1932 !operands.is_empty(),
1933 "native einsum requires at least one operand"
1934 );
1935
1936 let output_ids_u32 = checked_native_einsum_labels(output_ids)?;
1937 let mut input_ids = Vec::with_capacity(operands.len());
1938 let mut read_inputs = Vec::with_capacity(operands.len());
1939
1940 for (tensor, ids) in operands {
1941 ensure!(
1942 tensor.shape().len() == ids.len(),
1943 "einsum id list {:?} does not match tensor shape {:?}",
1944 ids,
1945 tensor.shape()
1946 );
1947 input_ids.push(checked_native_einsum_labels(ids)?);
1948 read_inputs.push(tensor.as_read());
1949 }
1950
1951 let subscripts = Subscripts {
1952 inputs: input_ids,
1953 output: output_ids_u32,
1954 };
1955 cached_einsum_native_reads(&read_inputs, &subscripts)
1956}
1957
1958pub fn permute_native_tensor(
1964 tensor: &NativeTensor,
1965 perm: &[usize],
1966) -> std::result::Result<NativeTensor, BridgeError> {
1967 with_default_session(|session| tensor.transpose(perm, session))
1968 .map_err(|e| anyhow!("native permute failed: {e}"))
1969 .map_err(BridgeError::from)
1970}
1971
1972pub fn contract_native_tensor(
1979 lhs: &NativeTensor,
1980 axes_a: &[usize],
1981 rhs: &NativeTensor,
1982 axes_b: &[usize],
1983) -> Result<NativeTensor> {
1984 let (lhs_ids, rhs_ids, output_ids) =
1985 build_binary_einsum_ids(lhs.shape().len(), axes_a, rhs.shape().len(), axes_b)?;
1986 let lhs_ids_usize = lhs_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
1987 let rhs_ids_usize = rhs_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
1988 let output_ids_usize = output_ids.iter().map(|&id| id as usize).collect::<Vec<_>>();
1989 let operands = [
1990 (lhs, lhs_ids_usize.as_slice()),
1991 (rhs, rhs_ids_usize.as_slice()),
1992 ];
1993 einsum_native_tensors(&operands, &output_ids_usize)
1994}
1995
1996pub fn outer_product_native_tensor(
2001 lhs: &NativeTensor,
2002 rhs: &NativeTensor,
2003) -> std::result::Result<NativeTensor, BridgeError> {
2004 contract_native_tensor(lhs, &[], rhs, &[]).map_err(BridgeError::from)
2005}
2006
2007pub fn conj_native_tensor(tensor: &NativeTensor) -> std::result::Result<NativeTensor, BridgeError> {
2014 match tensor.dtype() {
2015 DType::F32 | DType::F64 | DType::I32 | DType::I64 | DType::Bool => tensor
2016 .duplicate()
2017 .map_err(|e| anyhow!("native tensor duplication failed: {e}"))
2018 .map_err(BridgeError::from),
2019 DType::External(_) => Err(BridgeError::from(anyhow::Error::new(
2020 tenferro_tensor::Error::unsupported(
2021 "conjugation",
2022 "external scalar kinds have no conjugation",
2023 ),
2024 ))),
2025 DType::C32 => native_tensor_from_vec(
2026 tensor.shape().to_vec(),
2027 native_slice::<Complex32>(tensor, "failed to read c32 native tensor")?
2028 .iter()
2029 .map(|&value| value.conj())
2030 .collect::<Vec<_>>(),
2031 ),
2032 DType::C64 => native_tensor_from_vec(
2033 tensor.shape().to_vec(),
2034 native_slice::<Complex64>(tensor, "failed to read c64 native tensor")?
2035 .iter()
2036 .map(|&value| value.conj())
2037 .collect::<Vec<_>>(),
2038 ),
2039 }
2040}
2041
2042pub fn permute_storage_native(
2048 storage: &Storage,
2049 logical_dims: &[usize],
2050 perm: &[usize],
2051) -> std::result::Result<Storage, BridgeError> {
2052 let native = storage_to_native_tensor(storage, logical_dims)?;
2053 let permuted = permute_native_tensor(&native, perm)?;
2054 native_tensor_primal_to_storage(&permuted)
2055}
2056
2057pub fn contract_storage_native(
2063 storage_a: &Storage,
2064 dims_a: &[usize],
2065 axes_a: &[usize],
2066 storage_b: &Storage,
2067 dims_b: &[usize],
2068 axes_b: &[usize],
2069 _result_dims: &[usize],
2070) -> std::result::Result<Storage, BridgeError> {
2071 let lhs = storage_to_native_tensor(storage_a, dims_a)?;
2072 let rhs = storage_to_native_tensor(storage_b, dims_b)?;
2073 let result = contract_native_tensor(&lhs, axes_a, &rhs, axes_b)?;
2074 native_tensor_primal_to_storage(&result)
2075}
2076
2077pub fn outer_product_storage_native(
2083 lhs: &Storage,
2084 lhs_dims: &[usize],
2085 rhs: &Storage,
2086 rhs_dims: &[usize],
2087 _result_dims: &[usize],
2088) -> std::result::Result<Storage, BridgeError> {
2089 let lhs = storage_to_native_tensor(lhs, lhs_dims)?;
2090 let rhs = storage_to_native_tensor(rhs, rhs_dims)?;
2091 let result = outer_product_native_tensor(&lhs, &rhs)?;
2092 native_tensor_primal_to_storage(&result)
2093}
2094
2095pub fn scale_storage_native(
2101 storage: &Storage,
2102 logical_dims: &[usize],
2103 scalar: &BackendScalar,
2104) -> std::result::Result<Storage, BridgeError> {
2105 let native = storage_to_native_tensor(storage, logical_dims)?;
2106 let scaled = scale_native_tensor(&native, scalar)?;
2107 native_tensor_primal_to_storage(&scaled)
2108}
2109
2110pub fn axpby_storage_native(
2116 lhs: &Storage,
2117 lhs_dims: &[usize],
2118 a: &BackendScalar,
2119 rhs: &Storage,
2120 rhs_dims: &[usize],
2121 b: &BackendScalar,
2122) -> std::result::Result<Storage, BridgeError> {
2123 let lhs = storage_to_native_tensor(lhs, lhs_dims)?;
2124 let rhs = storage_to_native_tensor(rhs, rhs_dims)?;
2125 let combined = axpby_native_tensor(&lhs, a, &rhs, b)?;
2126 native_tensor_primal_to_storage(&combined)
2127}
2128
2129#[cfg(test)]
2130mod tests;