1use crate::defaults::DynIndex;
2use crate::index_like::IndexLike;
3use crate::index_ops::{common_ind_positions, prepare_contraction, prepare_contraction_pairs};
4use crate::tensor_like::LinearizationOrder;
5use crate::AnyScalar;
6use anyhow::{Context, Result};
7use num_complex::{Complex32, Complex64};
8use num_traits::Zero;
9use rand::Rng;
10use rand_distr::{Distribution, StandardNormal};
11use std::cell::RefCell;
12use std::cmp::Reverse;
13use std::collections::{HashMap, HashSet};
14use std::env;
15use std::sync::{Arc, OnceLock};
16use std::time::{Duration, Instant};
17use tenferro::{
18 DType, DotGeneralConfig, Tensor as NativeTensor, TensorRead, TensorValue, TensorView,
19};
20use tenferro_ad::{extension::adopt_untracked_eager_value, EagerRuntime, EagerTensor};
21use tenferro_einsum::{EagerSessionEinsumExt, EinsumSubscripts};
22use tenferro_linalg::{EagerSessionLinalgExt, RankRevealingQrOptions};
23use tensor4all_tensorbackend::{
24 default_eager_ctx, dense_native_tensor_from_col_major,
25 dense_native_tensor_from_col_major_owned, diag_native_tensor_from_col_major,
26 native_tensor_primal_to_diag, storage_payload_native_read_input, storage_to_native_tensor,
27 ExecutionContext, NativeTensorReadInput, TensorElement,
28};
29use tensor4all_tensorbackend::{
30 src_error_estimate as backend_src_error_estimate,
31 src_error_estimate_general as backend_src_error_estimate_general, Matrix,
32};
33use tensor4all_tensorbackend::{IncrementalQr, IncrementalQrScalar};
34use tensor4all_tensorbackend::{NonfiniteFlags, Storage, StorageKind, StorageScalar};
35
36use super::contract::{ContractionOptions, PairwiseContractionOptions};
37use super::eager_session::{with_eager_session, with_engine, with_execution_engine};
38use super::structured_contraction::{
39 normalize_payload_read_for_roots, storage_from_payload_native, storage_payload_native,
40 OperandLayout, StructuredContractionPlan, StructuredContractionSpec,
41};
42
43fn conjugate_eager(
44 inner: &EagerTensor,
45) -> std::result::Result<EagerTensor, Arc<dyn std::error::Error + Send + Sync + 'static>> {
46 with_eager_session(inner.runtime(), None, |session| session.conj(inner))
47 .map_err(|source| Arc::new(source) as _)
48}
49
50#[derive(Debug, Default, Clone)]
51struct PairwiseContractProfileEntry {
52 calls: usize,
53 total_time: Duration,
54 total_bytes: usize,
55}
56
57#[derive(Debug, Clone)]
79pub struct TensorHermitianEigendecomposition {
80 pub eigenvalues: Vec<f64>,
82 pub eigenvectors: IdxTensor,
84 pub eigenvector_index: DynIndex,
86}
87
88#[cfg(test)]
89thread_local! {
90 static EAGER_PAYLOAD_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
93}
94
95#[cfg(test)]
96fn note_eager_payload_read() {
97 EAGER_PAYLOAD_READS.with(|reads| reads.set(reads.get() + 1));
98}
99
100thread_local! {
101 static PAIRWISE_CONTRACT_PROFILE_STATE: RefCell<HashMap<&'static str, PairwiseContractProfileEntry>> =
102 RefCell::new(HashMap::new());
103}
104
105fn pairwise_contract_profile_enabled() -> bool {
106 static ENABLED: OnceLock<bool> = OnceLock::new();
107 *ENABLED.get_or_init(|| env::var("T4A_PROFILE_PAIRWISE_CONTRACT").is_ok())
108}
109
110fn record_pairwise_contract_profile(section: &'static str, elapsed: Duration) {
111 if !pairwise_contract_profile_enabled() {
112 return;
113 }
114 PAIRWISE_CONTRACT_PROFILE_STATE.with(|state| {
115 let mut state = state.borrow_mut();
116 let entry = state.entry(section).or_default();
117 entry.calls += 1;
118 entry.total_time += elapsed;
119 });
120}
121
122fn record_pairwise_contract_profile_bytes(section: &'static str, bytes: usize) {
123 if !pairwise_contract_profile_enabled() {
124 return;
125 }
126 PAIRWISE_CONTRACT_PROFILE_STATE.with(|state| {
127 let mut state = state.borrow_mut();
128 let entry = state.entry(section).or_default();
129 entry.total_bytes += bytes;
130 });
131}
132
133fn profile_pairwise_contract_section<T>(section: &'static str, f: impl FnOnce() -> T) -> T {
134 if !pairwise_contract_profile_enabled() {
135 return f();
136 }
137 let started = Instant::now();
138 let result = f();
139 record_pairwise_contract_profile(section, started.elapsed());
140 result
141}
142
143pub fn reset_pairwise_contract_profile() {
145 PAIRWISE_CONTRACT_PROFILE_STATE.with(|state| state.borrow_mut().clear());
146}
147
148pub fn print_and_reset_pairwise_contract_profile() {
150 if !pairwise_contract_profile_enabled() {
151 return;
152 }
153 PAIRWISE_CONTRACT_PROFILE_STATE.with(|state| {
154 let mut entries: Vec<_> = state
155 .borrow()
156 .iter()
157 .map(|(section, entry)| (*section, entry.clone()))
158 .collect();
159 state.borrow_mut().clear();
160 entries.sort_by_key(|(_, entry)| Reverse(entry.total_time));
161
162 eprintln!("=== IdxTensor pairwise contract profile ===");
163 for (section, entry) in entries {
164 let per_call_us = if entry.calls == 0 {
165 0.0
166 } else {
167 entry.total_time.as_secs_f64() * 1.0e6 / entry.calls as f64
168 };
169 eprintln!(
170 "{section}: calls={} total={:.6}ms per_call={:.3}us bytes={}",
171 entry.calls,
172 entry.total_time.as_secs_f64() * 1.0e3,
173 per_call_us,
174 entry.total_bytes,
175 );
176 }
177 });
178}
179
180fn tensor_profile_bytes(dtype: DType, shape: &[usize]) -> Result<usize> {
181 let element_size = match dtype {
182 DType::F32 => 4,
183 DType::F64 => 8,
184 DType::C32 => 8,
185 DType::C64 => 16,
186 DType::I32 => 4,
187 DType::I64 => 8,
188 DType::Bool => 1,
189 DType::External(_) => {
192 return Err(anyhow::anyhow!(
193 "external scalar kind {dtype:?} has no host payload size"
194 ));
195 }
196 };
197 Ok(shape
198 .iter()
199 .try_fold(1usize, |bytes, &dim| bytes.checked_mul(dim))
200 .and_then(|elements| elements.checked_mul(element_size))
201 .unwrap_or(usize::MAX))
202}
203
204pub trait RandomScalar: TensorElement {
223 fn random_value<R: Rng + ?Sized>(rng: &mut R) -> Self;
244}
245
246impl RandomScalar for f64 {
247 fn random_value<R: Rng + ?Sized>(rng: &mut R) -> Self {
248 StandardNormal.sample(rng)
249 }
250}
251
252impl RandomScalar for Complex64 {
253 fn random_value<R: Rng + ?Sized>(rng: &mut R) -> Self {
254 Complex64::new(StandardNormal.sample(rng), StandardNormal.sample(rng))
255 }
256}
257
258pub fn compute_permutation_from_indices(
284 original_indices: &[DynIndex],
285 new_indices: &[DynIndex],
286) -> std::result::Result<Vec<usize>, IdxTensorError> {
287 if !(new_indices.len() == original_indices.len()) {
288 return Err(
289 anyhow::anyhow!("new_indices length must match original_indices length").into(),
290 );
291 };
292
293 let mut perm = Vec::with_capacity(new_indices.len());
294 let mut used = std::collections::HashSet::new();
295
296 for new_idx in new_indices {
297 let pos = original_indices
300 .iter()
301 .position(|old_idx| old_idx == new_idx)
302 .ok_or_else(|| {
303 anyhow::anyhow!("new_indices must be a permutation of original_indices")
304 })?;
305
306 if !(used.insert(pos)) {
307 return Err(anyhow::anyhow!("duplicate index in new_indices").into());
308 };
309 perm.push(pos);
310 }
311
312 Ok(perm)
313}
314
315#[derive(Clone)]
320pub(crate) struct StructuredPayload {
321 payload: Arc<EagerTensor>,
322 payload_dims: Vec<usize>,
323 axis_classes: Vec<usize>,
324}
325
326#[derive(Debug, Clone, thiserror::Error)]
345pub enum TensorStorageError {
346 #[error("failed to materialize IdxTensor storage: {source}")]
348 Materialization {
349 #[source]
351 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
352 },
353 #[error(
355 "compact IdxTensor storage does not support dtype {dtype}; the eager payload remains authoritative"
356 )]
357 UnsupportedDtype {
358 dtype: &'static str,
360 },
361 #[error("failed to conjugate IdxTensor storage: {source}")]
364 Conjugation {
365 #[source]
367 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
368 },
369}
370
371#[derive(Debug, Clone, thiserror::Error)]
397pub enum IdxTensorError {
398 #[error("IdxTensor storage operation failed: {source}")]
400 Storage {
401 #[source]
403 source: TensorStorageError,
404 },
405 #[error("IdxTensor materialization failed: {source}")]
407 Materialization {
408 #[source]
410 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
411 },
412 #[error("IdxTensor engine entry failed: {source}")]
423 EngineEntry {
424 #[source]
426 source: tensor4all_tensorbackend::ExecutionContextEntryError,
427 },
428 #[error("IdxTensor scalar extraction failed: {source}")]
430 ScalarExtraction {
431 #[source]
433 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
434 },
435 #[error("IdxTensor payload scalar type does not match the requested type")]
442 PayloadScalarTypeMismatch,
443
444 #[error("IdxTensor scalar type mismatch: expected {expected}, got {actual}")]
447 ScalarTypeMismatch {
448 expected: &'static str,
450 actual: String,
452 },
453 #[error("IdxTensor shape mismatch during {operation}: expected {expected}, got {actual}")]
457 ShapeMismatch {
458 operation: &'static str,
460 expected: String,
462 actual: String,
464 },
465 #[error("IdxTensor subtraction failed: {source}")]
467 Subtraction {
468 #[source]
470 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
471 },
472 #[error("IdxTensor {operation} received NaN input")]
475 NaNInput {
476 operation: &'static str,
478 },
479 #[error("IdxTensor tolerance {name} is invalid: {value}")]
481 InvalidTolerance {
482 name: &'static str,
484 value: f64,
486 },
487 #[error("IdxTensor {operation} failed: {source}")]
489 Operation {
490 operation: &'static str,
492 #[source]
494 source: Arc<dyn std::error::Error + Send + Sync + 'static>,
495 },
496 #[error(
500 "IdxTensor cannot execute in the supplied execution context; \
501 rebuild tensor4all-core with `{required_feature}` enabled"
502 )]
503 UnsupportedExecutionContext {
504 required_feature: &'static str,
506 },
507}
508
509impl From<anyhow::Error> for IdxTensorError {
510 fn from(source: anyhow::Error) -> Self {
514 source
515 .downcast::<Self>()
516 .unwrap_or_else(|source| Self::operation("IdxTensor", source))
517 }
518}
519
520impl From<TensorStorageError> for IdxTensorError {
521 fn from(source: TensorStorageError) -> Self {
522 Self::Storage { source }
523 }
524}
525
526impl From<tenferro_ad::Error> for IdxTensorError {
527 fn from(source: tenferro_ad::Error) -> Self {
528 Self::Materialization {
529 source: Arc::from(anyhow::Error::new(source).into_boxed_dyn_error()),
530 }
531 }
532}
533
534impl From<tensor4all_tensorbackend::EagerContextError> for IdxTensorError {
535 fn from(source: tensor4all_tensorbackend::EagerContextError) -> Self {
536 Self::Materialization {
537 source: Arc::from(anyhow::Error::new(source).into_boxed_dyn_error()),
538 }
539 }
540}
541
542impl From<tensor4all_tensorbackend::CpuExecutionContextError> for IdxTensorError {
543 fn from(source: tensor4all_tensorbackend::CpuExecutionContextError) -> Self {
544 Self::EngineEntry {
547 source: source.into(),
548 }
549 }
550}
551
552#[cfg(feature = "tenferro-cuda")]
553impl From<tensor4all_tensorbackend::CudaExecutionContextError> for IdxTensorError {
554 fn from(source: tensor4all_tensorbackend::CudaExecutionContextError) -> Self {
555 Self::EngineEntry {
556 source: source.into(),
557 }
558 }
559}
560
561impl From<tensor4all_tensorbackend::ExecutionContextEntryError> for IdxTensorError {
562 fn from(source: tensor4all_tensorbackend::ExecutionContextEntryError) -> Self {
563 Self::EngineEntry { source }
564 }
565}
566
567impl From<tensor4all_tensorbackend::BridgeError> for IdxTensorError {
568 fn from(source: tensor4all_tensorbackend::BridgeError) -> Self {
569 Self::Materialization {
570 source: Arc::new(source),
571 }
572 }
573}
574
575impl IdxTensorError {
576 fn boxed(error: anyhow::Error) -> Arc<dyn std::error::Error + Send + Sync + 'static> {
577 Arc::from(error.into_boxed_dyn_error())
578 }
579
580 pub(crate) fn materialization(error: anyhow::Error) -> Self {
581 Self::Materialization {
582 source: Self::boxed(error),
583 }
584 }
585
586 pub fn is_payload_scalar_type_mismatch(&self) -> bool {
589 matches!(self, Self::PayloadScalarTypeMismatch)
590 }
591
592 fn scalar_extraction(error: anyhow::Error) -> Self {
593 Self::ScalarExtraction {
594 source: Self::boxed(error),
595 }
596 }
597
598 pub(crate) fn operation(operation: &'static str, error: anyhow::Error) -> Self {
599 Self::Operation {
600 operation,
601 source: Self::boxed(error),
602 }
603 }
604}
605
606#[cfg(not(feature = "tenferro-cuda"))]
608pub(crate) fn unsupported_execution_context_error() -> IdxTensorError {
609 IdxTensorError::UnsupportedExecutionContext {
610 required_feature: "tensor4all-core/tenferro-cuda",
611 }
612}
613
614#[cfg(not(feature = "tenferro-cuda"))]
616fn unsupported_factorize_context_error() -> FactorizeError {
617 FactorizeError::ComputationError(anyhow::Error::new(unsupported_execution_context_error()))
618}
619
620#[cfg(not(feature = "tenferro-cuda"))]
625fn ensure_supported_execution_context(
626 context: &ExecutionContext,
627) -> std::result::Result<(), IdxTensorError> {
628 if matches!(context, ExecutionContext::Cpu(_)) {
629 Ok(())
630 } else {
631 Err(unsupported_execution_context_error())
632 }
633}
634
635#[derive(Clone)]
636pub(crate) enum IdxTensorStorage {
637 Materialized(Arc<Storage>),
638 Eager {
639 inner: Arc<EagerTensor>,
640 axis_classes: Vec<usize>,
641 },
642 Compact(Arc<StructuredPayload>),
644 Deferred {
647 source: Box<Self>,
648 error: Arc<TensorStorageError>,
649 },
650}
651
652const CONTENT_DIGEST_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
658const CONTENT_DIGEST_MULTIPLIER: u64 = 0x0000_0100_0000_01b3;
659
660#[inline]
668fn digest_word(state: u64, word: u64) -> u64 {
669 let mixed = state
670 .wrapping_add(word)
671 .wrapping_mul(CONTENT_DIGEST_MULTIPLIER);
672 mixed ^ (mixed >> 32)
673}
674
675fn storage_kind_tag(kind: StorageKind) -> u64 {
676 match kind {
677 StorageKind::Dense => 1,
678 StorageKind::Diagonal => 2,
679 StorageKind::Structured => 3,
680 }
681}
682
683fn dtype_tag(dtype: DType) -> Option<u64> {
685 Some(match dtype {
686 DType::F32 => 1,
687 DType::F64 => 2,
688 DType::C32 => 3,
689 DType::C64 => 4,
690 _ => return None,
691 })
692}
693
694trait DigestScalar {
700 fn digest_words(value: &Self) -> (u64, Option<u64>);
702}
703
704impl DigestScalar for f32 {
705 fn digest_words(value: &Self) -> (u64, Option<u64>) {
706 (u64::from(value.to_bits()), None)
707 }
708}
709
710impl DigestScalar for f64 {
711 fn digest_words(value: &Self) -> (u64, Option<u64>) {
712 (value.to_bits(), None)
713 }
714}
715
716impl DigestScalar for Complex32 {
717 fn digest_words(value: &Self) -> (u64, Option<u64>) {
718 (
719 u64::from(value.re.to_bits()),
720 Some(u64::from(value.im.to_bits())),
721 )
722 }
723}
724
725impl DigestScalar for Complex64 {
726 fn digest_words(value: &Self) -> (u64, Option<u64>) {
727 (value.re.to_bits(), Some(value.im.to_bits()))
728 }
729}
730
731fn digest_values<T: DigestScalar>(mut state: u64, values: &[T]) -> u64 {
733 state = digest_word(state, values.len() as u64);
734 for value in values {
735 let (real, imaginary) = T::digest_words(value);
736 state = digest_word(state, real);
737 if let Some(imaginary) = imaginary {
738 state = digest_word(state, imaginary);
739 }
740 }
741 state
742}
743
744fn digest_storage(state: u64, storage: &Storage) -> Option<u64> {
746 match storage.dtype() {
747 DType::F32 => Some(digest_values(
748 state,
749 <f32 as StorageScalar>::payload_view(storage).ok()??,
750 )),
751 DType::F64 => Some(digest_values(
752 state,
753 <f64 as StorageScalar>::payload_view(storage).ok()??,
754 )),
755 DType::C32 => Some(digest_values(
756 state,
757 <Complex32 as StorageScalar>::payload_view(storage).ok()??,
758 )),
759 DType::C64 => Some(digest_values(
760 state,
761 <Complex64 as StorageScalar>::payload_view(storage).ok()??,
762 )),
763 _ => None,
764 }
765}
766
767fn digest_eager(mut state: u64, inner: &EagerTensor) -> Option<u64> {
773 let read = inner.tensor_read();
774 state = match inner.dtype() {
775 DType::F32 => digest_values(state, read.as_slice::<f32>().ok()?),
776 DType::F64 => digest_values(state, read.as_slice::<f64>().ok()?),
777 DType::C32 => digest_values(state, read.as_slice::<Complex32>().ok()?),
778 DType::C64 => digest_values(state, read.as_slice::<Complex64>().ok()?),
779 _ => return None,
780 };
781 if !inner.tracks_grad() {
782 return Some(state);
783 }
784 let Some(gradient) = inner.grad().ok()? else {
785 return Some(digest_word(state, 0));
786 };
787 state = digest_word(state, 1);
788 state = digest_word(state, gradient.shape().len() as u64);
789 for &dim in gradient.shape() {
790 state = digest_word(state, dim as u64);
791 }
792 let state = match gradient.dtype() {
793 DType::F32 => digest_values(state, gradient.as_slice::<f32>().ok()?),
794 DType::F64 => digest_values(state, gradient.as_slice::<f64>().ok()?),
795 DType::C32 => digest_values(state, gradient.as_slice::<Complex32>().ok()?),
796 DType::C64 => digest_values(state, gradient.as_slice::<Complex64>().ok()?),
797 _ => return None,
798 };
799 Some(state)
800}
801
802impl IdxTensorStorage {
803 fn from_storage(storage: Arc<Storage>) -> Self {
804 Self::Materialized(storage)
805 }
806
807 fn from_eager_dense(inner: EagerTensor, rank: usize) -> Self {
808 Self::Eager {
809 inner: Arc::new(inner),
810 axis_classes: IdxTensor::dense_axis_classes(rank),
811 }
812 }
813
814 fn eager(&self) -> Option<&EagerTensor> {
815 match self {
816 Self::Materialized(_) => None,
817 Self::Eager { inner, .. } => Some(inner.as_ref()),
818 Self::Compact(payload) => Some(payload.payload.as_ref()),
819 Self::Deferred { source, .. } => source.eager(),
820 }
821 }
822
823 fn deferred_error(&self) -> Option<&TensorStorageError> {
824 match self {
825 Self::Deferred { error, .. } => Some(error.as_ref()),
826 _ => None,
827 }
828 }
829
830 fn with_deferred_error(self, error: TensorStorageError) -> Self {
831 if self.deferred_error().is_some() {
832 self
833 } else {
834 Self::Deferred {
835 source: Box::new(self),
836 error: Arc::new(error),
837 }
838 }
839 }
840
841 fn axis_classes(&self) -> &[usize] {
842 match self {
843 Self::Materialized(storage) => storage.axis_classes(),
844 Self::Eager { axis_classes, .. } => axis_classes,
845 Self::Compact(payload) => &payload.axis_classes,
846 Self::Deferred { source, .. } => source.axis_classes(),
847 }
848 }
849
850 fn payload_dims(&self) -> &[usize] {
851 match self {
852 Self::Materialized(storage) => storage.payload_dims(),
853 Self::Eager { inner, .. } => inner.shape(),
854 Self::Compact(payload) => &payload.payload_dims,
855 Self::Deferred { source, .. } => source.payload_dims(),
856 }
857 }
858
859 fn payload_strides_vec(&self) -> Vec<isize> {
860 match self {
861 Self::Materialized(storage) => storage.payload_strides().to_vec(),
862 Self::Eager { inner, .. } => {
863 IdxTensor::col_major_strides(inner.shape()).unwrap_or_default()
864 }
865 Self::Compact(payload) => {
866 IdxTensor::col_major_strides(&payload.payload_dims).unwrap_or_default()
867 }
868 Self::Deferred { source, .. } => source.payload_strides_vec(),
869 }
870 }
871
872 fn is_f64(&self) -> bool {
873 match self {
874 Self::Materialized(storage) => storage.is_f64(),
875 Self::Eager { inner, .. } => inner.dtype() == DType::F64,
876 Self::Compact(payload) => payload.payload.dtype() == DType::F64,
877 Self::Deferred { source, .. } => source.is_f64(),
878 }
879 }
880
881 fn is_c64(&self) -> bool {
882 match self {
883 Self::Materialized(storage) => storage.is_c64(),
884 Self::Eager { inner, .. } => inner.dtype() == DType::C64,
885 Self::Compact(payload) => payload.payload.dtype() == DType::C64,
886 Self::Deferred { source, .. } => source.is_c64(),
887 }
888 }
889
890 fn dtype(&self) -> Option<DType> {
891 match self {
892 Self::Materialized(storage) => Some(storage.dtype()),
893 Self::Eager { inner, .. } => Some(inner.dtype()),
894 Self::Compact(payload) => Some(payload.payload.dtype()),
895 Self::Deferred { source, .. } => source.dtype(),
896 }
897 }
898
899 fn is_complex(&self) -> bool {
900 match self {
901 Self::Materialized(storage) => storage.is_complex(),
902 Self::Eager { inner, .. } => matches!(inner.dtype(), DType::C32 | DType::C64),
903 Self::Compact(payload) => {
904 matches!(payload.payload.dtype(), DType::C32 | DType::C64)
905 }
906 Self::Deferred { source, .. } => source.is_complex(),
907 }
908 }
909
910 fn is_diag(&self) -> bool {
911 match self {
912 Self::Materialized(storage) => storage.is_diag(),
913 Self::Eager { axis_classes, .. } => IdxTensor::is_diag_axis_classes(axis_classes),
914 Self::Compact(payload) => IdxTensor::is_diag_axis_classes(&payload.axis_classes),
915 Self::Deferred { source, .. } => source.is_diag(),
916 }
917 }
918
919 fn storage_kind(&self) -> StorageKind {
920 match self {
921 Self::Materialized(storage) => storage.storage_kind(),
922 Self::Eager { axis_classes, .. } => {
923 if axis_classes.iter().copied().eq(0..axis_classes.len()) {
924 StorageKind::Dense
925 } else if IdxTensor::is_diag_axis_classes(axis_classes) {
926 StorageKind::Diagonal
927 } else {
928 StorageKind::Structured
929 }
930 }
931 Self::Compact(payload) => {
932 if payload
933 .axis_classes
934 .iter()
935 .copied()
936 .eq(0..payload.axis_classes.len())
937 {
938 StorageKind::Dense
939 } else if IdxTensor::is_diag_axis_classes(&payload.axis_classes) {
940 StorageKind::Diagonal
941 } else {
942 StorageKind::Structured
943 }
944 }
945 Self::Deferred { source, .. } => source.storage_kind(),
946 }
947 }
948
949 fn content_digest(&self) -> Option<u64> {
956 let dtype = self.dtype()?;
957 let mut state = digest_word(CONTENT_DIGEST_OFFSET, storage_kind_tag(self.storage_kind()));
958 state = digest_word(state, dtype_tag(dtype)?);
959 state = digest_word(state, self.payload_dims().len() as u64);
960 for &dim in self.payload_dims() {
961 state = digest_word(state, dim as u64);
962 }
963 for &axis_class in self.axis_classes() {
964 state = digest_word(state, axis_class as u64);
965 }
966 let tracked = self.eager().is_some_and(EagerTensor::tracks_grad);
969 state = digest_word(state, u64::from(tracked));
970 match self {
971 Self::Materialized(storage) => digest_storage(state, storage),
972 Self::Eager { inner, .. } => digest_eager(state, inner),
973 Self::Compact(payload) => digest_eager(state, &payload.payload),
974 Self::Deferred { .. } => None,
975 }
976 }
977
978 fn refuse_unsupported_compact_dtype(
984 native: &NativeTensor,
985 ) -> std::result::Result<(), TensorStorageError> {
986 if matches!(native.dtype(), DType::External(_)) {
987 return Err(TensorStorageError::UnsupportedDtype {
988 dtype: IdxTensor::dtype_name(native.dtype()),
989 });
990 }
991 Ok(())
992 }
993
994 fn materialize_eager_payload(inner: &EagerTensor) -> Result<NativeTensor> {
995 let native = with_engine(inner.runtime(), None, |engine| {
996 inner.duplicate_value(engine)
997 })?;
998 if native.is_col_major_contiguous()? {
999 return Ok(native);
1000 }
1001 let read = inner.tensor_read();
1002 let view = read
1003 .tensor_view()
1004 .ok_or_else(|| anyhow::anyhow!("external scalar kind payload has no tensor view"))?;
1005 with_engine(inner.runtime(), None, |engine| {
1006 engine.to_contiguous_read(TensorRead::from_view(view))
1007 })
1008 .map_err(anyhow::Error::new)
1009 }
1010
1011 fn materialize(
1012 &self,
1013 logical_rank: usize,
1014 ) -> std::result::Result<Arc<Storage>, TensorStorageError> {
1015 match self {
1016 Self::Materialized(storage) => Ok(Arc::clone(storage)),
1017 Self::Eager {
1018 inner,
1019 axis_classes,
1020 } => {
1021 let native = Self::materialize_eager_payload(inner).map_err(|source| {
1022 TensorStorageError::Materialization {
1023 source: Arc::from(source.into_boxed_dyn_error()),
1024 }
1025 })?;
1026 Self::refuse_unsupported_compact_dtype(&native)?;
1027 IdxTensor::storage_from_native_with_axis_classes(
1028 &native,
1029 axis_classes,
1030 logical_rank,
1031 )
1032 .map(Arc::new)
1033 .map_err(|source| TensorStorageError::Materialization {
1034 source: Arc::from(source.into_boxed_dyn_error()),
1035 })
1036 }
1037 Self::Compact(payload) => {
1038 let native =
1039 Self::materialize_eager_payload(&payload.payload).map_err(|source| {
1040 TensorStorageError::Materialization {
1041 source: Arc::from(source.into_boxed_dyn_error()),
1042 }
1043 })?;
1044 Self::refuse_unsupported_compact_dtype(&native)?;
1045 IdxTensor::storage_from_native_with_axis_classes(
1046 &native,
1047 &payload.axis_classes,
1048 logical_rank,
1049 )
1050 .map(Arc::new)
1051 .map_err(|source| TensorStorageError::Materialization {
1052 source: Arc::from(source.into_boxed_dyn_error()),
1053 })
1054 }
1055 Self::Deferred { error, .. } => Err((**error).clone()),
1056 }
1057 }
1058
1059 fn scale_eager_payload(&self, scalar: &AnyScalar) -> Result<Self> {
1060 let (payload, payload_dims, axis_classes) = match self {
1061 Self::Materialized(storage) => {
1062 let native = storage_payload_native(storage)?;
1065 (
1066 IdxTensor::untracked_inner(native, default_eager_ctx()?)?,
1067 storage.payload_dims().to_vec(),
1068 storage.axis_classes().to_vec(),
1069 )
1070 }
1071 Self::Eager {
1072 inner,
1073 axis_classes,
1074 } => (
1075 (**inner).clone(),
1076 inner.shape().to_vec(),
1077 axis_classes.clone(),
1078 ),
1079 Self::Compact(compact) => (
1080 (*compact.payload).clone(),
1081 compact.payload_dims.clone(),
1082 compact.axis_classes.clone(),
1083 ),
1084 Self::Deferred { error, .. } => return Err(anyhow::Error::new((**error).clone())),
1085 };
1086 let scalar_inner = scalar.as_tensor()?.try_materialized_inner()?;
1087 let target_dtype = IdxTensor::scale_target_dtype(payload.dtype(), scalar_inner.dtype())?;
1088 let payload = if payload.dtype() == target_dtype {
1089 payload
1090 } else {
1091 with_eager_session(payload.runtime(), None, |session| {
1092 session.cast(&payload, target_dtype)
1093 })?
1094 };
1095 let scalar_inner = if scalar_inner.dtype() == target_dtype {
1096 scalar_inner.clone()
1097 } else {
1098 with_eager_session(scalar_inner.runtime(), None, |session| {
1099 session.cast(scalar_inner, target_dtype)
1100 })?
1101 };
1102 let scaled = if payload.shape().is_empty() {
1103 with_eager_session(payload.runtime(), None, |session| {
1104 session.mul(&payload, &scalar_inner)
1105 })?
1106 } else {
1107 let subscripts = IdxTensor::scale_subscripts(payload.shape().len())?;
1108 with_eager_session(payload.runtime(), None, |session| {
1109 session.einsum_subscripts(&[&payload, &scalar_inner], &subscripts)
1110 })?
1111 };
1112 match self {
1113 Self::Eager { .. } => Ok(Self::Eager {
1114 inner: Arc::new(scaled),
1115 axis_classes,
1116 }),
1117 Self::Compact(_) | Self::Materialized(_) => {
1118 Ok(Self::Compact(Arc::new(StructuredPayload {
1119 payload: Arc::new(scaled),
1120 payload_dims,
1121 axis_classes,
1122 })))
1123 }
1124 Self::Deferred { error, .. } => Err(anyhow::Error::new((**error).clone())),
1125 }
1126 }
1127
1128 fn conjugate_with<F>(&self, conjugate: &F) -> std::result::Result<Self, TensorStorageError>
1129 where
1130 F: Fn(
1131 &EagerTensor,
1132 ) -> std::result::Result<
1133 EagerTensor,
1134 Arc<dyn std::error::Error + Send + Sync + 'static>,
1135 >,
1136 {
1137 match self {
1138 Self::Materialized(storage) => Ok(Self::Materialized(Arc::new(storage.conj()))),
1139 Self::Eager {
1140 inner,
1141 axis_classes,
1142 } => conjugate(inner)
1143 .map(|conjugated| Self::Eager {
1144 inner: Arc::new(conjugated),
1145 axis_classes: axis_classes.clone(),
1146 })
1147 .map_err(|source| TensorStorageError::Conjugation { source }),
1148 Self::Compact(payload) => conjugate(payload.payload.as_ref())
1149 .map(|conjugated| {
1150 Self::Compact(Arc::new(StructuredPayload {
1151 payload: Arc::new(conjugated),
1152 payload_dims: payload.payload_dims.clone(),
1153 axis_classes: payload.axis_classes.clone(),
1154 }))
1155 })
1156 .map_err(|source| TensorStorageError::Conjugation { source }),
1157 Self::Deferred { error, .. } => Err((**error).clone()),
1158 }
1159 }
1160
1161 fn sum_scalar(&self) -> Result<AnyScalar> {
1162 match self {
1163 Self::Materialized(storage) => {
1164 if storage.is_f64() {
1165 Ok(AnyScalar::new_real(storage.sum::<f64>()))
1166 } else {
1167 let value = storage.sum::<Complex64>();
1168 Ok(AnyScalar::new_complex(value.re, value.im))
1169 }
1170 }
1171 Self::Eager { inner, .. } => IdxTensor::native_sum_scalar(inner),
1172 Self::Compact(payload) => IdxTensor::native_sum_scalar(&payload.payload),
1173 Self::Deferred { error, .. } => Err(anyhow::Error::new((**error).clone())),
1174 }
1175 }
1176
1177 fn nonfinite_flags(&self) -> Result<NonfiniteFlags> {
1178 match self {
1179 Self::Materialized(storage) => Ok(storage.payload_nonfinite_flags()),
1180 Self::Eager { inner, .. } => IdxTensor::native_nonfinite_flags(inner),
1181 Self::Compact(payload) => IdxTensor::native_nonfinite_flags(&payload.payload),
1182 Self::Deferred { error, .. } => Err(anyhow::Error::new((**error).clone())),
1183 }
1184 }
1185
1186 fn payload_value_at(&self, payload_coords: &[usize]) -> Result<Complex64> {
1187 match self {
1188 Self::Materialized(storage) => storage
1189 .scalar_at(payload_coords)
1190 .map(Complex64::from)
1191 .map_err(anyhow::Error::new),
1192 Self::Eager { inner, .. } => {
1193 IdxTensor::native_complex_payload_value_at(inner, payload_coords)
1194 }
1195 Self::Compact(payload) => {
1196 IdxTensor::native_complex_payload_value_at(&payload.payload, payload_coords)
1197 }
1198 Self::Deferred { error, .. } => Err(anyhow::Error::new((**error).clone())),
1199 }
1200 }
1201
1202 fn for_each_payload_value(&self, mut f: impl FnMut(Complex64)) -> Result<()> {
1203 let payload_dims = self.payload_dims();
1204 let payload_len = checked_product(payload_dims)?;
1205 let native = match self {
1208 Self::Eager { inner, .. } => Some(inner),
1209 Self::Compact(payload) => Some(&payload.payload),
1210 _ => None,
1211 };
1212 let value = match native {
1213 Some(native) => {
1214 #[cfg(test)]
1215 note_eager_payload_read();
1216 Some(native.value()?)
1217 }
1218 None => None,
1219 };
1220 let mut payload_coords = vec![0usize; payload_dims.len()];
1221 for _ in 0..payload_len {
1222 match &value {
1223 Some(value) => f(IdxTensor::view_complex_value_at(
1224 value.as_tensor_view(),
1225 &payload_coords,
1226 )?),
1227 None => f(self.payload_value_at(&payload_coords)?),
1228 }
1229 let mut carry = true;
1230 for (coordinate, &dim) in payload_coords.iter_mut().zip(payload_dims.iter()) {
1231 if !carry {
1232 break;
1233 }
1234 *coordinate += 1;
1235 if *coordinate == dim {
1236 *coordinate = 0;
1237 } else {
1238 carry = false;
1239 }
1240 }
1241 }
1242 Ok(())
1243 }
1244
1245 fn compact_payload(&self) -> Option<&StructuredPayload> {
1246 match self {
1247 Self::Compact(payload) => Some(payload.as_ref()),
1248 Self::Deferred { source, .. } => source.compact_payload(),
1249 _ => None,
1250 }
1251 }
1252}
1253
1254#[derive(Debug, thiserror::Error)]
1269pub enum StructuredSelectorError {
1270 #[error("copy-selector bond dimensions differ: left={left}, right={right}")]
1272 BondDimensionMismatch {
1273 left: usize,
1275 right: usize,
1277 },
1278 #[error("copy-selector {axis} dimension must be positive")]
1280 ZeroDimension {
1281 axis: &'static str,
1283 },
1284 #[error("selected site value {value} is outside 0..{site_dim}")]
1286 SelectedValueOutOfBounds {
1287 value: usize,
1289 site_dim: usize,
1291 },
1292 #[error("copy-selector payload size overflows usize for dimensions {bond_dim} x {site_dim}")]
1294 PayloadSizeOverflow {
1295 bond_dim: usize,
1297 site_dim: usize,
1299 },
1300 #[error("copy-selector bond stride {bond_dim} exceeds isize::MAX")]
1302 StrideOverflow {
1303 bond_dim: usize,
1305 },
1306 #[error("could not allocate copy-selector payload with {elements} elements")]
1308 AllocationFailed {
1309 elements: usize,
1311 },
1312 #[error("invalid copy-selector storage: {message}")]
1314 InvalidStorage {
1315 message: String,
1317 },
1318}
1319
1320#[derive(Clone)]
1364pub struct IdxTensor {
1365 pub(crate) indices: Vec<DynIndex>,
1370 pub(crate) storage: IdxTensorStorage,
1374 pub(crate) eager_cache: Arc<OnceLock<Arc<EagerTensor>>>,
1376}
1377
1378impl IdxTensor {
1379 fn dense_axis_classes(rank: usize) -> Vec<usize> {
1380 (0..rank).collect()
1381 }
1382
1383 fn dtype_name(dtype: DType) -> &'static str {
1384 match dtype {
1385 DType::F32 => "f32",
1386 DType::F64 => "f64",
1387 DType::C32 => "c32",
1388 DType::C64 => "c64",
1389 DType::I32 => "i32",
1390 DType::I64 => "i64",
1391 DType::Bool => "bool",
1392 DType::External(_) => "external",
1393 }
1394 }
1395
1396 fn scalar_dtype(&self) -> Result<DType> {
1397 if let Some(inner) = self.storage.eager() {
1398 return Ok(inner.dtype());
1399 }
1400 self.storage
1401 .dtype()
1402 .ok_or_else(|| anyhow::anyhow!("unable to determine IdxTensor scalar dtype"))
1403 }
1404
1405 fn diag_axis_classes(rank: usize) -> Vec<usize> {
1406 if rank == 0 {
1407 vec![]
1408 } else {
1409 vec![0; rank]
1410 }
1411 }
1412
1413 fn canonicalize_axis_classes(axis_classes: &[usize]) -> Vec<usize> {
1414 let mut map = std::collections::HashMap::new();
1415 let mut next = 0usize;
1416 axis_classes
1417 .iter()
1418 .map(|&class_id| {
1419 *map.entry(class_id).or_insert_with(|| {
1420 let canonical = next;
1421 next += 1;
1422 canonical
1423 })
1424 })
1425 .collect()
1426 }
1427
1428 fn permute_axis_classes(&self, perm: &[usize]) -> Vec<usize> {
1429 let axis_classes = self.storage.axis_classes();
1430 let permuted: Vec<usize> = perm.iter().map(|&index| axis_classes[index]).collect();
1431 Self::canonicalize_axis_classes(&permuted)
1432 }
1433
1434 fn normalize_insert_axis(op: &str, axis: isize, rank: usize) -> Result<usize> {
1435 let normalized = if axis < 0 {
1436 rank as isize + 1 + axis
1437 } else {
1438 axis
1439 };
1440 if !(normalized >= 0 && normalized <= rank as isize) {
1441 return Err(anyhow::anyhow!(
1442 "{op}: axis {axis} is out of bounds for inserting into rank {rank}"
1443 ));
1444 };
1445 Ok(normalized as usize)
1446 }
1447
1448 fn is_diag_axis_classes(axis_classes: &[usize]) -> bool {
1449 axis_classes.len() >= 2 && axis_classes.iter().all(|&class_id| class_id == 0)
1450 }
1451
1452 fn validate_axis_classes(axis_classes: &[usize], rank: usize) -> Result<()> {
1453 if axis_classes.len() != rank {
1454 return Err(anyhow::anyhow!(
1455 "axis-class rank {} does not match tensor rank {rank}",
1456 axis_classes.len()
1457 ));
1458 }
1459 if Self::canonicalize_axis_classes(axis_classes) != axis_classes {
1460 return Err(anyhow::anyhow!(
1461 "axis classes must be canonical first-occurrence labels: {axis_classes:?}"
1462 ));
1463 }
1464 Ok(())
1465 }
1466
1467 fn einsum_subscripts_from_usize_ids(
1468 inputs: &[Vec<usize>],
1469 output: &[usize],
1470 ) -> Result<EinsumSubscripts> {
1471 let input_labels = inputs
1472 .iter()
1473 .map(|ids| {
1474 ids.iter()
1475 .map(|&id| {
1476 u32::try_from(id)
1477 .map_err(|_| anyhow::anyhow!("einsum label {id} exceeds u32 range"))
1478 })
1479 .collect::<Result<Vec<_>>>()
1480 })
1481 .collect::<Result<Vec<_>>>()?;
1482 let output_labels = output
1483 .iter()
1484 .map(|&id| {
1485 u32::try_from(id)
1486 .map_err(|_| anyhow::anyhow!("einsum label {id} exceeds u32 range"))
1487 })
1488 .collect::<Result<Vec<_>>>()?;
1489 let input_refs = input_labels.iter().map(Vec::as_slice).collect::<Vec<_>>();
1490 Ok(EinsumSubscripts::new(&input_refs, &output_labels))
1491 }
1492
1493 fn build_binary_einsum_subscripts(
1494 lhs_rank: usize,
1495 axes_a: &[usize],
1496 rhs_rank: usize,
1497 axes_b: &[usize],
1498 ) -> Result<EinsumSubscripts> {
1499 if !(axes_a.len() == axes_b.len()) {
1500 return Err(anyhow::anyhow!(
1501 "contract axis length mismatch: lhs {:?}, rhs {:?}",
1502 axes_a,
1503 axes_b
1504 ));
1505 };
1506
1507 let mut lhs_ids = vec![usize::MAX; lhs_rank];
1508 let mut rhs_ids = vec![usize::MAX; rhs_rank];
1509 let mut next_id = 0usize;
1510
1511 let mut seen_lhs = vec![false; lhs_rank];
1512 let mut seen_rhs = vec![false; rhs_rank];
1513
1514 for (&lhs_axis, &rhs_axis) in axes_a.iter().zip(axes_b.iter()) {
1515 if !(lhs_axis < lhs_rank) {
1516 return Err(anyhow::anyhow!("lhs contract axis {lhs_axis} out of range"));
1517 };
1518 if !(rhs_axis < rhs_rank) {
1519 return Err(anyhow::anyhow!("rhs contract axis {rhs_axis} out of range"));
1520 };
1521 if !(!seen_lhs[lhs_axis]) {
1522 return Err(anyhow::anyhow!("duplicate lhs contract axis {lhs_axis}"));
1523 };
1524 if !(!seen_rhs[rhs_axis]) {
1525 return Err(anyhow::anyhow!("duplicate rhs contract axis {rhs_axis}"));
1526 };
1527 seen_lhs[lhs_axis] = true;
1528 seen_rhs[rhs_axis] = true;
1529 lhs_ids[lhs_axis] = next_id;
1530 rhs_ids[rhs_axis] = next_id;
1531 next_id += 1;
1532 }
1533
1534 let mut output_ids = Vec::with_capacity(lhs_rank + rhs_rank - 2 * axes_a.len());
1535 for id in &mut lhs_ids {
1536 if *id == usize::MAX {
1537 *id = next_id;
1538 output_ids.push(next_id);
1539 next_id += 1;
1540 }
1541 }
1542 for id in &mut rhs_ids {
1543 if *id == usize::MAX {
1544 *id = next_id;
1545 output_ids.push(next_id);
1546 next_id += 1;
1547 }
1548 }
1549
1550 Self::einsum_subscripts_from_usize_ids(&[lhs_ids, rhs_ids], &output_ids)
1551 }
1552
1553 fn binary_dot_general_config(axes_a: &[usize], axes_b: &[usize]) -> Result<DotGeneralConfig> {
1554 if !(axes_a.len() == axes_b.len()) {
1555 return Err(anyhow::anyhow!(
1556 "contract axis length mismatch: lhs {:?}, rhs {:?}",
1557 axes_a,
1558 axes_b
1559 ));
1560 };
1561 Ok(DotGeneralConfig {
1562 lhs_contracting_dims: axes_a.into(),
1563 rhs_contracting_dims: axes_b.into(),
1564 lhs_batch_dims: [].as_slice().into(),
1565 rhs_batch_dims: [].as_slice().into(),
1566 })
1567 }
1568
1569 fn binary_contraction_axis_classes(
1570 lhs_axis_classes: &[usize],
1571 axes_a: &[usize],
1572 rhs_axis_classes: &[usize],
1573 axes_b: &[usize],
1574 ) -> Result<Vec<usize>> {
1575 debug_assert_eq!(axes_a.len(), axes_b.len());
1576
1577 fn find(parent: &mut [usize], value: usize) -> usize {
1578 if parent[value] != value {
1579 parent[value] = find(parent, parent[value]);
1580 }
1581 parent[value]
1582 }
1583
1584 fn union(parent: &mut [usize], lhs: usize, rhs: usize) {
1585 let lhs_root = find(parent, lhs);
1586 let rhs_root = find(parent, rhs);
1587 if lhs_root != rhs_root {
1588 parent[rhs_root] = lhs_root;
1589 }
1590 }
1591
1592 let lhs_payload_rank = match lhs_axis_classes.iter().copied().max() {
1593 Some(value) => value
1594 .checked_add(1)
1595 .ok_or_else(|| anyhow::anyhow!("left payload rank overflows usize"))?,
1596 None => 0,
1597 };
1598 let rhs_payload_rank = match rhs_axis_classes.iter().copied().max() {
1599 Some(value) => value
1600 .checked_add(1)
1601 .ok_or_else(|| anyhow::anyhow!("right payload rank overflows usize"))?,
1602 None => 0,
1603 };
1604 let rhs_offset = lhs_payload_rank;
1605 let parent_len = lhs_payload_rank
1606 .checked_add(rhs_payload_rank)
1607 .ok_or_else(|| anyhow::anyhow!("payload rank sum overflows usize"))?;
1608 let mut parent: Vec<usize> = (0..parent_len).collect();
1609
1610 for (&lhs_axis, &rhs_axis) in axes_a.iter().zip(axes_b.iter()) {
1611 union(
1612 &mut parent,
1613 lhs_axis_classes[lhs_axis],
1614 rhs_offset
1615 .checked_add(rhs_axis_classes[rhs_axis])
1616 .ok_or_else(|| anyhow::anyhow!("rhs axis-class offset overflows usize"))?,
1617 );
1618 }
1619
1620 let mut lhs_contracted = vec![false; lhs_axis_classes.len()];
1621 for &axis in axes_a {
1622 lhs_contracted[axis] = true;
1623 }
1624 let mut rhs_contracted = vec![false; rhs_axis_classes.len()];
1625 for &axis in axes_b {
1626 rhs_contracted[axis] = true;
1627 }
1628
1629 let mut root_to_class = std::collections::HashMap::new();
1630 let mut next_class = 0usize;
1631 let mut axis_classes = Vec::with_capacity(
1632 lhs_axis_classes
1633 .len()
1634 .saturating_add(rhs_axis_classes.len()),
1635 );
1636
1637 for (axis, &class_id) in lhs_axis_classes.iter().enumerate() {
1638 if !lhs_contracted[axis] {
1639 let root = find(&mut parent, class_id);
1640 let class = *root_to_class.entry(root).or_insert_with(|| {
1641 let value = next_class;
1642 next_class += 1;
1643 value
1644 });
1645 axis_classes.push(class);
1646 }
1647 }
1648 for (axis, &class_id) in rhs_axis_classes.iter().enumerate() {
1649 if !rhs_contracted[axis] {
1650 let rhs_class = rhs_offset
1651 .checked_add(class_id)
1652 .ok_or_else(|| anyhow::anyhow!("rhs axis-class offset overflows usize"))?;
1653 let root = find(&mut parent, rhs_class);
1654 let class = *root_to_class.entry(root).or_insert_with(|| {
1655 let value = next_class;
1656 next_class += 1;
1657 value
1658 });
1659 axis_classes.push(class);
1660 }
1661 }
1662
1663 Ok(axis_classes)
1664 }
1665
1666 fn scale_subscripts(rank: usize) -> Result<EinsumSubscripts> {
1667 let ids: Vec<usize> = (0..rank).collect();
1668 Self::einsum_subscripts_from_usize_ids(&[ids.clone(), Vec::new()], &ids)
1669 }
1670
1671 fn scale_target_dtype(payload: DType, scalar: DType) -> Result<DType> {
1672 tensor4all_tensorbackend::scale_target_dtype(payload, scalar).map_err(anyhow::Error::new)
1674 }
1675
1676 fn validate_indices(indices: &[DynIndex]) -> Result<()> {
1677 if !Self::indices_are_unique(indices) {
1678 return Err(anyhow::anyhow!("Tensor indices must all be unique"));
1679 }
1680 Ok(())
1681 }
1682
1683 fn validate_diag_dims(dims: &[usize]) -> Result<()> {
1684 if !dims.is_empty() {
1685 let first_dim = dims[0];
1686 for (i, &dim) in dims.iter().enumerate() {
1687 if !(dim == first_dim) {
1688 return Err(anyhow::anyhow!("DiagTensor requires all indices to have the same dimension, but dims[{i}] = {dim} != dims[0] = {first_dim}"));
1689 };
1690 }
1691 }
1692 Ok(())
1693 }
1694
1695 fn seed_native_payload(storage: &Storage, dims: &[usize]) -> Result<NativeTensor> {
1696 Ok(storage_to_native_tensor(storage, dims)?)
1697 }
1698
1699 fn untracked_inner(
1703 native: NativeTensor,
1704 runtime: Arc<EagerRuntime>,
1705 ) -> std::result::Result<EagerTensor, tenferro_ad::Error> {
1706 adopt_untracked_eager_value(runtime, TensorValue::from_tensor(native))
1707 }
1708
1709 fn empty_eager_cache() -> Arc<OnceLock<Arc<EagerTensor>>> {
1710 Arc::new(OnceLock::new())
1711 }
1712
1713 fn eager_cache_with(inner: EagerTensor) -> Arc<OnceLock<Arc<EagerTensor>>> {
1714 let cache = Arc::new(OnceLock::new());
1715 let _ = cache.set(Arc::new(inner));
1716 cache
1717 }
1718
1719 fn compact_payload_inner(&self) -> Result<EagerTensor> {
1720 self.ensure_storage_ready()?;
1721 if let Some(inner) = self.storage.eager() {
1722 return Ok(inner.clone());
1723 }
1724 Ok(IdxTensor::untracked_inner(
1725 storage_payload_native(self.storage.materialize(self.indices.len())?.as_ref())?,
1726 default_eager_ctx()?,
1727 )?)
1728 }
1729
1730 fn dense_inner_from_payload(
1731 payload: &EagerTensor,
1732 axis_classes: &[usize],
1733 logical_dims: &[usize],
1734 ) -> Result<EagerTensor> {
1735 let payload_rank = match axis_classes.iter().copied().max() {
1736 Some(class_id) => class_id
1737 .checked_add(1)
1738 .ok_or_else(|| anyhow::anyhow!("structured payload class rank overflows usize"))?,
1739 None => 0,
1740 };
1741 if !(payload.shape().len() == payload_rank) {
1742 return Err(anyhow::anyhow!(
1743 "structured payload rank {} does not match axis classes {:?}",
1744 payload.shape().len(),
1745 axis_classes
1746 ));
1747 };
1748 if !(logical_dims.len() == axis_classes.len()) {
1749 return Err(anyhow::anyhow!(
1750 "logical rank {} does not match axis class rank {}",
1751 logical_dims.len(),
1752 axis_classes.len()
1753 ));
1754 };
1755
1756 if axis_classes == Self::dense_axis_classes(logical_dims.len()) {
1757 if !(payload.shape() == logical_dims) {
1758 return Err(anyhow::anyhow!(
1759 "dense payload dims {:?} do not match logical dims {:?}",
1760 payload.shape(),
1761 logical_dims
1762 ));
1763 };
1764 return Ok(payload.clone());
1765 }
1766
1767 let mut first_axis_by_class = vec![None; payload_rank];
1768 let mut dense = payload.clone();
1769 for (logical_axis, &class_id) in axis_classes.iter().enumerate() {
1770 let first_axis = match first_axis_by_class[class_id] {
1771 Some(first_axis) => first_axis,
1772 None => {
1773 first_axis_by_class[class_id] = Some(logical_axis);
1774 continue;
1775 }
1776 };
1777 dense = with_eager_session(dense.runtime(), None, |session| {
1778 session.embed_diag(&dense, first_axis, logical_axis)
1779 })?;
1780 }
1781 if !(dense.shape() == logical_dims) {
1782 return Err(anyhow::anyhow!(
1783 "expanded structured payload dims {:?} do not match logical dims {:?}",
1784 dense.shape(),
1785 logical_dims
1786 ));
1787 };
1788 Ok(dense)
1789 }
1790
1791 fn tracked_compact_payload_value(&self) -> Option<&StructuredPayload> {
1792 self.storage
1793 .deferred_error()
1794 .is_none()
1795 .then_some(self.storage.compact_payload())
1796 .flatten()
1797 .filter(|value| value.payload.tracks_grad())
1798 }
1799
1800 fn ensure_storage_ready(&self) -> Result<()> {
1801 if let Some(error) = self.storage.deferred_error() {
1802 return Err(anyhow::Error::new(error.clone()));
1803 }
1804 Ok(())
1805 }
1806
1807 fn compact_payload_is_logical_dense(&self, payload_dims: &[usize]) -> bool {
1808 self.storage.axis_classes() == Self::dense_axis_classes(self.indices.len())
1809 && payload_dims == self.dims()
1810 }
1811
1812 fn uses_tracked_compact_storage(&self) -> bool {
1813 self.tracked_compact_payload_value()
1814 .is_some_and(|value| !self.compact_payload_is_logical_dense(&value.payload_dims))
1815 }
1816
1817 fn ensure_shape_packing_preserves_ad(&self, op_name: &str) -> Result<()> {
1818 self.ensure_storage_ready()?;
1819 if !(!self.uses_tracked_compact_storage()) {
1820 return Err(anyhow::anyhow!("{op_name}: structured AD tensors with compact storage are not supported because materializing compact storage would detach gradients"));
1821 };
1822 Ok(())
1823 }
1824
1825 pub(crate) fn operand_indices_for_contraction(&self, conjugate: bool) -> Vec<DynIndex> {
1826 if conjugate {
1827 self.indices.iter().map(|index| index.conj()).collect()
1828 } else {
1829 self.indices.clone()
1830 }
1831 }
1832
1833 fn build_binary_contraction_labels(
1834 lhs_rank: usize,
1835 axes_a: &[usize],
1836 rhs_rank: usize,
1837 axes_b: &[usize],
1838 ) -> Result<(Vec<usize>, Vec<usize>, Vec<usize>)> {
1839 if !(axes_a.len() == axes_b.len()) {
1840 return Err(anyhow::anyhow!(
1841 "contract axis length mismatch: lhs {:?}, rhs {:?}",
1842 axes_a,
1843 axes_b
1844 ));
1845 };
1846
1847 let mut lhs_ids = vec![usize::MAX; lhs_rank];
1848 let mut rhs_ids = vec![usize::MAX; rhs_rank];
1849 let mut next_id = 0usize;
1850
1851 let mut seen_lhs = vec![false; lhs_rank];
1852 let mut seen_rhs = vec![false; rhs_rank];
1853
1854 for (&lhs_axis, &rhs_axis) in axes_a.iter().zip(axes_b.iter()) {
1855 if !(lhs_axis < lhs_rank) {
1856 return Err(anyhow::anyhow!("lhs contract axis {lhs_axis} out of range"));
1857 };
1858 if !(rhs_axis < rhs_rank) {
1859 return Err(anyhow::anyhow!("rhs contract axis {rhs_axis} out of range"));
1860 };
1861 if !(!seen_lhs[lhs_axis]) {
1862 return Err(anyhow::anyhow!("duplicate lhs contract axis {lhs_axis}"));
1863 };
1864 if !(!seen_rhs[rhs_axis]) {
1865 return Err(anyhow::anyhow!("duplicate rhs contract axis {rhs_axis}"));
1866 };
1867 seen_lhs[lhs_axis] = true;
1868 seen_rhs[rhs_axis] = true;
1869 lhs_ids[lhs_axis] = next_id;
1870 rhs_ids[rhs_axis] = next_id;
1871 next_id += 1;
1872 }
1873
1874 let mut output_ids = Vec::with_capacity(lhs_rank + rhs_rank - 2 * axes_a.len());
1875 for id in &mut lhs_ids {
1876 if *id == usize::MAX {
1877 *id = next_id;
1878 output_ids.push(next_id);
1879 next_id += 1;
1880 }
1881 }
1882 for id in &mut rhs_ids {
1883 if *id == usize::MAX {
1884 *id = next_id;
1885 output_ids.push(next_id);
1886 next_id += 1;
1887 }
1888 }
1889
1890 Ok((lhs_ids, rhs_ids, output_ids))
1891 }
1892
1893 fn build_payload_einsum_subscripts(
1894 input_roots: &[Vec<usize>],
1895 output_roots: &[usize],
1896 ) -> Result<EinsumSubscripts> {
1897 Self::einsum_subscripts_from_usize_ids(input_roots, output_roots)
1898 }
1899
1900 fn normalize_eager_payload_for_roots(
1901 payload: &EagerTensor,
1902 roots: &[usize],
1903 context: Option<&ExecutionContext>,
1904 ) -> Result<(Option<EagerTensor>, Vec<usize>)> {
1905 if !(payload.shape().len() == roots.len()) {
1906 return Err(anyhow::anyhow!(
1907 "payload rank {} does not match root label count {}",
1908 payload.shape().len(),
1909 roots.len()
1910 ));
1911 };
1912
1913 let mut current_payload = None;
1914 let mut current_roots = roots.to_vec();
1915 while let Some((axis_a, axis_b)) = Self::first_duplicate_pair(¤t_roots) {
1916 let source = current_payload.as_ref().unwrap_or(payload);
1917 current_payload = Some(with_eager_session(source.runtime(), context, |session| {
1918 session.extract_diag(source, axis_a, axis_b)
1919 })?);
1920 current_roots.remove(axis_b);
1921 }
1922
1923 Ok((current_payload, current_roots))
1924 }
1925
1926 fn first_duplicate_pair(values: &[usize]) -> Option<(usize, usize)> {
1927 let mut first_axis_by_value = std::collections::HashMap::new();
1928 for (axis, &value) in values.iter().enumerate() {
1929 if let Some(&first_axis) = first_axis_by_value.get(&value) {
1930 return Some((first_axis, axis));
1931 }
1932 first_axis_by_value.insert(value, axis);
1933 }
1934 None
1935 }
1936
1937 fn from_structured_payload_inner(
1938 indices: Vec<DynIndex>,
1939 payload_inner: EagerTensor,
1940 payload_dims: Vec<usize>,
1941 axis_classes: Vec<usize>,
1942 ) -> Result<Self> {
1943 Self::validate_indices(&indices)?;
1944 if payload_inner.shape() != payload_dims {
1945 return Err(anyhow::anyhow!(
1946 "structured payload dims {:?} do not match planned payload dims {:?}",
1947 payload_inner.shape(),
1948 payload_dims
1949 ));
1950 }
1951 if axis_classes == Self::dense_axis_classes(indices.len()) {
1952 return Self::from_inner_with_axis_classes(indices, payload_inner, axis_classes);
1953 }
1954 let structured_payload = Arc::new(StructuredPayload {
1955 payload: Arc::new(payload_inner),
1956 payload_dims,
1957 axis_classes,
1958 });
1959 Ok(Self {
1960 indices,
1961 storage: IdxTensorStorage::Compact(structured_payload),
1962 eager_cache: Self::empty_eager_cache(),
1963 })
1964 }
1965
1966 fn contract_structured_payloads(
1968 &self,
1969 other: &Self,
1970 result_indices: Vec<DynIndex>,
1971 axes_a: &[usize],
1972 axes_b: &[usize],
1973 context: Option<&ExecutionContext>,
1974 ) -> Result<Self> {
1975 let (lhs_labels, rhs_labels, output_labels) = Self::build_binary_contraction_labels(
1976 self.indices.len(),
1977 axes_a,
1978 other.indices.len(),
1979 axes_b,
1980 )?;
1981 Self::contract_structured_payloads_nary_in(
1982 &[self, other],
1983 result_indices,
1984 vec![lhs_labels, rhs_labels],
1985 output_labels,
1986 context,
1987 )
1988 }
1989
1990 pub(crate) fn contract_structured_payloads_nary(
1991 operands: &[&Self],
1992 result_indices: Vec<DynIndex>,
1993 input_labels: Vec<Vec<usize>>,
1994 output_labels: Vec<usize>,
1995 ) -> Result<Self> {
1996 Self::contract_structured_payloads_nary_in(
1997 operands,
1998 result_indices,
1999 input_labels,
2000 output_labels,
2001 None,
2002 )
2003 }
2004
2005 pub(crate) fn contract_structured_payloads_nary_in(
2014 operands: &[&Self],
2015 result_indices: Vec<DynIndex>,
2016 input_labels: Vec<Vec<usize>>,
2017 output_labels: Vec<usize>,
2018 context: Option<&ExecutionContext>,
2019 ) -> Result<Self> {
2020 if !(!operands.is_empty()) {
2021 return Err(anyhow::anyhow!("structured contraction needs operands"));
2022 };
2023 for operand in operands {
2024 operand.ensure_storage_ready()?;
2025 }
2026 let layouts = operands
2027 .iter()
2028 .map(|operand| {
2029 OperandLayout::new(operand.dims(), operand.storage.axis_classes().to_vec())
2030 })
2031 .collect::<Result<Vec<_>>>()?;
2032 let spec = StructuredContractionSpec {
2033 input_labels,
2034 output_labels,
2035 retained_labels: Default::default(),
2036 };
2037 let plan = StructuredContractionPlan::new(&layouts, &spec)?;
2038 let any_grad = operands.iter().any(|operand| operand.tracks_grad());
2039 let eager_payloads = any_grad
2043 || (context.is_some()
2044 && operands
2045 .iter()
2046 .all(|operand| operand.storage.eager().is_some()));
2047
2048 if eager_payloads {
2049 let dtypes = operands
2050 .iter()
2051 .map(|operand| {
2052 operand.storage.dtype().ok_or_else(|| {
2053 anyhow::anyhow!("structured contraction operand has no scalar dtype")
2054 })
2055 })
2056 .collect::<Result<Vec<_>>>()?;
2057 let target = Self::common_eager_dtype(&dtypes)?;
2058 let mut payloads = Vec::with_capacity(operands.len());
2059 for operand in operands {
2060 let payload = operand.compact_payload_inner()?;
2061 payloads.push(if payload.dtype() == target {
2062 payload
2063 } else {
2064 with_eager_session(payload.runtime(), context, |session| {
2065 session.cast(&payload, target)
2066 })?
2067 });
2068 }
2069
2070 let mut normalized = Vec::with_capacity(payloads.len());
2071 let mut labels = Vec::with_capacity(payloads.len());
2072 for (operand_idx, (payload, operand_plan)) in
2073 payloads.iter().zip(plan.operand_plans.iter()).enumerate()
2074 {
2075 let (payload, roots) = Self::normalize_eager_payload_for_roots(
2076 payload,
2077 &operand_plan.class_roots,
2078 context,
2079 )?;
2080 normalized.push(payload.unwrap_or_else(|| payloads[operand_idx].clone()));
2081 labels.push(roots);
2082 }
2083 let refs = normalized.iter().collect::<Vec<_>>();
2084 let subscripts =
2085 Self::build_payload_einsum_subscripts(&labels, &plan.output_payload_roots)?;
2086 let runtime = refs
2087 .first()
2088 .map(|payload| payload.runtime())
2089 .ok_or_else(|| {
2090 anyhow::anyhow!("structured einsum requires at least one operand")
2091 })?;
2092 let payload = with_eager_session(runtime, context, |session| {
2093 session.einsum_subscripts(&refs, &subscripts)
2094 })?;
2095 return Self::from_structured_payload_inner(
2096 result_indices,
2097 payload,
2098 plan.output_payload_dims,
2099 plan.output_axis_classes,
2100 );
2101 }
2102
2103 let storage_owners = operands
2107 .iter()
2108 .map(|operand| match &operand.storage {
2109 IdxTensorStorage::Materialized(storage) => Some(Arc::clone(storage)),
2110 IdxTensorStorage::Deferred { source, .. } => match source.as_ref() {
2111 IdxTensorStorage::Materialized(storage) => Some(Arc::clone(storage)),
2112 _ => None,
2113 },
2114 _ => None,
2115 })
2116 .collect::<Vec<_>>();
2117 let mut inputs = Vec::with_capacity(operands.len());
2118 for (operand_idx, operand) in operands.iter().enumerate() {
2119 if let Some(storage) = storage_owners[operand_idx].as_ref() {
2120 inputs.push(storage_payload_native_read_input(storage.as_ref())?);
2121 } else {
2122 let inner = operand
2123 .storage
2124 .eager()
2125 .ok_or_else(|| anyhow::anyhow!("structured operand has no compact payload"))?;
2126 inputs.push(NativeTensorReadInput::Borrowed(inner.tensor_read()));
2127 }
2128 }
2129 let mut normalized = Vec::with_capacity(inputs.len());
2130 let mut labels = Vec::with_capacity(inputs.len());
2131 for (input, operand_plan) in inputs.into_iter().zip(plan.operand_plans.iter()) {
2132 let (input, roots) =
2133 normalize_payload_read_for_roots(input, &operand_plan.class_roots)?;
2134 normalized.push(input);
2135 labels.push(roots);
2136 }
2137 let refs = normalized
2138 .iter()
2139 .zip(labels.iter())
2140 .map(|(input, labels)| (input, labels.as_slice()))
2141 .collect::<Vec<_>>();
2142 let payload = tensor4all_tensorbackend::einsum_native_tensor_reads(
2143 &refs,
2144 &plan.output_payload_roots,
2145 )?;
2146 let mut common: Option<Arc<EagerRuntime>> = None;
2152 let mut mixed = false;
2153 for operand in operands {
2154 if let Some(inner) = operand.storage.eager() {
2155 match &common {
2156 None => common = Some(Arc::clone(inner.runtime())),
2157 Some(runtime) => {
2158 if runtime.id() != inner.ctx_id() {
2159 mixed = true;
2160 }
2161 }
2162 }
2163 }
2164 }
2165 let payload_inner = match (common, mixed) {
2166 (Some(runtime), false) => IdxTensor::untracked_inner(payload, runtime)?,
2167 _ => IdxTensor::untracked_inner(payload, default_eager_ctx()?)?,
2168 };
2169 Self::from_structured_payload_inner(
2170 result_indices,
2171 payload_inner,
2172 plan.output_payload_dims,
2173 plan.output_axis_classes,
2174 )
2175 }
2176
2177 fn common_eager_dtype(dtypes: &[DType]) -> Result<DType> {
2178 let target = if dtypes.contains(&DType::C64)
2179 || (dtypes.contains(&DType::C32) && dtypes.contains(&DType::F64))
2180 {
2181 DType::C64
2182 } else if dtypes.contains(&DType::F64) || dtypes.contains(&DType::C32) {
2183 if dtypes.contains(&DType::C32) {
2184 DType::C32
2185 } else {
2186 DType::F64
2187 }
2188 } else {
2189 DType::F32
2190 };
2191 if !(dtypes
2192 .iter()
2193 .all(|dtype| matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64)))
2194 {
2195 return Err(anyhow::anyhow!(
2196 "eager operations support only f32, f64, c32, and c64 operands"
2197 ));
2198 };
2199 Ok(target)
2200 }
2201
2202 fn promote_eager_operands(
2212 lhs: &EagerTensor,
2213 rhs: &EagerTensor,
2214 ) -> Result<(EagerTensor, EagerTensor)> {
2215 let target = Self::common_eager_dtype(&[lhs.dtype(), rhs.dtype()])?;
2216 let cast = |eager: &EagerTensor| -> Result<EagerTensor> {
2217 if eager.dtype() == target {
2218 Ok(eager.clone())
2219 } else {
2220 Ok(with_eager_session(eager.runtime(), None, |session| {
2221 session.cast(eager, target)
2222 })?)
2223 }
2224 };
2225 Ok((cast(lhs)?, cast(rhs)?))
2226 }
2227
2228 fn should_use_structured_payload_contract(&self, other: &Self) -> bool {
2229 self.tracks_grad()
2230 || other.tracks_grad()
2231 || self.storage.axis_classes() != Self::dense_axis_classes(self.indices.len())
2232 || other.storage.axis_classes() != Self::dense_axis_classes(other.indices.len())
2233 }
2234
2235 fn storage_from_native_with_axis_classes(
2236 native: &NativeTensor,
2237 axis_classes: &[usize],
2238 logical_rank: usize,
2239 ) -> Result<Storage> {
2240 if Self::is_diag_axis_classes(axis_classes) {
2241 match native.dtype() {
2242 DType::F32 => Storage::from_diag_col_major(
2243 native_tensor_primal_to_diag::<f32>(native)?,
2244 logical_rank,
2245 ),
2246 DType::F64 | DType::I32 | DType::I64 | DType::Bool => Storage::from_diag_col_major(
2247 native_tensor_primal_to_diag::<f64>(native)?,
2248 logical_rank,
2249 ),
2250 DType::C32 => Storage::from_diag_col_major(
2251 native_tensor_primal_to_diag::<Complex32>(native)?,
2252 logical_rank,
2253 ),
2254 DType::C64 => Storage::from_diag_col_major(
2255 native_tensor_primal_to_diag::<Complex64>(native)?,
2256 logical_rank,
2257 ),
2258 DType::External(_) => Err(anyhow::anyhow!(
2259 "compact IdxTensor storage does not support external scalar kind {:?}",
2260 native.dtype()
2261 )),
2262 }
2263 } else {
2264 storage_from_payload_native(native.duplicate()?, native.shape(), axis_classes.to_vec())
2265 }
2266 }
2267
2268 fn dense_selected_diag_payload<T: TensorElement + Copy + Zero>(
2269 payload: Vec<T>,
2270 kept_dims: &[usize],
2271 selected_positions: &[usize],
2272 ) -> Result<Vec<T>> {
2273 let output_len = checked_product(kept_dims)?;
2274 let mut data = vec![T::zero(); output_len];
2275 if output_len == 0 {
2276 return Ok(data);
2277 }
2278
2279 let Some((&first_position, rest)) = selected_positions.split_first() else {
2280 return Ok(data);
2281 };
2282 if rest.iter().any(|&position| position != first_position) {
2283 return Ok(data);
2284 }
2285
2286 let value = payload[first_position];
2287 if kept_dims.is_empty() {
2288 data[0] = value;
2289 return Ok(data);
2290 }
2291
2292 let mut offset = 0usize;
2293 let mut stride = 1usize;
2294 for &dim in kept_dims {
2295 let term = first_position
2296 .checked_mul(stride)
2297 .ok_or_else(|| anyhow::anyhow!("diagonal selection offset overflow"))?;
2298 offset = offset
2299 .checked_add(term)
2300 .ok_or_else(|| anyhow::anyhow!("diagonal selection offset overflow"))?;
2301 stride = stride
2302 .checked_mul(dim)
2303 .ok_or_else(|| anyhow::anyhow!("diagonal selection stride overflow"))?;
2304 }
2305 data[offset] = value;
2306 Ok(data)
2307 }
2308
2309 fn select_diag_indices(
2310 &self,
2311 kept_indices: Vec<DynIndex>,
2312 kept_dims: Vec<usize>,
2313 positions: &[usize],
2314 ) -> Result<Self> {
2315 if self.storage.is_f64() {
2316 let storage = self.storage.materialize(self.indices.len())?;
2317 let payload = storage
2318 .payload_f64_col_major_vec()
2319 .map_err(anyhow::Error::new)?;
2320 let data = Self::dense_selected_diag_payload(payload, &kept_dims, positions)?;
2321 Self::from_host_dense(kept_indices, data).map_err(anyhow::Error::from)
2322 } else if self.storage.is_c64() {
2323 let storage = self.storage.materialize(self.indices.len())?;
2324 let payload = storage
2325 .payload_c64_col_major_vec()
2326 .map_err(anyhow::Error::new)?;
2327 let data = Self::dense_selected_diag_payload(payload, &kept_dims, positions)?;
2328 Self::from_host_dense(kept_indices, data).map_err(anyhow::Error::from)
2329 } else if self.storage.dtype() == Some(DType::F32) {
2330 let storage = self.storage.materialize(self.indices.len())?;
2331 let payload = storage
2332 .payload_f32_col_major_vec()
2333 .map_err(anyhow::Error::new)?;
2334 let data = Self::dense_selected_diag_payload(payload, &kept_dims, positions)?;
2335 Self::from_host_dense(kept_indices, data).map_err(anyhow::Error::from)
2336 } else if self.storage.dtype() == Some(DType::C32) {
2337 let storage = self.storage.materialize(self.indices.len())?;
2338 let payload = storage
2339 .payload_c32_col_major_vec()
2340 .map_err(anyhow::Error::new)?;
2341 let data = Self::dense_selected_diag_payload(payload, &kept_dims, positions)?;
2342 Self::from_host_dense(kept_indices, data).map_err(anyhow::Error::from)
2343 } else {
2344 Err(anyhow::anyhow!("unsupported diagonal storage scalar type"))
2345 }
2346 }
2347
2348 fn select_dense_host<T: tensor4all_tensorbackend::StorageScalar>(
2354 &self,
2355 kept_axes: &[usize],
2356 kept_indices: Vec<DynIndex>,
2357 kept_dims: &[usize],
2358 selected_axes: &[usize],
2359 positions: &[usize],
2360 ) -> Result<Self> {
2361 let dims = self.dims();
2362 let strides = Self::col_major_strides(&dims)?;
2363 let kept_len = checked_product(kept_dims)?;
2364 self.with_dense_host_slice::<T, _>(|data| {
2367 let mut output = Vec::with_capacity(kept_len);
2368 for linear in 0..kept_len {
2369 let kept_coordinates = Self::col_major_coordinates(linear, kept_dims);
2370 let mut source_coordinates = vec![0usize; dims.len()];
2371 for (position, &axis) in kept_axes.iter().enumerate() {
2372 source_coordinates[axis] = kept_coordinates[position];
2373 }
2374 for (&axis, &position) in selected_axes.iter().zip(positions.iter()) {
2375 source_coordinates[axis] = position;
2376 }
2377 let offset = Self::col_major_offset(&source_coordinates, &strides)?;
2378 output.push(*data.get(offset).ok_or_else(|| {
2379 anyhow::anyhow!("column-major selection read offset {offset} is out of bounds")
2380 })?);
2381 }
2382 Self::from_host_dense(kept_indices, output).map_err(anyhow::Error::from)
2383 })?
2384 }
2385
2386 fn select_dense_host_dispatch(
2387 &self,
2388 kept_axes: &[usize],
2389 kept_indices: Vec<DynIndex>,
2390 kept_dims: &[usize],
2391 selected_axes: &[usize],
2392 positions: &[usize],
2393 ) -> Result<Self> {
2394 match self.storage.dtype() {
2395 Some(DType::F64) => self.select_dense_host::<f64>(
2396 kept_axes,
2397 kept_indices,
2398 kept_dims,
2399 selected_axes,
2400 positions,
2401 ),
2402 Some(DType::F32) => self.select_dense_host::<f32>(
2403 kept_axes,
2404 kept_indices,
2405 kept_dims,
2406 selected_axes,
2407 positions,
2408 ),
2409 Some(DType::C64) => self.select_dense_host::<Complex64>(
2410 kept_axes,
2411 kept_indices,
2412 kept_dims,
2413 selected_axes,
2414 positions,
2415 ),
2416 Some(DType::C32) => self.select_dense_host::<Complex32>(
2417 kept_axes,
2418 kept_indices,
2419 kept_dims,
2420 selected_axes,
2421 positions,
2422 ),
2423 _ => Err(anyhow::anyhow!("unsupported dense storage scalar type")),
2424 }
2425 }
2426
2427 fn stack_host<T: tensor4all_tensorbackend::StorageScalar>(
2436 tensors: &[&Self],
2437 base_dims: &[usize],
2438 insert_axis: usize,
2439 result_indices: Vec<DynIndex>,
2440 ) -> Result<Self> {
2441 let base_rank = base_dims.len();
2442 let base_len = checked_product(base_dims)?;
2443 let mut result_dims = base_dims.to_vec();
2444 result_dims.insert(insert_axis, tensors.len());
2445 let result_len = checked_product(&result_dims)?;
2446 let strides = Self::col_major_strides(&result_dims)?;
2447 let mut output = vec![T::default(); result_len];
2448 for (batch, tensor) in tensors.iter().enumerate() {
2449 tensor.with_dense_host_slice::<T, _>(|data| -> anyhow::Result<()> {
2450 if data.len() != base_len {
2451 return Err(anyhow::anyhow!(
2452 "stack input payload length {} does not match {} base elements",
2453 data.len(),
2454 base_len
2455 ));
2456 }
2457 for (linear, &value) in data.iter().enumerate().take(base_len) {
2458 let base_coordinates = Self::col_major_coordinates(linear, base_dims);
2459 let mut result_coordinates = Vec::with_capacity(result_dims.len());
2460 for axis in 0..=base_rank {
2461 if axis == insert_axis {
2462 result_coordinates.push(batch);
2463 } else {
2464 let base_axis = if axis < insert_axis { axis } else { axis - 1 };
2465 result_coordinates.push(base_coordinates[base_axis]);
2466 }
2467 }
2468 let offset = Self::col_major_offset(&result_coordinates, &strides)?;
2469 let slot = output.get_mut(offset).ok_or_else(|| {
2470 anyhow::anyhow!("stack write offset {offset} is out of bounds")
2471 })?;
2472 *slot = value;
2473 }
2474 Ok(())
2475 })??;
2476 }
2477 Self::from_host_dense(result_indices, output).map_err(anyhow::Error::from)
2478 }
2479
2480 fn stack_host_dispatch(
2481 tensors: &[&Self],
2482 base_dims: &[usize],
2483 insert_axis: usize,
2484 result_indices: Vec<DynIndex>,
2485 dtype: DType,
2486 ) -> Result<Self> {
2487 match dtype {
2488 DType::F64 => Self::stack_host::<f64>(tensors, base_dims, insert_axis, result_indices),
2489 DType::F32 => Self::stack_host::<f32>(tensors, base_dims, insert_axis, result_indices),
2490 DType::C64 => {
2491 Self::stack_host::<Complex64>(tensors, base_dims, insert_axis, result_indices)
2492 }
2493 DType::C32 => {
2494 Self::stack_host::<Complex32>(tensors, base_dims, insert_axis, result_indices)
2495 }
2496 _ => Err(anyhow::anyhow!("unsupported dense storage scalar type")),
2497 }
2498 }
2499
2500 fn col_major_strides(dims: &[usize]) -> Result<Vec<isize>> {
2501 let mut strides = Vec::with_capacity(dims.len());
2502 let mut stride = 1isize;
2503 for &dim in dims {
2504 strides.push(stride);
2505 let dim = isize::try_from(dim)
2506 .map_err(|_| anyhow::anyhow!("dimension does not fit in isize"))?;
2507 stride = stride
2508 .checked_mul(dim)
2509 .ok_or_else(|| anyhow::anyhow!("column-major stride overflow"))?;
2510 }
2511 Ok(strides)
2512 }
2513
2514 fn col_major_coordinates(linear: usize, dims: &[usize]) -> Vec<usize> {
2516 let mut remaining = linear;
2517 dims.iter()
2518 .map(|&dim| {
2519 if dim == 0 {
2520 0
2521 } else {
2522 let coordinate = remaining % dim;
2523 remaining /= dim;
2524 coordinate
2525 }
2526 })
2527 .collect()
2528 }
2529
2530 fn col_major_offset(coordinates: &[usize], strides: &[isize]) -> Result<usize> {
2532 coordinates
2533 .iter()
2534 .zip(strides.iter())
2535 .try_fold(0usize, |offset, (&coordinate, &stride)| {
2536 let stride = usize::try_from(stride)
2537 .map_err(|_| anyhow::anyhow!("negative column-major stride {stride}"))?;
2538 let term = coordinate
2539 .checked_mul(stride)
2540 .ok_or_else(|| anyhow::anyhow!("column-major offset overflow"))?;
2541 offset
2542 .checked_add(term)
2543 .ok_or_else(|| anyhow::anyhow!("column-major offset overflow"))
2544 })
2545 }
2546
2547 fn permute_host_col_major<T: Copy>(
2549 data: &[T],
2550 dims: &[usize],
2551 perm: &[usize],
2552 ) -> Result<Vec<T>> {
2553 let new_dims: Vec<usize> = perm.iter().map(|&axis| dims[axis]).collect();
2554 let len = checked_product(&new_dims)?;
2555 if len == 0 {
2556 return Ok(Vec::new());
2557 }
2558 let strides = Self::col_major_strides(dims)?;
2559 let mut output = Vec::with_capacity(len);
2560 for linear in 0..len {
2561 let coordinates = Self::col_major_coordinates(linear, &new_dims);
2562 let mut source_coordinates = vec![0usize; dims.len()];
2563 for (new_axis, &old_axis) in perm.iter().enumerate() {
2564 source_coordinates[old_axis] = coordinates[new_axis];
2565 }
2566 let offset = Self::col_major_offset(&source_coordinates, &strides)?;
2567 output.push(*data.get(offset).ok_or_else(|| {
2568 anyhow::anyhow!("column-major permutation read offset {offset} is out of bounds")
2569 })?);
2570 }
2571 Ok(output)
2572 }
2573
2574 pub fn dense_host_values<T: tensor4all_tensorbackend::StorageScalar>(
2603 &self,
2604 ) -> std::result::Result<Vec<T>, IdxTensorError> {
2605 if let IdxTensorStorage::Materialized(storage) = &self.storage {
2606 return T::dense_values(storage, &self.dims())
2607 .map_err(|error| IdxTensorError::materialization(anyhow::Error::new(error)));
2608 }
2609 self.with_dense_slice::<T, _>(|values| values.to_vec())
2610 }
2611
2612 pub(crate) fn host_dense_permuted<T: tensor4all_tensorbackend::StorageScalar>(
2619 &self,
2620 new_indices: &[DynIndex],
2621 ) -> std::result::Result<(Vec<usize>, Vec<T>), IdxTensorError> {
2622 let perm = compute_permutation_from_indices(&self.indices, new_indices)?;
2623 let dims = self.dims();
2624 let data = self.dense_host_values::<T>()?;
2625 if perm.iter().copied().eq(0..perm.len()) {
2626 return Ok((dims, data));
2627 }
2628 let permuted = Self::permute_host_col_major(&data, &dims, &perm)
2629 .map_err(|error| IdxTensorError::operation("host dense permutation", error))?;
2630 let new_dims = new_indices.iter().map(|index| index.dim()).collect();
2631 Ok((new_dims, permuted))
2632 }
2633
2634 fn zero_structured_selection<T>(
2635 kept_indices: Vec<DynIndex>,
2636 kept_dims: &[usize],
2637 ) -> Result<Self>
2638 where
2639 T: TensorElement + Zero,
2640 {
2641 let output_len = checked_product(kept_dims)?;
2642 Self::from_dense(kept_indices, vec![T::zero(); output_len]).map_err(anyhow::Error::from)
2643 }
2644
2645 fn selected_structured_class_positions(
2646 axis_classes: &[usize],
2647 payload_rank: usize,
2648 selected_axes: &[usize],
2649 positions: &[usize],
2650 ) -> Option<Vec<Option<usize>>> {
2651 let mut selected_class_positions = vec![None; payload_rank];
2652 for (&axis, &position) in selected_axes.iter().zip(positions.iter()) {
2653 let class_id = axis_classes[axis];
2654 if let Some(existing) = selected_class_positions[class_id] {
2655 if existing != position {
2656 return None;
2657 }
2658 } else {
2659 selected_class_positions[class_id] = Some(position);
2660 }
2661 }
2662 Some(selected_class_positions)
2663 }
2664
2665 fn select_structured_indices_typed<T, F>(
2666 &self,
2667 payload: Vec<T>,
2668 kept_axes: &[usize],
2669 kept_indices: Vec<DynIndex>,
2670 kept_dims: Vec<usize>,
2671 selected: (&[usize], &[usize]),
2672 make_output: F,
2673 ) -> Result<Self>
2674 where
2675 T: TensorElement + Zero + tensor4all_tensorbackend::StorageScalar,
2676 F: FnOnce(Vec<T>, Vec<usize>, Vec<isize>, Vec<usize>) -> Result<Self>,
2677 {
2678 let (selected_axes, positions) = selected;
2679 let payload_dims = self.storage.payload_dims();
2680 let axis_classes = self.storage.axis_classes();
2681 let payload_rank = payload_dims.len();
2682 let Some(selected_class_positions) = Self::selected_structured_class_positions(
2683 axis_classes,
2684 payload_rank,
2685 selected_axes,
2686 positions,
2687 ) else {
2688 return Self::zero_structured_selection::<T>(kept_indices, &kept_dims);
2689 };
2690
2691 let selected_class_kept = kept_axes
2692 .iter()
2693 .any(|&axis| selected_class_positions[axis_classes[axis]].is_some());
2694 if selected_class_kept {
2695 return self.select_structured_indices_dense(
2696 payload,
2697 kept_axes,
2698 kept_indices,
2699 kept_dims,
2700 &selected_class_positions,
2701 );
2702 }
2703
2704 let mut old_to_new_class = vec![None; payload_rank];
2705 let mut output_payload_dims = Vec::new();
2706 let mut output_axis_classes = Vec::with_capacity(kept_axes.len());
2707 for &axis in kept_axes {
2708 let class_id = axis_classes[axis];
2709 let new_class = match old_to_new_class[class_id] {
2710 Some(new_class) => new_class,
2711 None => {
2712 let new_class = output_payload_dims.len();
2713 old_to_new_class[class_id] = Some(new_class);
2714 output_payload_dims.push(payload_dims[class_id]);
2715 new_class
2716 }
2717 };
2718 output_axis_classes.push(new_class);
2719 }
2720
2721 let output_len = checked_product(&output_payload_dims)?;
2722 let mut output_payload = Vec::with_capacity(output_len);
2723 let mut output_payload_index = Vec::with_capacity(output_payload_dims.len());
2724 let mut input_payload_index = vec![0usize; payload_rank];
2725 for linear in 0..output_len {
2726 decode_col_major_linear_into(linear, &output_payload_dims, &mut output_payload_index);
2727 for class_id in 0..payload_rank {
2728 input_payload_index[class_id] =
2729 if let Some(position) = selected_class_positions[class_id] {
2730 position
2731 } else if let Some(new_class) = old_to_new_class[class_id] {
2732 output_payload_index[new_class]
2733 } else {
2734 return Err(anyhow::anyhow!(
2735 "structured payload class {class_id} is neither selected nor kept"
2736 ));
2737 };
2738 }
2739 let input_linear = encode_col_major_linear(&input_payload_index, payload_dims)?;
2740 output_payload.push(payload[input_linear]);
2741 }
2742
2743 let output_strides = Self::col_major_strides(&output_payload_dims)?;
2744 make_output(
2745 output_payload,
2746 output_payload_dims,
2747 output_strides,
2748 output_axis_classes,
2749 )
2750 }
2751
2752 fn select_structured_indices_dense<T>(
2753 &self,
2754 payload: Vec<T>,
2755 kept_axes: &[usize],
2756 kept_indices: Vec<DynIndex>,
2757 kept_dims: Vec<usize>,
2758 selected_class_positions: &[Option<usize>],
2759 ) -> Result<Self>
2760 where
2761 T: TensorElement + Zero + tensor4all_tensorbackend::StorageScalar,
2762 {
2763 let payload_dims = self.storage.payload_dims();
2764 let axis_classes = self.storage.axis_classes();
2765 let output_len = checked_product(&kept_dims)?;
2766 let mut output = Vec::with_capacity(output_len);
2767 let mut kept_position = Vec::with_capacity(kept_dims.len());
2768 let mut input_payload_index = vec![None; payload_dims.len()];
2769 let mut dense_positions = Vec::with_capacity(payload_dims.len());
2770
2771 for linear in 0..output_len {
2772 decode_col_major_linear_into(linear, &kept_dims, &mut kept_position);
2773 input_payload_index.copy_from_slice(selected_class_positions);
2774 let mut is_structural_zero = false;
2775
2776 for (&axis, &position) in kept_axes.iter().zip(kept_position.iter()) {
2777 let class_id = axis_classes[axis];
2778 match input_payload_index[class_id] {
2779 Some(existing) if existing != position => {
2780 is_structural_zero = true;
2781 break;
2782 }
2783 Some(_) => {}
2784 None => input_payload_index[class_id] = Some(position),
2785 }
2786 }
2787
2788 if is_structural_zero {
2789 output.push(T::zero());
2790 continue;
2791 }
2792
2793 dense_positions.clear();
2794 for (class_id, position) in input_payload_index.iter().enumerate() {
2795 match position {
2796 Some(position) => dense_positions.push(*position),
2797 None => {
2798 return Err(anyhow::anyhow!(
2799 "structured payload class {class_id} is neither selected nor kept"
2800 ))
2801 }
2802 }
2803 }
2804 let input_linear = encode_col_major_linear(&dense_positions, payload_dims)?;
2805 output.push(payload[input_linear]);
2806 }
2807
2808 Self::from_host_dense(kept_indices, output).map_err(anyhow::Error::from)
2809 }
2810
2811 fn select_structured_indices(
2812 &self,
2813 kept_axes: &[usize],
2814 kept_indices: Vec<DynIndex>,
2815 kept_dims: Vec<usize>,
2816 selected_axes: &[usize],
2817 positions: &[usize],
2818 ) -> Result<Self> {
2819 if self.storage.is_f64() {
2820 let storage = self.storage.materialize(self.indices.len())?;
2821 let payload = storage
2822 .payload_f64_col_major_vec()
2823 .map_err(anyhow::Error::new)?;
2824 let output_indices = kept_indices.clone();
2825 self.select_structured_indices_typed(
2826 payload,
2827 kept_axes,
2828 kept_indices,
2829 kept_dims,
2830 (selected_axes, positions),
2831 move |payload, dims, strides, classes| {
2832 let storage = Storage::new_structured(payload, dims, strides, classes)?;
2833 Self::from_storage(output_indices, Arc::new(storage))
2834 .map_err(anyhow::Error::from)
2835 },
2836 )
2837 } else if self.storage.is_c64() {
2838 let storage = self.storage.materialize(self.indices.len())?;
2839 let payload = storage
2840 .payload_c64_col_major_vec()
2841 .map_err(anyhow::Error::new)?;
2842 let output_indices = kept_indices.clone();
2843 self.select_structured_indices_typed(
2844 payload,
2845 kept_axes,
2846 kept_indices,
2847 kept_dims,
2848 (selected_axes, positions),
2849 move |payload, dims, strides, classes| {
2850 let storage = Storage::new_structured(payload, dims, strides, classes)?;
2851 Self::from_storage(output_indices, Arc::new(storage))
2852 .map_err(anyhow::Error::from)
2853 },
2854 )
2855 } else if self.storage.dtype() == Some(DType::F32) {
2856 let storage = self.storage.materialize(self.indices.len())?;
2857 let payload = storage
2858 .payload_f32_col_major_vec()
2859 .map_err(anyhow::Error::new)?;
2860 let output_indices = kept_indices.clone();
2861 self.select_structured_indices_typed(
2862 payload,
2863 kept_axes,
2864 kept_indices,
2865 kept_dims,
2866 (selected_axes, positions),
2867 move |payload, dims, strides, classes| {
2868 let storage = Storage::new_structured(payload, dims, strides, classes)?;
2869 Self::from_storage(output_indices, Arc::new(storage))
2870 .map_err(anyhow::Error::from)
2871 },
2872 )
2873 } else if self.storage.dtype() == Some(DType::C32) {
2874 let storage = self.storage.materialize(self.indices.len())?;
2875 let payload = storage
2876 .payload_c32_col_major_vec()
2877 .map_err(anyhow::Error::new)?;
2878 let output_indices = kept_indices.clone();
2879 self.select_structured_indices_typed(
2880 payload,
2881 kept_axes,
2882 kept_indices,
2883 kept_dims,
2884 (selected_axes, positions),
2885 move |payload, dims, strides, classes| {
2886 let storage = Storage::new_structured(payload, dims, strides, classes)?;
2887 Self::from_storage(output_indices, Arc::new(storage))
2888 .map_err(anyhow::Error::from)
2889 },
2890 )
2891 } else {
2892 Err(anyhow::anyhow!(
2893 "unsupported structured storage scalar type"
2894 ))
2895 }
2896 }
2897
2898 fn validate_storage_matches_indices(indices: &[DynIndex], storage: &Storage) -> Result<()> {
2899 let dims = Self::expected_dims_from_indices(indices);
2900 let storage_dims = storage.logical_dims();
2901 if storage_dims != dims {
2902 return Err(anyhow::anyhow!(
2903 "storage logical dims {:?} do not match indices dims {:?}",
2904 storage_dims,
2905 dims
2906 ));
2907 }
2908 if storage.is_diag() {
2909 Self::validate_diag_dims(&dims)?;
2910 }
2911 Ok(())
2912 }
2913
2914 fn try_materialized_inner(&self) -> Result<&EagerTensor> {
2915 self.ensure_storage_ready()?;
2916 let logical_dims = self.dims();
2917 if let Some(value) = self.tracked_compact_payload_value() {
2918 if self.compact_payload_is_logical_dense(&value.payload_dims) {
2919 return Ok(value.payload.as_ref());
2920 }
2921 if self.eager_cache.get().is_none() {
2922 let dense = Self::dense_inner_from_payload(
2923 value.payload.as_ref(),
2924 &value.axis_classes,
2925 &logical_dims,
2926 )?;
2927 let _ = self.eager_cache.set(Arc::new(dense));
2928 }
2929 return self
2930 .eager_cache
2931 .get()
2932 .map(|inner| inner.as_ref())
2933 .ok_or_else(|| {
2934 anyhow::anyhow!("IdxTensor structured AD cache was not initialized")
2935 });
2936 }
2937 if let Some(inner) = self.storage.eager() {
2938 if self.storage.axis_classes() == Self::dense_axis_classes(self.indices.len()) {
2939 return Ok(inner);
2940 }
2941 if self.eager_cache.get().is_none() {
2942 let dense = Self::dense_inner_from_payload(
2943 inner,
2944 self.storage.axis_classes(),
2945 &logical_dims,
2946 )?;
2947 let _ = self.eager_cache.set(Arc::new(dense));
2948 }
2949 return self
2950 .eager_cache
2951 .get()
2952 .map(|inner| inner.as_ref())
2953 .ok_or_else(|| {
2954 anyhow::anyhow!("IdxTensor structured eager cache was not initialized")
2955 });
2956 }
2957 if self.eager_cache.get().is_none() {
2958 let native = profile_pairwise_contract_section("materialize_storage_to_native", || {
2959 let storage = self.storage.materialize(self.indices.len())?;
2960 Self::seed_native_payload(storage.as_ref(), &logical_dims)
2961 })
2962 .context("IdxTensor materialization failed")?;
2963 record_pairwise_contract_profile_bytes(
2964 "materialize_storage_to_native",
2965 tensor_profile_bytes(native.dtype(), native.shape())?,
2966 );
2967 let _ = self.eager_cache.set(Arc::new(IdxTensor::untracked_inner(
2968 native,
2969 default_eager_ctx()?,
2970 )?));
2971 }
2972 self.eager_cache
2973 .get()
2974 .map(|inner| inner.as_ref())
2975 .ok_or_else(|| anyhow::anyhow!("IdxTensor materialization cache was not initialized"))
2976 }
2977
2978 pub(crate) fn as_inner(&self) -> Result<&EagerTensor> {
2979 self.try_materialized_inner()
2980 }
2981
2982 #[inline]
2984 fn expected_dims_from_indices(indices: &[DynIndex]) -> Vec<usize> {
2985 indices.iter().map(|idx| idx.dim()).collect()
2986 }
2987
2988 pub fn dims(&self) -> Vec<usize> {
3007 Self::expected_dims_from_indices(&self.indices)
3008 }
3009
3010 pub fn select_indices(
3057 &self,
3058 selected_indices: &[DynIndex],
3059 positions: &[usize],
3060 ) -> std::result::Result<Self, IdxTensorError> {
3061 if selected_indices.len() != positions.len() {
3062 return Err(anyhow::anyhow!(
3063 "selected_indices length {} does not match positions length {}",
3064 selected_indices.len(),
3065 positions.len()
3066 )
3067 .into());
3068 }
3069 if selected_indices.is_empty() {
3070 return Ok(self.clone());
3071 }
3072
3073 let mut selected_axes = Vec::with_capacity(selected_indices.len());
3074 let mut seen_axes = HashSet::with_capacity(selected_indices.len());
3075 for (selected, &position) in selected_indices.iter().zip(positions.iter()) {
3076 let axis = self
3077 .indices
3078 .iter()
3079 .position(|index| index == selected)
3080 .ok_or_else(|| anyhow::anyhow!("selected index is not present in tensor"))?;
3081 if !seen_axes.insert(axis) {
3082 return Err(anyhow::anyhow!("selected index appears more than once").into());
3083 }
3084 let dim = self.indices[axis].dim();
3085 if position >= dim {
3086 return Err(anyhow::anyhow!(
3087 "selected coordinate {position} is out of range for axis {axis} with dim {dim}"
3088 )
3089 .into());
3090 }
3091 selected_axes.push(axis);
3092 }
3093
3094 let kept_axes = self
3095 .indices
3096 .iter()
3097 .enumerate()
3098 .filter(|(axis, _)| !seen_axes.contains(axis))
3099 .map(|(axis, _)| axis)
3100 .collect::<Vec<_>>();
3101 let kept_indices = kept_axes
3102 .iter()
3103 .map(|&axis| self.indices[axis].clone())
3104 .collect::<Vec<_>>();
3105 let kept_dims = kept_axes
3106 .iter()
3107 .map(|&axis| self.indices[axis].dim())
3108 .collect::<Vec<_>>();
3109
3110 if matches!(&self.storage, IdxTensorStorage::Materialized(_))
3113 && self.storage.storage_kind() == StorageKind::Dense
3114 {
3115 return self
3116 .select_dense_host_dispatch(
3117 &kept_axes,
3118 kept_indices,
3119 &kept_dims,
3120 &selected_axes,
3121 positions,
3122 )
3123 .map_err(IdxTensorError::from);
3124 }
3125
3126 if matches!(
3127 self.storage.storage_kind(),
3128 StorageKind::Diagonal | StorageKind::Structured
3129 ) {
3130 self.ensure_shape_packing_preserves_ad("select_indices")?;
3131 }
3132 if self.storage.storage_kind() == StorageKind::Diagonal {
3133 return self
3134 .select_diag_indices(kept_indices, kept_dims, positions)
3135 .map_err(IdxTensorError::from);
3136 }
3137 if self.storage.storage_kind() == StorageKind::Structured {
3138 return self
3139 .select_structured_indices(
3140 &kept_axes,
3141 kept_indices,
3142 kept_dims,
3143 &selected_axes,
3144 positions,
3145 )
3146 .map_err(IdxTensorError::from);
3147 }
3148 if self.storage.storage_kind() != StorageKind::Dense {
3149 return Err(anyhow::anyhow!(
3150 "select_indices got unsupported storage kind {:?}",
3151 self.storage.storage_kind()
3152 )
3153 .into());
3154 }
3155
3156 let mut sliced = self.try_materialized_inner()?.clone();
3161 for (&axis, &position) in selected_axes.iter().zip(positions.iter()) {
3162 sliced = with_eager_session(sliced.runtime(), None, |session| {
3163 session.slice_axis(&sliced, axis, position..position + 1)
3164 })
3165 .map_err(|error| anyhow::anyhow!("select_indices slicing failed: {error}"))?;
3166 }
3167 let reshaped = with_eager_session(sliced.runtime(), None, |session| {
3168 session.reshape(&sliced, &kept_dims)
3169 })?;
3170 Self::from_inner(kept_indices, reshaped).map_err(IdxTensorError::from)
3171 }
3172
3173 pub fn stack_along_new_index(
3210 tensors: &[&Self],
3211 new_index: DynIndex,
3212 axis: isize,
3213 ) -> std::result::Result<Self, IdxTensorError> {
3214 let first = tensors
3215 .first()
3216 .copied()
3217 .ok_or_else(|| anyhow::anyhow!("stack_along_new_index requires at least one tensor"))?;
3218 if !(new_index.dim() == tensors.len()) {
3219 return Err(anyhow::anyhow!(
3220 "stack_along_new_index: new index dim {} does not match tensor count {}",
3221 new_index.dim(),
3222 tensors.len()
3223 )
3224 .into());
3225 };
3226
3227 let base_indices = first.indices.clone();
3228 for tensor in tensors.iter().copied().skip(1) {
3229 if !(tensor.indices == base_indices) {
3230 return Err(anyhow::anyhow!(
3231 "stack_along_new_index: input tensors must have identical index order"
3232 )
3233 .into());
3234 };
3235 }
3236 for &tensor in tensors {
3237 tensor.ensure_shape_packing_preserves_ad("stack_along_new_index")?;
3238 }
3239
3240 let insert_axis =
3241 Self::normalize_insert_axis("stack_along_new_index", axis, base_indices.len())?;
3242 let base_dims = first.dims();
3243 let all_compact = tensors
3244 .iter()
3245 .all(|tensor| matches!(&tensor.storage, IdxTensorStorage::Materialized(_)));
3246 if all_compact {
3251 let dtype = first.storage.dtype().ok_or_else(|| {
3252 anyhow::anyhow!("stack_along_new_index: unable to determine the payload dtype")
3253 })?;
3254 for tensor in tensors.iter().copied() {
3255 if tensor.storage.dtype() != Some(dtype) {
3256 return Err(anyhow::anyhow!(
3257 "stack_along_new_index: input tensors must have the same scalar type"
3258 )
3259 .into());
3260 }
3261 if tensor.dims() != base_dims {
3262 return Err(anyhow::anyhow!(
3263 "stack_along_new_index: input tensor dims {:?} do not match {:?}",
3264 tensor.dims(),
3265 base_dims
3266 )
3267 .into());
3268 }
3269 }
3270 let mut result_indices = base_indices;
3271 result_indices.insert(insert_axis, new_index);
3272 return Self::stack_host_dispatch(
3273 tensors,
3274 &base_dims,
3275 insert_axis,
3276 result_indices,
3277 dtype,
3278 )
3279 .map_err(IdxTensorError::from);
3280 }
3281
3282 let mut result_indices = base_indices;
3283 result_indices.insert(insert_axis, new_index);
3284
3285 let inner_refs = tensors
3286 .iter()
3287 .map(|tensor| tensor.try_materialized_inner())
3288 .collect::<Result<Vec<_>>>()?;
3289 let runtime = inner_refs
3290 .first()
3291 .map(|tensor| tensor.runtime())
3292 .ok_or_else(|| anyhow::anyhow!("stack_along_new_index requires at least one tensor"))?;
3293 let stacked =
3294 with_eager_session(runtime, None, |session| session.stack(&inner_refs, axis))?;
3295 Self::from_inner(result_indices, stacked).map_err(IdxTensorError::from)
3296 }
3297
3298 pub fn index_select(
3328 &self,
3329 source_index: &DynIndex,
3330 target_index: DynIndex,
3331 positions: &[usize],
3332 ) -> std::result::Result<Self, IdxTensorError> {
3333 if !(target_index.dim() == positions.len()) {
3334 return Err(anyhow::anyhow!(
3335 "index_select: target index dim {} does not match position count {}",
3336 target_index.dim(),
3337 positions.len()
3338 )
3339 .into());
3340 };
3341 let axis = self
3342 .indices
3343 .iter()
3344 .position(|index| index == source_index)
3345 .ok_or_else(|| anyhow::anyhow!("index_select: source index is not present"))?;
3346 let source_dim = self.indices[axis].dim();
3347 for &position in positions {
3348 if !(position < source_dim) {
3349 return Err(anyhow::anyhow!(
3350 "index_select: position {position} is out of range for source dim {source_dim}"
3351 )
3352 .into());
3353 };
3354 }
3355 self.ensure_shape_packing_preserves_ad("index_select")?;
3356
3357 let axis = isize::try_from(axis)
3358 .map_err(|_| anyhow::anyhow!("index_select: axis does not fit in isize"))?;
3359 let inner = self.try_materialized_inner()?;
3360 let selected = with_eager_session(inner.runtime(), None, |session| {
3361 session.index_select(inner, axis, positions)
3362 })?;
3363 let mut result_indices = self.indices.clone();
3364 result_indices[axis as usize] = target_index;
3365 Self::from_inner(result_indices, selected).map_err(IdxTensorError::from)
3366 }
3367
3368 pub fn new(
3386 indices: Vec<DynIndex>,
3387 storage: Arc<Storage>,
3388 ) -> std::result::Result<Self, IdxTensorError> {
3389 Self::from_storage(indices, storage)
3390 }
3391
3392 pub fn from_indices(
3412 indices: Vec<DynIndex>,
3413 storage: Arc<Storage>,
3414 ) -> std::result::Result<Self, IdxTensorError> {
3415 Self::new(indices, storage)
3416 }
3417
3418 pub fn from_storage(
3438 indices: Vec<DynIndex>,
3439 storage: Arc<Storage>,
3440 ) -> std::result::Result<Self, IdxTensorError> {
3441 Self::validate_indices(&indices)?;
3442 Self::validate_storage_matches_indices(&indices, storage.as_ref())?;
3443 Ok(Self {
3444 indices,
3445 storage: IdxTensorStorage::from_storage(storage),
3446 eager_cache: Self::empty_eager_cache(),
3447 })
3448 }
3449
3450 pub fn from_structured_storage(
3472 indices: Vec<DynIndex>,
3473 storage: Arc<Storage>,
3474 ) -> std::result::Result<Self, IdxTensorError> {
3475 Self::from_storage(indices, storage)
3476 }
3477
3478 pub fn from_copy_selector<T>(
3533 left: DynIndex,
3534 site: DynIndex,
3535 right: DynIndex,
3536 selected_value: usize,
3537 scale: T,
3538 ) -> std::result::Result<Self, StructuredSelectorError>
3539 where
3540 T: TensorElement + Copy + Zero,
3541 {
3542 if left.dim == 0 {
3543 return Err(StructuredSelectorError::ZeroDimension { axis: "left" });
3544 }
3545 if site.dim == 0 {
3546 return Err(StructuredSelectorError::ZeroDimension { axis: "site" });
3547 }
3548 if right.dim == 0 {
3549 return Err(StructuredSelectorError::ZeroDimension { axis: "right" });
3550 }
3551 if left.dim != right.dim {
3552 return Err(StructuredSelectorError::BondDimensionMismatch {
3553 left: left.dim,
3554 right: right.dim,
3555 });
3556 }
3557 if selected_value >= site.dim {
3558 return Err(StructuredSelectorError::SelectedValueOutOfBounds {
3559 value: selected_value,
3560 site_dim: site.dim,
3561 });
3562 }
3563
3564 let payload_len =
3565 left.dim
3566 .checked_mul(site.dim)
3567 .ok_or(StructuredSelectorError::PayloadSizeOverflow {
3568 bond_dim: left.dim,
3569 site_dim: site.dim,
3570 })?;
3571 let _site_stride = isize::try_from(left.dim)
3572 .map_err(|_| StructuredSelectorError::StrideOverflow { bond_dim: left.dim })?;
3573 let selected_offset = left.dim.checked_mul(selected_value).ok_or(
3574 StructuredSelectorError::PayloadSizeOverflow {
3575 bond_dim: left.dim,
3576 site_dim: site.dim,
3577 },
3578 )?;
3579
3580 let mut payload = Vec::new();
3581 payload.try_reserve_exact(payload_len).map_err(|_| {
3582 StructuredSelectorError::AllocationFailed {
3583 elements: payload_len,
3584 }
3585 })?;
3586 payload.resize(payload_len, T::zero());
3587 for bond in 0..left.dim {
3588 payload[selected_offset + bond] = scale;
3589 }
3590 let payload_native = dense_native_tensor_from_col_major(&payload, &[left.dim, site.dim])
3591 .map_err(|error| StructuredSelectorError::InvalidStorage {
3592 message: error.to_string(),
3593 })?;
3594 let payload_dtype = payload_native.dtype();
3595 let payload_inner = IdxTensor::untracked_inner(
3596 payload_native,
3597 default_eager_ctx().map_err(|error| StructuredSelectorError::InvalidStorage {
3598 message: error.to_string(),
3599 })?,
3600 )
3601 .map_err(|error| StructuredSelectorError::InvalidStorage {
3602 message: error.to_string(),
3603 })?;
3604 let payload_dims = vec![left.dim, site.dim];
3605 let indices = vec![left, site, right];
3606 if !matches!(
3607 payload_dtype,
3608 DType::F32 | DType::F64 | DType::C32 | DType::C64
3609 ) {
3610 return Err(StructuredSelectorError::InvalidStorage {
3611 message: format!("unsupported selector dtype {:?}", payload_dtype),
3612 });
3613 }
3614 Self::from_structured_payload_inner(indices, payload_inner, payload_dims, vec![0, 1, 0])
3615 .map_err(|error| StructuredSelectorError::InvalidStorage {
3616 message: error.to_string(),
3617 })
3618 }
3619
3620 pub(crate) fn from_native(indices: Vec<DynIndex>, native: NativeTensor) -> Result<Self> {
3622 let axis_classes = Self::dense_axis_classes(indices.len());
3623 Self::from_native_with_axis_classes(indices, native, axis_classes)
3624 }
3625
3626 pub(crate) fn from_native_with_axis_classes(
3627 indices: Vec<DynIndex>,
3628 native: NativeTensor,
3629 axis_classes: Vec<usize>,
3630 ) -> Result<Self> {
3631 Self::from_inner_with_axis_classes(
3632 indices,
3633 IdxTensor::untracked_inner(native, default_eager_ctx()?)?,
3634 axis_classes,
3635 )
3636 }
3637
3638 pub(crate) fn from_untracked_native_with_axis_classes(
3641 indices: Vec<DynIndex>,
3642 native: NativeTensor,
3643 axis_classes: Vec<usize>,
3644 ) -> Result<Self> {
3645 Self::from_inner_with_validated_metadata(
3646 indices,
3647 adopt_untracked_eager_value(default_eager_ctx()?, TensorValue::from_tensor(native))?,
3648 axis_classes,
3649 )
3650 }
3651
3652 pub(crate) fn from_inner(indices: Vec<DynIndex>, inner: EagerTensor) -> Result<Self> {
3653 let axis_classes = Self::dense_axis_classes(indices.len());
3654 Self::from_inner_with_axis_classes(indices, inner, axis_classes)
3655 }
3656
3657 pub fn hermitian_eigendecomposition(
3698 &self,
3699 hermitian_tol: f64,
3700 ) -> std::result::Result<TensorHermitianEigendecomposition, IdxTensorError> {
3701 if !(self.indices.len() == 2) {
3702 return Err(anyhow::anyhow!(
3703 "IdxTensor::hermitian_eigendecomposition requires a rank-2 tensor, got rank {}",
3704 self.indices.len()
3705 )
3706 .into());
3707 };
3708 let dims = self.dims();
3709 if !(dims[0] == dims[1]) {
3710 return Err(anyhow::anyhow!(
3711 "IdxTensor::hermitian_eigendecomposition requires a square matrix, got {}x{}",
3712 dims[0],
3713 dims[1]
3714 )
3715 .into());
3716 };
3717 if !(dims[0] > 0) {
3718 return Err(anyhow::anyhow!(
3719 "IdxTensor::hermitian_eigendecomposition requires a non-empty matrix"
3720 )
3721 .into());
3722 };
3723 if !(hermitian_tol.is_finite() && hermitian_tol >= 0.0) {
3724 return Err(anyhow::anyhow!(
3725 "IdxTensor::hermitian_eigendecomposition requires a finite non-negative tolerance"
3726 )
3727 .into());
3728 };
3729
3730 let input = self.try_materialized_inner()?;
3731 let (values, vectors) =
3732 with_eager_session(input.runtime(), None, |session| session.eigh(input)).map_err(
3733 |source| anyhow::anyhow!("Hermitian eigendecomposition failed: {source}"),
3734 )?;
3735
3736 let eigenvalue_index = DynIndex::new_dyn(dims[0]);
3737 let eigenvector_index = DynIndex::new_dyn(dims[0]);
3738 let eigenvalue_tensor = Self::from_inner(vec![eigenvalue_index], values)?;
3739 let eigenvalues = Self::read_real_eigenvalues(&eigenvalue_tensor, hermitian_tol)
3740 .with_context(|| {
3741 "IdxTensor::hermitian_eigendecomposition failed to read eigenvalues"
3742 })?;
3743 let eigenvectors = Self::from_inner(
3744 vec![self.indices[0].clone(), eigenvector_index.clone()],
3745 vectors,
3746 )?;
3747
3748 Ok(TensorHermitianEigendecomposition {
3749 eigenvalues,
3750 eigenvectors,
3751 eigenvector_index,
3752 })
3753 }
3754
3755 fn read_real_eigenvalues(values: &Self, hermitian_tol: f64) -> Result<Vec<f64>> {
3756 if values.is_complex() {
3757 let values = if values.is_c32() {
3758 values
3759 .to_vec::<Complex32>()?
3760 .into_iter()
3761 .map(|value| Complex64::new(f64::from(value.re), f64::from(value.im)))
3762 .collect()
3763 } else {
3764 values.to_vec::<Complex64>()?
3765 };
3766 values
3767 .into_iter()
3768 .enumerate()
3769 .map(|(index, value)| {
3770 let imaginary = value.im.abs();
3771 let allowed = hermitian_tol * value.norm().max(1.0);
3772 if !matches!(
3773 imaginary.partial_cmp(&allowed),
3774 Some(std::cmp::Ordering::Less) | Some(std::cmp::Ordering::Equal)
3775 ) {
3776 return Err(anyhow::anyhow!("Hermitian eigenvalue {index} has imaginary part {imaginary}, exceeding tolerance {allowed}"));
3777 };
3778 Ok(value.re)
3779 })
3780 .collect()
3781 } else if values.is_f32() {
3782 Ok(values.to_vec::<f32>()?.into_iter().map(f64::from).collect())
3783 } else {
3784 values.to_vec::<f64>().map_err(anyhow::Error::from)
3785 }
3786 }
3787
3788 pub(crate) fn from_diag_inner(
3789 indices: Vec<DynIndex>,
3790 payload_inner: EagerTensor,
3791 ) -> Result<Self> {
3792 let dims = Self::expected_dims_from_indices(&indices);
3793 Self::validate_indices(&indices)?;
3794 Self::validate_diag_dims(&dims)?;
3795 let payload_len = checked_product(payload_inner.shape())?;
3796 Self::validate_diag_payload_len(payload_len, &dims)?;
3797 let axis_classes = Self::diag_axis_classes(dims.len());
3798 let payload_dims = payload_inner.shape().to_vec();
3802 Ok(Self {
3803 indices,
3804 storage: IdxTensorStorage::Compact(Arc::new(StructuredPayload {
3805 payload: Arc::new(payload_inner),
3806 payload_dims,
3807 axis_classes,
3808 })),
3809 eager_cache: Self::empty_eager_cache(),
3810 })
3811 }
3812
3813 fn compact_inner_from_logical(
3814 inner: &EagerTensor,
3815 axis_classes: &[usize],
3816 ) -> Result<EagerTensor> {
3817 let mut payload = inner.clone();
3818 let mut classes = axis_classes.to_vec();
3819 while let Some((axis_a, axis_b)) = Self::first_duplicate_pair(&classes) {
3820 payload = with_eager_session(payload.runtime(), None, |session| {
3821 session.extract_diag(&payload, axis_a, axis_b)
3822 })?;
3823 classes.remove(axis_b);
3824 }
3825 Ok(payload)
3826 }
3827
3828 pub(crate) fn from_inner_with_axis_classes(
3829 indices: Vec<DynIndex>,
3830 inner: EagerTensor,
3831 axis_classes: Vec<usize>,
3832 ) -> Result<Self> {
3833 Self::from_inner_with_axis_classes_impl(indices, inner, axis_classes, false)
3834 }
3835
3836 fn from_inner_with_validated_metadata(
3837 indices: Vec<DynIndex>,
3838 inner: EagerTensor,
3839 axis_classes: Vec<usize>,
3840 ) -> Result<Self> {
3841 Self::from_inner_with_axis_classes_impl(indices, inner, axis_classes, true)
3842 }
3843
3844 fn from_inner_with_axis_classes_impl(
3845 indices: Vec<DynIndex>,
3846 inner: EagerTensor,
3847 axis_classes: Vec<usize>,
3848 validated: bool,
3849 ) -> Result<Self> {
3850 let dims = profile_pairwise_contract_section("from_inner_expected_dims", || {
3851 Self::expected_dims_from_indices(&indices)
3852 });
3853 let dense_axis_classes = axis_classes.iter().copied().eq(0..indices.len());
3854 if validated {
3855 debug_assert!(Self::validate_indices(&indices).is_ok());
3856 if !dense_axis_classes {
3857 Self::validate_axis_classes(&axis_classes, indices.len())?;
3858 }
3859 } else {
3860 profile_pairwise_contract_section("from_inner_validate_indices", || {
3861 Self::validate_indices(&indices)
3862 })?;
3863 Self::validate_axis_classes(&axis_classes, indices.len())?;
3864 }
3865 if dims != inner.shape() {
3866 return Err(anyhow::anyhow!(
3867 "native payload dims {:?} do not match indices dims {:?}",
3868 inner.shape(),
3869 dims
3870 ));
3871 }
3872 if Self::is_diag_axis_classes(&axis_classes) {
3873 profile_pairwise_contract_section("from_inner_validate_diag_dims", || {
3874 Self::validate_diag_dims(&dims)
3875 })?;
3876 }
3877 let storage = if dense_axis_classes {
3878 IdxTensorStorage::from_eager_dense(inner, indices.len())
3879 } else {
3880 let payload = Self::compact_inner_from_logical(&inner, &axis_classes)?;
3881 let payload_dims = payload.shape().to_vec();
3882 IdxTensorStorage::Compact(Arc::new(StructuredPayload {
3883 payload: Arc::new(payload),
3884 payload_dims,
3885 axis_classes,
3886 }))
3887 };
3888 Ok(Self {
3889 indices,
3890 storage,
3891 eager_cache: Self::empty_eager_cache(),
3892 })
3893 }
3894
3895 pub fn indices(&self) -> &[DynIndex] {
3897 &self.indices
3898 }
3899
3900 pub(crate) fn axis_classes(&self) -> &[usize] {
3901 self.storage.axis_classes()
3902 }
3903
3904 #[cfg(feature = "tenferro-cuda")]
3905 pub(crate) fn deferred_storage_error(&self) -> Option<&TensorStorageError> {
3906 self.storage.deferred_error()
3907 }
3908
3909 #[cfg(feature = "tenferro-cuda")]
3910 pub(crate) fn cuda_eager_inner(&self) -> Option<&EagerTensor> {
3911 self.storage
3912 .eager()
3913 .or_else(|| self.eager_cache.get().map(AsRef::as_ref))
3914 }
3915
3916 pub(crate) fn eager_runtime(&self) -> Option<Arc<EagerRuntime>> {
3921 self.storage
3922 .eager()
3923 .or_else(|| self.eager_cache.get().map(AsRef::as_ref))
3924 .map(|inner| Arc::clone(inner.runtime()))
3925 }
3926
3927 #[cfg(feature = "tenferro-cuda")]
3934 pub(crate) fn is_cuda_resident(&self) -> bool {
3935 self.cuda_eager_inner().is_some_and(|inner| {
3936 inner.tensor_read().placement().memory_kind == tenferro_tensor::MemoryKind::Device
3937 })
3938 }
3939
3940 #[cfg(not(feature = "tenferro-cuda"))]
3944 pub(crate) fn is_cuda_resident(&self) -> bool {
3945 false
3946 }
3947
3948 #[cfg(feature = "tenferro-cuda")]
3949 pub(crate) fn cuda_duplicate_native(&self) -> Result<NativeTensor> {
3950 let inner = self.try_materialized_inner()?;
3951 with_engine(inner.runtime(), None, |engine| {
3952 inner.duplicate_value(engine)
3953 })
3954 .map_err(anyhow::Error::new)
3955 }
3956
3957 pub fn enable_grad(self) -> std::result::Result<Self, IdxTensorError> {
3964 self.ensure_storage_ready()?;
3965 let eager_payload = self
3969 .storage
3970 .eager()
3971 .or_else(|| self.eager_cache.get().map(AsRef::as_ref))
3972 .filter(|inner| inner.shape() == self.storage.payload_dims());
3973 let payload = match eager_payload {
3974 Some(inner) => with_engine(inner.runtime(), None, |engine| {
3975 inner.duplicate_value(engine)
3976 })?,
3977 None => {
3978 let materialized = self.storage.materialize(self.indices.len())?;
3979 storage_payload_native(materialized.as_ref())
3980 .context("IdxTensor::enable_grad failed")?
3981 }
3982 };
3983 let payload_dims = self.storage.payload_dims().to_vec();
3984 let axis_classes = self.storage.axis_classes().to_vec();
3985 let tracked = Arc::new(EagerTensor::requires_grad_in(
3986 payload,
3987 default_eager_ctx()?,
3988 )?);
3989 let storage = if axis_classes == Self::dense_axis_classes(self.indices.len()) {
3990 IdxTensorStorage::Eager {
3991 inner: tracked,
3992 axis_classes,
3993 }
3994 } else {
3995 IdxTensorStorage::Compact(Arc::new(StructuredPayload {
3996 payload: tracked,
3997 payload_dims,
3998 axis_classes,
3999 }))
4000 };
4001 Ok(Self {
4002 indices: self.indices,
4003 storage,
4004 eager_cache: Self::empty_eager_cache(),
4005 })
4006 }
4007
4008 pub fn tracks_grad(&self) -> bool {
4010 self.storage.eager().is_some_and(EagerTensor::tracks_grad)
4011 || self
4012 .eager_cache
4013 .get()
4014 .is_some_and(|inner| inner.tracks_grad())
4015 }
4016
4017 pub fn grad(&self) -> std::result::Result<Option<Self>, IdxTensorError> {
4030 if let Some(value) = self.tracked_compact_payload_value() {
4031 let Some(gradient) = value.payload.grad()? else {
4032 return Ok(None);
4033 };
4034 let gradient_shape = gradient.shape().to_vec();
4035 let gradient_tensor = with_engine(value.payload.runtime(), None, |engine| {
4036 gradient.to_tensor(engine)
4037 })?;
4038 if self.compact_payload_is_logical_dense(&value.payload_dims) {
4039 return Ok(Some(Self::from_native_with_axis_classes(
4040 self.indices.clone(),
4041 gradient_tensor,
4042 value.axis_classes.clone(),
4043 )?));
4044 }
4045 if gradient_shape != value.payload_dims {
4046 return Err(anyhow::anyhow!(
4047 "gradient payload dims {:?} do not match {:?}",
4048 gradient_shape,
4049 value.payload_dims
4050 )
4051 .into());
4052 }
4053 let gradient = IdxTensor::untracked_inner(gradient_tensor, default_eager_ctx()?)?;
4054 return Ok(Some(Self::from_structured_payload_inner(
4055 self.indices.clone(),
4056 gradient,
4057 value.payload_dims.clone(),
4058 value.axis_classes.clone(),
4059 )?));
4060 }
4061
4062 let inner = self.try_materialized_inner()?;
4063 let Some(gradient) = inner.grad()? else {
4064 return Ok(None);
4065 };
4066 Ok(Some(Self::from_native_with_axis_classes(
4067 self.indices.clone(),
4068 with_engine(inner.runtime(), None, |engine| gradient.to_tensor(engine))?,
4069 self.storage.axis_classes().to_vec(),
4070 )?))
4071 }
4072
4073 pub fn clear_grad(&self) -> std::result::Result<(), IdxTensorError> {
4079 self.ensure_storage_ready()?;
4080 if let Some(value) = self.tracked_compact_payload_value() {
4081 value.payload.clear_grad()?;
4082 }
4083 if let Some(inner) = self.storage.eager() {
4084 inner.clear_grad()?;
4085 }
4086 if let Some(inner) = self.eager_cache.get() {
4087 inner.clear_grad()?;
4088 }
4089 Ok(())
4090 }
4091
4092 pub fn backward(&self) -> std::result::Result<(), IdxTensorError> {
4104 if let Some(value) = self.tracked_compact_payload_value() {
4105 return with_execution_engine(value.payload.runtime(), None, |engine| {
4106 value.payload.backward(engine)
4107 })
4108 .map(|_| ());
4109 }
4110 let inner = self.try_materialized_inner()?;
4111 with_execution_engine(inner.runtime(), None, |engine| inner.backward(engine)).map(|_| ())
4112 }
4113
4114 pub fn detach(&self) -> std::result::Result<Self, IdxTensorError> {
4126 let inner = self.try_materialized_inner()?;
4127 let detached = with_engine(inner.runtime(), None, |engine| inner.detach(engine))?;
4128 Self::from_inner_with_axis_classes(
4129 self.indices.clone(),
4130 detached,
4131 self.storage.axis_classes().to_vec(),
4132 )
4133 .map_err(IdxTensorError::from)
4134 }
4135
4136 pub fn is_simple(&self) -> bool {
4138 true
4139 }
4140
4141 pub fn to_storage(&self) -> std::result::Result<Arc<Storage>, TensorStorageError> {
4153 self.storage.materialize(self.indices.len())
4154 }
4155
4156 pub fn storage(&self) -> std::result::Result<Arc<Storage>, TensorStorageError> {
4178 self.storage.materialize(self.indices.len())
4179 }
4180
4181 pub fn storage_kind(&self) -> StorageKind {
4200 self.storage.storage_kind()
4201 }
4202
4203 pub fn content_digest(&self) -> Option<u64> {
4251 self.storage.content_digest()
4252 }
4253
4254 #[cfg(feature = "tenferro-cuda")]
4283 pub fn cuda_dtype(&self) -> std::result::Result<DType, IdxTensorError> {
4284 self.scalar_dtype().map_err(IdxTensorError::from)
4285 }
4286
4287 pub fn sum(&self) -> std::result::Result<AnyScalar, IdxTensorError> {
4303 self.ensure_storage_ready()?;
4304 if self.indices.is_empty() {
4305 return AnyScalar::from_tensor(self.clone()).map_err(IdxTensorError::from);
4306 }
4307 if let Some(payload) = self.storage.eager().filter(|payload| payload.tracks_grad()) {
4308 let axes: Vec<usize> = (0..payload.shape().len()).collect();
4309 let reduced = with_eager_session(payload.runtime(), None, |session| {
4310 session.reduce_sum(payload, Some(&axes))
4311 })?;
4312 return AnyScalar::from_tensor(Self::from_inner(Vec::new(), reduced)?)
4313 .map_err(IdxTensorError::from);
4314 }
4315 self.storage.sum_scalar().map_err(IdxTensorError::from)
4316 }
4317
4318 pub fn only(&self) -> std::result::Result<AnyScalar, IdxTensorError> {
4339 let dims = self.dims();
4340 let total_size = checked_product(&dims)?;
4341 if !(total_size == 1 || dims.is_empty()) {
4342 return Err(anyhow::anyhow!(
4343 "only() requires a scalar tensor (1 element), got {} elements with dims {:?}",
4344 if dims.is_empty() { 1 } else { total_size },
4345 dims
4346 )
4347 .into());
4348 };
4349 self.sum()
4350 }
4351
4352 pub fn permute_indices(
4387 &self,
4388 new_indices: &[DynIndex],
4389 ) -> std::result::Result<Self, IdxTensorError> {
4390 let perm = compute_permutation_from_indices(&self.indices, new_indices)?;
4392 if perm.iter().copied().eq(0..perm.len()) {
4393 return Ok(Self {
4394 indices: new_indices.to_vec(),
4395 storage: self.storage.clone(),
4396 eager_cache: Arc::clone(&self.eager_cache),
4397 });
4398 }
4399
4400 let inner = self.try_materialized_inner()?;
4401 let permuted = with_eager_session(inner.runtime(), None, |session| {
4402 session.transpose(inner, &perm)
4403 })?;
4404 let axis_classes = self.permute_axis_classes(&perm);
4405 Self::from_inner_with_axis_classes(new_indices.to_vec(), permuted, axis_classes)
4406 .map_err(IdxTensorError::from)
4407 }
4408
4409 pub fn permute(&self, perm: &[usize]) -> std::result::Result<Self, IdxTensorError> {
4441 if let Err(message) = validate_permutation(self.indices.len(), perm) {
4442 return Err(anyhow::anyhow!(message).into());
4443 }
4444 if perm.iter().copied().eq(0..perm.len()) {
4445 return Ok(self.clone());
4446 }
4447
4448 let new_indices: Vec<DynIndex> = perm.iter().map(|&i| self.indices[i].clone()).collect();
4450 let inner = self.try_materialized_inner()?;
4451 let permuted = with_eager_session(inner.runtime(), None, |session| {
4452 session.transpose(inner, perm)
4453 })?;
4454 let axis_classes = self.permute_axis_classes(perm);
4455 Self::from_inner_with_axis_classes(new_indices, permuted, axis_classes)
4456 .map_err(IdxTensorError::from)
4457 }
4458
4459 pub(crate) fn try_contract_pairwise_default(&self, other: &Self) -> Result<Self> {
4460 self.try_contract_pairwise_default_with_options(other, PairwiseContractionOptions::new())
4461 }
4462
4463 pub(crate) fn try_contract_pairwise_default_with_options(
4464 &self,
4465 other: &Self,
4466 options: PairwiseContractionOptions,
4467 ) -> Result<Self> {
4468 self.try_contract_pairwise_default_with_options_scoped(other, options, None)
4469 }
4470
4471 pub(crate) fn try_contract_pairwise_default_with_options_in(
4476 &self,
4477 other: &Self,
4478 options: PairwiseContractionOptions,
4479 context: &ExecutionContext,
4480 ) -> Result<Self> {
4481 self.try_contract_pairwise_default_with_options_scoped(other, options, Some(context))
4482 }
4483
4484 fn try_contract_pairwise_default_with_options_scoped(
4485 &self,
4486 other: &Self,
4487 options: PairwiseContractionOptions,
4488 context: Option<&ExecutionContext>,
4489 ) -> Result<Self> {
4490 #[cfg(feature = "tenferro-cuda")]
4496 if self.is_cuda_resident()
4497 && other.is_cuda_resident()
4498 && self.should_use_structured_payload_contract(other)
4499 {
4500 return super::contract::contract_pair_via_plan(self, other, options);
4501 }
4502 let self_indices = profile_pairwise_contract_section("operand_indices", || {
4503 self.operand_indices_for_contraction(options.lhs_conj)
4504 });
4505 let other_indices = profile_pairwise_contract_section("operand_indices", || {
4506 other.operand_indices_for_contraction(options.rhs_conj)
4507 });
4508 let self_dims = profile_pairwise_contract_section("expected_dims", || {
4509 Self::expected_dims_from_indices(&self_indices)
4510 });
4511 let other_dims = profile_pairwise_contract_section("expected_dims", || {
4512 Self::expected_dims_from_indices(&other_indices)
4513 });
4514 let spec = profile_pairwise_contract_section("prepare_contraction", || {
4515 prepare_contraction(&self_indices, &self_dims, &other_indices, &other_dims)
4516 })
4517 .context("contraction preparation failed")?;
4518 let result_axis_classes = profile_pairwise_contract_section("result_axis_classes", || {
4519 Self::binary_contraction_axis_classes(
4520 self.storage.axis_classes(),
4521 &spec.axes_a,
4522 other.storage.axis_classes(),
4523 &spec.axes_b,
4524 )
4525 })?;
4526
4527 if profile_pairwise_contract_section("structured_check", || {
4528 self.should_use_structured_payload_contract(other)
4529 }) {
4530 if options.has_conj() {
4531 let lhs = if options.lhs_conj {
4532 self.conj()
4533 } else {
4534 self.clone()
4535 };
4536 let rhs = if options.rhs_conj {
4537 other.conj()
4538 } else {
4539 other.clone()
4540 };
4541 return profile_pairwise_contract_section("structured_conj_fallback", || {
4542 lhs.try_contract_pairwise_default(&rhs)
4543 });
4544 }
4545 return profile_pairwise_contract_section("structured_payload_contract", || {
4546 self.contract_structured_payloads(
4547 other,
4548 spec.result_indices,
4549 &spec.axes_a,
4550 &spec.axes_b,
4551 context,
4552 )
4553 });
4554 }
4555
4556 if self.indices.is_empty() && other.indices.is_empty() {
4557 if options.has_conj() {
4558 let lhs = if options.lhs_conj {
4559 self.conj()
4560 } else {
4561 self.clone()
4562 };
4563 let rhs = if options.rhs_conj {
4564 other.conj()
4565 } else {
4566 other.clone()
4567 };
4568 return lhs.try_contract_pairwise_default(&rhs);
4569 }
4570 let result = profile_pairwise_contract_section("scalar_mul", || {
4571 let lhs = self.try_materialized_inner()?;
4572 let rhs = other.try_materialized_inner()?;
4573 Ok::<_, anyhow::Error>(with_eager_session(lhs.runtime(), context, |session| {
4574 session.mul(lhs, rhs)
4575 })?)
4576 })?;
4577 return profile_pairwise_contract_section("from_inner", || {
4578 Self::from_inner(spec.result_indices, result)
4579 });
4580 }
4581
4582 let self_inner = self.try_materialized_inner()?;
4583 let other_inner = other.try_materialized_inner()?;
4584 let self_dtype = self_inner.dtype();
4585 let other_dtype = other_inner.dtype();
4586 if self_dtype != other_dtype && options.has_conj() {
4587 let lhs = if options.lhs_conj {
4588 self.conj()
4589 } else {
4590 self.clone()
4591 };
4592 let rhs = if options.rhs_conj {
4593 other.conj()
4594 } else {
4595 other.clone()
4596 };
4597 return lhs.try_contract_pairwise_default(&rhs);
4598 }
4599 let promoted_lhs;
4603 let promoted_rhs;
4604 let (lhs, rhs): (&EagerTensor, &EagerTensor) = if self_dtype != other_dtype {
4605 let (lhs, rhs) = profile_pairwise_contract_section("promote_mixed_dtype", || {
4606 Self::promote_eager_operands(self_inner, other_inner)
4607 })?;
4608 promoted_lhs = lhs;
4609 promoted_rhs = rhs;
4610 (&promoted_lhs, &promoted_rhs)
4611 } else {
4612 (self_inner, other_inner)
4613 };
4614
4615 let config = profile_pairwise_contract_section("build_dot_general_config", || {
4616 Self::binary_dot_general_config(&spec.axes_a, &spec.axes_b)
4617 })?;
4618 let result = profile_pairwise_contract_section("dot_general_with_conj", || {
4619 profile_pairwise_contract_section("dot_general_execute", || {
4620 with_eager_session(lhs.runtime(), context, |session| {
4621 session.dot_general_with_conj(
4622 lhs,
4623 rhs,
4624 config,
4625 options.lhs_conj,
4626 options.rhs_conj,
4627 )
4628 })
4629 })
4630 .map_err(anyhow::Error::from)
4631 })?;
4632 record_pairwise_contract_profile_bytes(
4633 "dot_general_output",
4634 tensor_profile_bytes(result.dtype(), result.shape())?,
4635 );
4636 profile_pairwise_contract_section("from_inner_axis_classes", || {
4637 Self::from_inner_with_axis_classes(spec.result_indices, result, result_axis_classes)
4638 })
4639 }
4640
4641 pub(crate) fn try_contract_pairwise_retaining(
4646 &self,
4647 other: &Self,
4648 retained_indices: &[DynIndex],
4649 ) -> Result<Self> {
4650 if retained_indices.is_empty()
4651 || self.should_use_structured_payload_contract(other)
4652 || self.try_materialized_inner()?.dtype() != other.try_materialized_inner()?.dtype()
4653 {
4654 let options = ContractionOptions::new().with_retain_indices(retained_indices);
4655 return super::contract::contract_with_options(&[self, other], options)
4656 .map_err(anyhow::Error::from);
4657 }
4658
4659 let common = common_ind_positions(&self.indices, &other.indices);
4660 if common.is_empty() {
4661 let options = ContractionOptions::new().with_retain_indices(retained_indices);
4662 return super::contract::contract_with_options(&[self, other], options)
4663 .map_err(anyhow::Error::from);
4664 }
4665
4666 let mut retained_pairs = Vec::with_capacity(retained_indices.len());
4667 for retained in retained_indices {
4668 let Some(pos_a) = self.indices.iter().position(|index| index == retained) else {
4669 let options = ContractionOptions::new().with_retain_indices(retained_indices);
4670 return super::contract::contract_with_options(&[self, other], options)
4671 .map_err(anyhow::Error::from);
4672 };
4673 let Some(pos_b) = other.indices.iter().position(|index| index == retained) else {
4674 let options = ContractionOptions::new().with_retain_indices(retained_indices);
4675 return super::contract::contract_with_options(&[self, other], options)
4676 .map_err(anyhow::Error::from);
4677 };
4678 if !self.indices[pos_a].is_contractable(&other.indices[pos_b]) {
4679 let options = ContractionOptions::new().with_retain_indices(retained_indices);
4680 return super::contract::contract_with_options(&[self, other], options)
4681 .map_err(anyhow::Error::from);
4682 }
4683 retained_pairs.push((pos_a, pos_b));
4684 }
4685
4686 let retained_pairs_set: HashSet<(usize, usize)> = retained_pairs.iter().copied().collect();
4687 let mut contracting_a = Vec::with_capacity(common.len());
4688 let mut contracting_b = Vec::with_capacity(common.len());
4689 for &(pos_a, pos_b) in &common {
4690 if !retained_pairs_set.contains(&(pos_a, pos_b)) {
4691 contracting_a.push(pos_a);
4692 contracting_b.push(pos_b);
4693 }
4694 }
4695
4696 let config = DotGeneralConfig {
4697 lhs_contracting_dims: contracting_a.as_slice().into(),
4698 rhs_contracting_dims: contracting_b.as_slice().into(),
4699 lhs_batch_dims: retained_pairs
4700 .iter()
4701 .map(|&(pos_a, _)| pos_a)
4702 .collect::<Vec<_>>()
4703 .as_slice()
4704 .into(),
4705 rhs_batch_dims: retained_pairs
4706 .iter()
4707 .map(|&(_, pos_b)| pos_b)
4708 .collect::<Vec<_>>()
4709 .as_slice()
4710 .into(),
4711 };
4712 let lhs = self.try_materialized_inner()?;
4713 let rhs = other.try_materialized_inner()?;
4714 let result = with_eager_session(lhs.runtime(), None, |session| {
4715 session.dot_general_with_conj(lhs, rhs, config, false, false)
4716 })
4717 .map_err(|error| anyhow::anyhow!("retained pairwise contraction failed: {error}"))?;
4718
4719 let contracting_a_set: HashSet<usize> = contracting_a.into_iter().collect();
4725 let contracting_b_set: HashSet<usize> = contracting_b.into_iter().collect();
4726 let retained_a_set: HashSet<usize> = retained_pairs.iter().map(|&(a, _)| a).collect();
4727 let retained_b_set: HashSet<usize> = retained_pairs.iter().map(|&(_, b)| b).collect();
4728 let mut current_indices = self
4729 .indices
4730 .iter()
4731 .enumerate()
4732 .filter(|(axis, _)| !retained_a_set.contains(axis) && !contracting_a_set.contains(axis))
4733 .map(|(_, index)| index.clone())
4734 .chain(
4735 other
4736 .indices
4737 .iter()
4738 .enumerate()
4739 .filter(|(axis, _)| {
4740 !retained_b_set.contains(axis) && !contracting_b_set.contains(axis)
4741 })
4742 .map(|(_, index)| index.clone()),
4743 )
4744 .collect::<Vec<_>>();
4745 current_indices.extend(
4746 retained_pairs
4747 .iter()
4748 .map(|&(pos_a, _)| self.indices[pos_a].clone()),
4749 );
4750
4751 let mut desired_indices = self
4752 .indices
4753 .iter()
4754 .enumerate()
4755 .filter(|(axis, _)| !contracting_a_set.contains(axis))
4756 .map(|(_, index)| index.clone())
4757 .chain(
4758 other
4759 .indices
4760 .iter()
4761 .enumerate()
4762 .filter(|(axis, _)| {
4763 !contracting_b_set.contains(axis) && !retained_b_set.contains(axis)
4764 })
4765 .map(|(_, index)| index.clone()),
4766 )
4767 .collect::<Vec<_>>();
4768 if desired_indices.is_empty() {
4769 desired_indices = current_indices.clone();
4770 }
4771 let result = if current_indices == desired_indices {
4772 result
4773 } else {
4774 let permutation = desired_indices
4775 .iter()
4776 .map(|desired| {
4777 current_indices
4778 .iter()
4779 .position(|current| current == desired)
4780 .ok_or_else(|| {
4781 anyhow::anyhow!(
4782 "retained pairwise result is missing index {:?}",
4783 desired
4784 )
4785 })
4786 })
4787 .collect::<Result<Vec<_>>>()?;
4788 with_eager_session(result.runtime(), None, |session| {
4789 session.transpose(&result, &permutation)
4790 })?
4791 };
4792 Self::from_inner(desired_indices, result)
4793 }
4794
4795 pub(crate) fn try_tensordot_pairwise_explicit(
4796 &self,
4797 other: &Self,
4798 pairs: &[(DynIndex, DynIndex)],
4799 ) -> Result<Self> {
4800 use crate::index_ops::ContractionError;
4801
4802 let self_dims = Self::expected_dims_from_indices(&self.indices);
4803 let other_dims = Self::expected_dims_from_indices(&other.indices);
4804 let spec = prepare_contraction_pairs(
4805 &self.indices,
4806 &self_dims,
4807 &other.indices,
4808 &other_dims,
4809 pairs,
4810 )
4811 .map_err(|e| match e {
4812 ContractionError::NoCommonIndices => {
4813 anyhow::anyhow!("tensordot: No pairs specified for contraction")
4814 }
4815 ContractionError::BatchContractionNotImplemented => anyhow::anyhow!(
4816 "tensordot: Common index found but not in contraction pairs. \
4817 Batch contraction is not yet implemented."
4818 ),
4819 ContractionError::IndexNotFound { tensor } => {
4820 anyhow::anyhow!("tensordot: Index not found in {} tensor", tensor)
4821 }
4822 ContractionError::DimensionMismatch {
4823 pos_a,
4824 pos_b,
4825 dim_a,
4826 dim_b,
4827 } => anyhow::anyhow!(
4828 "tensordot: Dimension mismatch: self[{}]={} != other[{}]={}",
4829 pos_a,
4830 dim_a,
4831 pos_b,
4832 dim_b
4833 ),
4834 ContractionError::DuplicateAxis { tensor, pos } => {
4835 anyhow::anyhow!("tensordot: Duplicate axis {} in {} tensor", pos, tensor)
4836 }
4837 })?;
4838 let result_axis_classes = Self::binary_contraction_axis_classes(
4839 self.storage.axis_classes(),
4840 &spec.axes_a,
4841 other.storage.axis_classes(),
4842 &spec.axes_b,
4843 )?;
4844
4845 if self.should_use_structured_payload_contract(other) {
4846 return self.contract_structured_payloads(
4847 other,
4848 spec.result_indices,
4849 &spec.axes_a,
4850 &spec.axes_b,
4851 None,
4852 );
4853 }
4854
4855 if self.indices.is_empty() && other.indices.is_empty() {
4856 let lhs = self.try_materialized_inner()?;
4857 let rhs = other.try_materialized_inner()?;
4858 let result = with_eager_session(lhs.runtime(), None, |session| session.mul(lhs, rhs))
4859 .map_err(|e| anyhow::anyhow!("tensordot scalar multiply failed: {e}"))?;
4860 return Self::from_inner(spec.result_indices, result);
4861 }
4862
4863 let self_inner = self.try_materialized_inner()?;
4864 let other_inner = other.try_materialized_inner()?;
4865 let self_dtype = self_inner.dtype();
4866 let other_dtype = other_inner.dtype();
4867 let promoted_lhs;
4871 let promoted_rhs;
4872 let (lhs, rhs): (&EagerTensor, &EagerTensor) = if self_dtype != other_dtype {
4873 let (lhs, rhs) = Self::promote_eager_operands(self_inner, other_inner)?;
4874 promoted_lhs = lhs;
4875 promoted_rhs = rhs;
4876 (&promoted_lhs, &promoted_rhs)
4877 } else {
4878 (self_inner, other_inner)
4879 };
4880
4881 let subscripts = Self::build_binary_einsum_subscripts(
4882 self.indices.len(),
4883 &spec.axes_a,
4884 other.indices.len(),
4885 &spec.axes_b,
4886 )?;
4887 let result = with_eager_session(lhs.runtime(), None, |session| {
4888 session.einsum_subscripts(&[lhs, rhs], &subscripts)
4889 })
4890 .map_err(|e| anyhow::anyhow!("tensordot failed: {e}"))?;
4891 Self::from_inner_with_axis_classes(spec.result_indices, result, result_axis_classes)
4892 }
4893
4894 pub(crate) fn try_outer_product_pairwise(&self, other: &Self) -> Result<Self> {
4895 use anyhow::Context;
4896
4897 let common_positions = common_ind_positions(&self.indices, &other.indices);
4899 if !common_positions.is_empty() {
4900 let common_ids: Vec<_> = common_positions
4901 .iter()
4902 .map(|(pos_a, _)| self.indices[*pos_a].id())
4903 .collect();
4904 return Err(anyhow::anyhow!(
4905 "outer_product: tensors have common indices {:?}. \
4906 Use tensordot to contract common indices, or use sim() to replace \
4907 indices with fresh IDs before computing outer product.",
4908 common_ids
4909 ))
4910 .context("outer_product: common indices found");
4911 }
4912
4913 let mut result_indices = self.indices.clone();
4915 result_indices.extend(other.indices.iter().cloned());
4916 let result_axis_classes = Self::binary_contraction_axis_classes(
4917 self.storage.axis_classes(),
4918 &[],
4919 other.storage.axis_classes(),
4920 &[],
4921 )?;
4922 if self.should_use_structured_payload_contract(other) {
4923 return self.contract_structured_payloads(other, result_indices, &[], &[], None);
4924 }
4925 let self_inner = self.try_materialized_inner()?;
4926 let other_inner = other.try_materialized_inner()?;
4927 let self_dtype = self_inner.dtype();
4928 let other_dtype = other_inner.dtype();
4929 let promoted_lhs;
4933 let promoted_rhs;
4934 let (lhs, rhs): (&EagerTensor, &EagerTensor) = if self_dtype != other_dtype {
4935 let (lhs, rhs) = Self::promote_eager_operands(self_inner, other_inner)?;
4936 promoted_lhs = lhs;
4937 promoted_rhs = rhs;
4938 (&promoted_lhs, &promoted_rhs)
4939 } else {
4940 (self_inner, other_inner)
4941 };
4942
4943 let subscripts = Self::build_binary_einsum_subscripts(
4944 self.indices.len(),
4945 &[],
4946 other.indices.len(),
4947 &[],
4948 )?;
4949 let result = with_eager_session(lhs.runtime(), None, |session| {
4950 session.einsum_subscripts(&[lhs, rhs], &subscripts)
4951 })
4952 .map_err(|e| anyhow::anyhow!("outer_product failed: {e}"))?;
4953 Self::from_inner_with_axis_classes(result_indices, result, result_axis_classes)
4954 }
4955}
4956
4957impl IdxTensor {
4962 pub fn random<T: RandomScalar, R: Rng + ?Sized>(
5000 rng: &mut R,
5001 indices: Vec<DynIndex>,
5002 ) -> std::result::Result<Self, IdxTensorError> {
5003 let dims: Vec<usize> = indices.iter().map(|idx| idx.dim()).collect();
5004 let size = checked_product(&dims)?;
5005 let data: Vec<T> = (0..size).map(|_| T::random_value(rng)).collect();
5006 Self::from_dense(indices, data)
5007 }
5008
5009 pub(crate) fn constant_dense_like<T: TensorElement>(
5013 &self,
5014 indices: Vec<DynIndex>,
5015 data: Vec<T>,
5016 ) -> std::result::Result<Self, IdxTensorError> {
5017 let Some(runtime) = self.eager_runtime() else {
5018 return Self::from_dense(indices, data);
5019 };
5020 let dims = Self::expected_dims_from_indices(&indices);
5021 Self::validate_indices(&indices)?;
5022 Self::validate_dense_payload_len(data.len(), &dims)?;
5023 let native = dense_native_tensor_from_col_major(&data, &dims)
5024 .map_err(|error| IdxTensorError::operation("constant sketch construction", error))?;
5025 let inner = Self::untracked_inner(native, runtime)
5026 .map_err(|error| IdxTensorError::operation("constant sketch wrapping", error.into()))?;
5027 Self::from_inner(indices, inner).map_err(IdxTensorError::from)
5028 }
5029}
5030
5031impl IdxTensor {
5032 pub fn add(&self, other: &Self) -> std::result::Result<Self, IdxTensorError> {
5071 if self.indices.len() != other.indices.len() {
5073 return Err(anyhow::anyhow!(
5074 "Index count mismatch: self has {} indices, other has {}",
5075 self.indices.len(),
5076 other.indices.len()
5077 )
5078 .into());
5079 }
5080
5081 let self_set: HashSet<_> = self.indices.iter().collect();
5083 let other_set: HashSet<_> = other.indices.iter().collect();
5084
5085 if self_set != other_set {
5086 return Err(
5087 anyhow::anyhow!("Index set mismatch: tensors must have the same indices").into(),
5088 );
5089 }
5090
5091 let other_aligned = other.permute_indices(&self.indices)?;
5093
5094 let self_expected_dims = Self::expected_dims_from_indices(&self.indices);
5096 let other_expected_dims = Self::expected_dims_from_indices(&other_aligned.indices);
5097 if self_expected_dims != other_expected_dims {
5098 use crate::TagSetLike;
5099 let fmt = |indices: &[DynIndex]| -> Vec<String> {
5100 indices
5101 .iter()
5102 .map(|idx| {
5103 let tags: Vec<String> = idx.tags().iter().collect();
5104 format!("{:?}(dim={},tags={:?})", idx.id(), idx.dim(), tags)
5105 })
5106 .collect()
5107 };
5108 return Err(anyhow::anyhow!(
5109 "Dimension mismatch after alignment.\n\
5110 self: dims={:?}, indices(order)={:?}\n\
5111 other_aligned: dims={:?}, indices(order)={:?}",
5112 self_expected_dims,
5113 fmt(&self.indices),
5114 other_expected_dims,
5115 fmt(&other_aligned.indices)
5116 )
5117 .into());
5118 }
5119
5120 self.axpby(
5121 AnyScalar::new_real(1.0),
5122 &other_aligned,
5123 AnyScalar::new_real(1.0),
5124 )
5125 }
5126
5127 pub fn axpby(
5153 &self,
5154 a: AnyScalar,
5155 other: &Self,
5156 b: AnyScalar,
5157 ) -> std::result::Result<Self, IdxTensorError> {
5158 if self.indices.len() != other.indices.len() {
5160 return Err(anyhow::anyhow!(
5161 "Index count mismatch: self has {} indices, other has {}",
5162 self.indices.len(),
5163 other.indices.len()
5164 )
5165 .into());
5166 }
5167
5168 let self_set: HashSet<_> = self.indices.iter().collect();
5170 let other_set: HashSet<_> = other.indices.iter().collect();
5171 if self_set != other_set {
5172 return Err(
5173 anyhow::anyhow!("Index set mismatch: tensors must have the same indices").into(),
5174 );
5175 }
5176
5177 let other_aligned = other.permute_indices(&self.indices)?;
5179
5180 let self_expected_dims = Self::expected_dims_from_indices(&self.indices);
5182 let other_expected_dims = Self::expected_dims_from_indices(&other_aligned.indices);
5183 if self_expected_dims != other_expected_dims {
5184 return Err(anyhow::anyhow!(
5185 "Dimension mismatch after alignment: self={:?}, other_aligned={:?}",
5186 self_expected_dims,
5187 other_expected_dims
5188 )
5189 .into());
5190 }
5191
5192 let axis_classes = if self.storage.axis_classes() == other_aligned.storage.axis_classes() {
5193 self.storage.axis_classes().to_vec()
5194 } else {
5195 Self::dense_axis_classes(self.indices.len())
5196 };
5197
5198 let same_compact_layout = self.storage.payload_dims()
5199 == other_aligned.storage.payload_dims()
5200 && self.storage.payload_strides_vec() == other_aligned.storage.payload_strides_vec()
5201 && self.storage.axis_classes() == other_aligned.storage.axis_classes();
5202 if same_compact_layout
5203 && matches!(&self.storage, IdxTensorStorage::Materialized(_))
5204 && matches!(&other_aligned.storage, IdxTensorStorage::Materialized(_))
5205 && !self.tracks_grad()
5206 && !other_aligned.tracks_grad()
5207 && !a.tracks_grad()
5208 && !b.tracks_grad()
5209 {
5210 let lhs_storage = self.storage.materialize(self.indices.len())?;
5211 let rhs_storage = other_aligned
5212 .storage
5213 .materialize(other_aligned.indices.len())?;
5214 let combined = lhs_storage
5215 .axpby(
5216 &a.to_backend_scalar(),
5217 rhs_storage.as_ref(),
5218 &b.to_backend_scalar(),
5219 )
5220 .map_err(|e| anyhow::anyhow!("storage axpby failed: {e}"))?;
5221 return Self::from_storage(self.indices.clone(), Arc::new(combined));
5222 }
5223
5224 let lhs = self.scale(a)?;
5225 let rhs = other_aligned.scale(b)?;
5226 let lhs_inner = lhs.try_materialized_inner()?;
5227 let rhs_inner = rhs.try_materialized_inner()?;
5228 let combined = with_eager_session(lhs_inner.runtime(), None, |session| {
5229 session.add(lhs_inner, rhs_inner)
5230 })
5231 .map_err(|e| anyhow::anyhow!("tensor addition failed: {e}"))?;
5232 Self::from_inner_with_axis_classes(self.indices.clone(), combined, axis_classes)
5233 .map_err(IdxTensorError::from)
5234 }
5235
5236 pub fn scale(&self, scalar: AnyScalar) -> std::result::Result<Self, IdxTensorError> {
5256 if matches!(
5257 &self.storage,
5258 IdxTensorStorage::Eager { .. }
5259 | IdxTensorStorage::Compact(_)
5260 | IdxTensorStorage::Materialized(_)
5261 ) {
5262 let storage = self.storage.scale_eager_payload(&scalar)?;
5267 return Ok(Self {
5268 indices: self.indices.clone(),
5269 storage,
5270 eager_cache: Self::empty_eager_cache(),
5271 });
5272 }
5273
5274 let self_dtype = self.try_materialized_inner()?.dtype();
5275 let scalar_dtype = scalar.as_tensor()?.try_materialized_inner()?.dtype();
5276 if self_dtype != scalar_dtype {
5277 let target_dtype = Self::scale_target_dtype(self_dtype, scalar_dtype)?;
5278 let self_inner = self.try_materialized_inner()?;
5279 let self_inner = if self_inner.dtype() == target_dtype {
5280 self_inner.clone()
5281 } else {
5282 with_eager_session(self_inner.runtime(), None, |session| {
5283 session.cast(self_inner, target_dtype)
5284 })?
5285 };
5286 let scalar_inner = scalar.as_tensor()?.try_materialized_inner()?;
5287 let scalar_inner = if scalar_inner.dtype() == target_dtype {
5288 scalar_inner.clone()
5289 } else {
5290 with_eager_session(scalar_inner.runtime(), None, |session| {
5291 session.cast(scalar_inner, target_dtype)
5292 })?
5293 };
5294 let scaled = if self.indices.is_empty() {
5295 with_eager_session(self_inner.runtime(), None, |session| {
5296 session.mul(&self_inner, &scalar_inner)
5297 })
5298 .map_err(|e| anyhow::anyhow!("scalar multiplication failed: {e}"))?
5299 } else {
5300 let subscripts = Self::scale_subscripts(self.indices.len())?;
5301 with_eager_session(self_inner.runtime(), None, |session| {
5302 session.einsum_subscripts(&[&self_inner, &scalar_inner], &subscripts)
5303 })
5304 .map_err(|e| anyhow::anyhow!("tensor scaling failed: {e}"))?
5305 };
5306 return Self::from_inner_with_axis_classes(
5307 self.indices.clone(),
5308 scaled,
5309 self.storage.axis_classes().to_vec(),
5310 )
5311 .map_err(IdxTensorError::from);
5312 }
5313 let self_inner = self.try_materialized_inner()?;
5314 let scalar_inner = scalar.as_tensor()?.try_materialized_inner()?;
5315 let scaled = if self.indices.is_empty() {
5316 with_eager_session(self_inner.runtime(), None, |session| {
5317 session.mul(self_inner, scalar_inner)
5318 })
5319 .map_err(|e| anyhow::anyhow!("scalar multiplication failed: {e}"))?
5320 } else {
5321 let subscripts = Self::scale_subscripts(self.indices.len())?;
5322 with_eager_session(self_inner.runtime(), None, |session| {
5323 session.einsum_subscripts(&[self_inner, scalar_inner], &subscripts)
5324 })
5325 .map_err(|e| anyhow::anyhow!("tensor scaling failed: {e}"))?
5326 };
5327 Self::from_inner_with_axis_classes(
5328 self.indices.clone(),
5329 scaled,
5330 self.storage.axis_classes().to_vec(),
5331 )
5332 .map_err(IdxTensorError::from)
5333 }
5334
5335 pub fn inner_product(&self, other: &Self) -> std::result::Result<AnyScalar, IdxTensorError> {
5356 if self.indices.len() == other.indices.len() {
5357 let self_set: HashSet<_> = self.indices.iter().collect();
5358 let other_set: HashSet<_> = other.indices.iter().collect();
5359 if self_set == other_set {
5360 let other_aligned = other.permute_indices(&self.indices)?;
5361 let result = super::contract::contract_pair_with_operand_options(
5362 self,
5363 &other_aligned,
5364 PairwiseContractionOptions::new().with_lhs_conj(true),
5365 )?;
5366 return result.sum();
5367 }
5368 }
5369
5370 let result = super::contract::contract_pair_with_operand_options(
5372 self,
5373 other,
5374 PairwiseContractionOptions::new().with_lhs_conj(true),
5375 )?;
5376 result.sum()
5378 }
5379}
5380
5381impl IdxTensor {
5386 pub fn replaceind(
5421 &self,
5422 old_index: &DynIndex,
5423 new_index: &DynIndex,
5424 ) -> std::result::Result<Self, IdxTensorError> {
5425 if old_index.dim() != new_index.dim() {
5427 return Err(IdxTensorError::ShapeMismatch {
5428 operation: "replaceind",
5429 expected: format!("dimension {}", old_index.dim()),
5430 actual: format!("dimension {}", new_index.dim()),
5431 });
5432 }
5433
5434 let new_indices: Vec<_> = self
5435 .indices
5436 .iter()
5437 .map(|idx| {
5438 if *idx == *old_index {
5439 new_index.clone()
5440 } else {
5441 idx.clone()
5442 }
5443 })
5444 .collect();
5445
5446 Ok(Self {
5447 indices: new_indices,
5448 storage: self.storage.clone(),
5449 eager_cache: Arc::clone(&self.eager_cache),
5450 })
5451 }
5452
5453 pub fn replace_indices(
5492 &self,
5493 old_indices: &[DynIndex],
5494 new_indices: &[DynIndex],
5495 ) -> std::result::Result<Self, IdxTensorError> {
5496 if old_indices.len() != new_indices.len() {
5497 return Err(IdxTensorError::ShapeMismatch {
5498 operation: "replace_indices",
5499 expected: format!("{} indices", old_indices.len()),
5500 actual: format!("{} indices", new_indices.len()),
5501 });
5502 }
5503
5504 for (old, new) in old_indices.iter().zip(new_indices.iter()) {
5506 if old.dim() != new.dim() {
5507 return Err(IdxTensorError::ShapeMismatch {
5508 operation: "replace_indices",
5509 expected: format!("dimension {}", old.dim()),
5510 actual: format!("dimension {}", new.dim()),
5511 });
5512 }
5513 }
5514
5515 let replacement_map: std::collections::HashMap<_, _> =
5517 old_indices.iter().zip(new_indices.iter()).collect();
5518
5519 let new_indices_vec: Vec<_> = self
5520 .indices
5521 .iter()
5522 .map(|idx| {
5523 if let Some(new_idx) = replacement_map.get(idx) {
5524 (*new_idx).clone()
5525 } else {
5526 idx.clone()
5527 }
5528 })
5529 .collect();
5530
5531 Ok(Self {
5532 indices: new_indices_vec,
5533 storage: self.storage.clone(),
5534 eager_cache: Arc::clone(&self.eager_cache),
5535 })
5536 }
5537}
5538
5539impl IdxTensor {
5544 pub fn conj(&self) -> Self {
5573 self.conj_with(&conjugate_eager)
5574 }
5575
5576 fn conj_with<F>(&self, conjugate: &F) -> Self
5577 where
5578 F: Fn(
5579 &EagerTensor,
5580 ) -> std::result::Result<
5581 EagerTensor,
5582 Arc<dyn std::error::Error + Send + Sync + 'static>,
5583 >,
5584 {
5585 let new_indices: Vec<DynIndex> = self.indices.iter().map(|idx| idx.conj()).collect();
5589 let mut storage = match self.storage.conjugate_with(conjugate) {
5590 Ok(storage) => storage,
5591 Err(error) => self.storage.clone().with_deferred_error(error),
5592 };
5593 let mut eager_cache = if storage.deferred_error().is_some() {
5594 Arc::clone(&self.eager_cache)
5595 } else {
5596 Self::empty_eager_cache()
5597 };
5598
5599 if storage.deferred_error().is_none() {
5600 if let Some(inner) = self.eager_cache.get() {
5601 match conjugate(inner.as_ref()) {
5602 Ok(conjugated) => eager_cache = Self::eager_cache_with(conjugated),
5603 Err(source) => {
5604 storage =
5605 storage.with_deferred_error(TensorStorageError::Conjugation { source });
5606 eager_cache = Arc::clone(&self.eager_cache);
5609 }
5610 }
5611 }
5612 }
5613
5614 Self {
5615 indices: new_indices,
5616 storage,
5617 eager_cache,
5618 }
5619 }
5620}
5621
5622pub(crate) fn lassq_norm<const SQUARED: bool>(
5629 values: impl IntoIterator<Item = Complex64>,
5630) -> std::option::Option<f64> {
5631 let mut norm = Lassq::default();
5632 for value in values {
5633 if value.re.is_nan() || value.im.is_nan() {
5634 return None;
5635 }
5636 norm.add_complex(value);
5637 }
5638 let value = if SQUARED {
5639 norm.norm_squared()
5640 } else {
5641 norm.norm()
5642 };
5643 (!value.is_nan()).then_some(value)
5644}
5645
5646#[derive(Debug, Default)]
5647struct Lassq {
5648 scale: f64,
5649 sumsq: f64,
5650 infinite: bool,
5651}
5652
5653impl Lassq {
5654 fn add_component(&mut self, value: f64) {
5655 let value = value.abs();
5656 if value == 0.0 {
5657 return;
5658 }
5659 if value.is_infinite() {
5660 self.infinite = true;
5661 return;
5662 }
5663 if self.scale < value {
5664 if self.scale == 0.0 {
5665 self.sumsq = 1.0;
5666 } else {
5667 let ratio = self.scale / value;
5668 self.sumsq = 1.0 + self.sumsq * ratio * ratio;
5669 }
5670 self.scale = value;
5671 } else {
5672 let ratio = value / self.scale;
5673 self.sumsq += ratio * ratio;
5674 }
5675 }
5676
5677 fn add_complex(&mut self, value: Complex64) {
5678 self.add_component(value.re);
5679 self.add_component(value.im);
5680 }
5681
5682 fn add_scaled(&mut self, scale: f64, coefficient: f64) {
5683 if scale == 0.0 || coefficient == 0.0 {
5684 return;
5685 }
5686 if self.scale < scale {
5687 if self.scale == 0.0 {
5688 self.sumsq = coefficient * coefficient;
5689 } else {
5690 let ratio = self.scale / scale;
5691 self.sumsq = coefficient * coefficient + self.sumsq * ratio * ratio;
5692 }
5693 self.scale = scale;
5694 } else {
5695 let ratio = scale / self.scale * coefficient;
5696 self.sumsq += ratio * ratio;
5697 }
5698 }
5699
5700 fn add_component_difference(&mut self, lhs: f64, rhs: f64) {
5701 if lhs == rhs {
5702 return;
5703 }
5704 let scale = lhs.abs().max(rhs.abs());
5705 if scale != 0.0 {
5706 self.add_scaled(scale, (lhs / scale - rhs / scale).abs());
5707 }
5708 }
5709
5710 fn add_complex_difference(&mut self, lhs: Complex64, rhs: Complex64) {
5711 self.add_component_difference(lhs.re, rhs.re);
5712 self.add_component_difference(lhs.im, rhs.im);
5713 }
5714
5715 fn is_zero(&self) -> bool {
5716 !self.infinite && self.scale == 0.0
5717 }
5718
5719 fn norm(&self) -> f64 {
5720 if self.infinite {
5721 f64::INFINITY
5722 } else if self.scale == 0.0 {
5723 0.0
5724 } else {
5725 self.scale * self.sumsq.sqrt()
5726 }
5727 }
5728
5729 fn norm_squared(&self) -> f64 {
5730 let norm = self.norm();
5731 norm * norm
5732 }
5733
5734 fn log_norm(&self) -> f64 {
5735 if self.infinite {
5736 f64::INFINITY
5737 } else if self.scale == 0.0 {
5738 f64::NEG_INFINITY
5739 } else {
5740 self.scale.ln() + 0.5 * self.sumsq.ln()
5741 }
5742 }
5743}
5744
5745impl IdxTensor {
5750 pub fn norm_squared(&self) -> std::result::Result<f64, IdxTensorError> {
5776 self.norm_impl::<true>()
5777 }
5778
5779 fn norm_impl<const SQUARED: bool>(&self) -> std::result::Result<f64, IdxTensorError> {
5780 let operation = if SQUARED { "norm_squared" } else { "norm" };
5781 let dtype = self
5782 .scalar_dtype()
5783 .map_err(IdxTensorError::scalar_extraction)?;
5784 if !matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64) {
5785 return Err(IdxTensorError::ScalarTypeMismatch {
5786 expected: "f32, f64, c32, or c64",
5787 actual: Self::dtype_name(dtype).to_string(),
5788 });
5789 }
5790 let flags = self
5791 .compact_nonfinite_flags()
5792 .map_err(IdxTensorError::materialization)?;
5793 if flags.has_nan {
5794 return Err(IdxTensorError::NaNInput { operation });
5795 }
5796
5797 let mut norm = Lassq::default();
5798 self.storage
5799 .for_each_payload_value(|value| norm.add_complex(value))
5800 .map_err(IdxTensorError::materialization)?;
5801 let value = if SQUARED {
5802 norm.norm_squared()
5803 } else {
5804 norm.norm()
5805 };
5806 if value.is_nan() {
5807 return Err(IdxTensorError::NaNInput { operation });
5808 }
5809 Ok(value)
5810 }
5811
5812 pub fn norm(&self) -> std::result::Result<f64, IdxTensorError> {
5833 self.norm_impl::<false>()
5834 }
5835
5836 pub fn maxabs(&self) -> std::result::Result<f64, IdxTensorError> {
5852 if let Some(error) = self.storage.deferred_error() {
5853 return Err(IdxTensorError::Storage {
5854 source: error.clone(),
5855 });
5856 }
5857 let dtype = self
5858 .storage
5859 .dtype()
5860 .ok_or_else(|| IdxTensorError::ScalarTypeMismatch {
5861 expected: "f32, f64, c32, or c64",
5862 actual: "unknown".to_string(),
5863 })?;
5864 if !matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64) {
5865 return Err(IdxTensorError::ScalarTypeMismatch {
5866 expected: "f32, f64, c32, or c64",
5867 actual: Self::dtype_name(dtype).to_string(),
5868 });
5869 }
5870 let flags = self
5871 .compact_nonfinite_flags()
5872 .map_err(IdxTensorError::materialization)?;
5873 if flags.has_nan {
5874 return Err(IdxTensorError::NaNInput {
5875 operation: "maxabs",
5876 });
5877 }
5878 let mut value = 0.0_f64;
5879 self.storage
5880 .for_each_payload_value(|scalar| {
5881 let magnitude = scalar.re.hypot(scalar.im);
5882 value = value.max(magnitude);
5883 })
5884 .map_err(IdxTensorError::materialization)?;
5885 Ok(value)
5886 }
5887
5888 pub fn min_nonzero_abs(&self) -> std::result::Result<Option<f64>, IdxTensorError> {
5915 if let Some(error) = self.storage.deferred_error() {
5916 return Err(IdxTensorError::Storage {
5917 source: error.clone(),
5918 });
5919 }
5920 let dtype = self
5921 .storage
5922 .dtype()
5923 .ok_or_else(|| IdxTensorError::ScalarTypeMismatch {
5924 expected: "f32, f64, c32, or c64",
5925 actual: "unknown".to_string(),
5926 })?;
5927 if !matches!(dtype, DType::F32 | DType::F64 | DType::C32 | DType::C64) {
5928 return Err(IdxTensorError::ScalarTypeMismatch {
5929 expected: "f32, f64, c32, or c64",
5930 actual: Self::dtype_name(dtype).to_string(),
5931 });
5932 }
5933 let flags = self
5934 .compact_nonfinite_flags()
5935 .map_err(IdxTensorError::materialization)?;
5936 if flags.has_nan {
5937 return Err(IdxTensorError::NaNInput {
5938 operation: "min_nonzero_abs",
5939 });
5940 }
5941 let mut value: Option<f64> = None;
5942 self.storage
5943 .for_each_payload_value(|scalar| {
5944 let largest_part = scalar.re.abs().max(scalar.im.abs());
5945 let magnitude = if scalar.re == 0.0 || scalar.im == 0.0 {
5946 largest_part
5947 } else {
5948 modulus_lower_bound(scalar.re.hypot(scalar.im)).max(largest_part)
5952 };
5953 if magnitude != 0.0 {
5954 value = Some(value.map_or(magnitude, |current| current.min(magnitude)));
5955 }
5956 })
5957 .map_err(IdxTensorError::materialization)?;
5958 Ok(value)
5959 }
5960
5961 fn native_complex_payload_value_at(
5962 native: &EagerTensor,
5963 payload_coords: &[usize],
5964 ) -> Result<Complex64> {
5965 if !(payload_coords.len() == native.shape().len()) {
5966 return Err(anyhow::anyhow!(
5967 "payload coordinate rank {} does not match payload rank {}",
5968 payload_coords.len(),
5969 native.shape().len()
5970 ));
5971 };
5972 for (&coordinate, &dim) in payload_coords.iter().zip(native.shape().iter()) {
5973 if coordinate >= dim {
5974 return Err(anyhow::anyhow!(
5975 "payload coordinate {coordinate} is out of bounds for dim {dim}"
5976 ));
5977 }
5978 }
5979 #[cfg(test)]
5980 note_eager_payload_read();
5981 let value = native.value()?;
5982 IdxTensor::view_complex_value_at(value.as_tensor_view(), payload_coords)
5983 }
5984
5985 fn view_complex_value_at(view: &TensorView<'_>, payload_coords: &[usize]) -> Result<Complex64> {
5986 match view {
5987 TensorView::F32(view) => view
5988 .get(payload_coords)
5989 .copied()
5990 .map(|value| Complex64::new(f64::from(value), 0.0))
5991 .ok_or_else(|| anyhow::anyhow!("failed to read f32 payload value")),
5992 TensorView::F64(view) => view
5993 .get(payload_coords)
5994 .copied()
5995 .map(|value| Complex64::new(value, 0.0))
5996 .ok_or_else(|| anyhow::anyhow!("failed to read f64 payload value")),
5997 TensorView::C32(view) => view
5998 .get(payload_coords)
5999 .copied()
6000 .map(|value| Complex64::new(f64::from(value.re), f64::from(value.im)))
6001 .ok_or_else(|| anyhow::anyhow!("failed to read c32 payload value")),
6002 TensorView::C64(view) => view
6003 .get(payload_coords)
6004 .copied()
6005 .ok_or_else(|| anyhow::anyhow!("failed to read c64 payload value")),
6006 view => Err(anyhow::anyhow!(
6007 "unsupported payload dtype {:?}",
6008 view.dtype()
6009 )),
6010 }
6011 }
6012
6013 fn native_sum_scalar(native: &EagerTensor) -> Result<AnyScalar> {
6014 let value = native.value()?;
6015 match native.dtype() {
6016 DType::F32 => Ok(AnyScalar::from_value(
6017 value.as_slice::<f32>()?.iter().copied().sum::<f32>(),
6018 )),
6019 DType::F64 => Ok(AnyScalar::from_value(
6020 value.as_slice::<f64>()?.iter().copied().sum::<f64>(),
6021 )),
6022 DType::C32 => Ok(AnyScalar::from_value(
6023 value
6024 .as_slice::<Complex32>()?
6025 .iter()
6026 .copied()
6027 .sum::<Complex32>(),
6028 )),
6029 DType::C64 => Ok(AnyScalar::from_value(
6030 value
6031 .as_slice::<Complex64>()?
6032 .iter()
6033 .copied()
6034 .sum::<Complex64>(),
6035 )),
6036 dtype => Err(anyhow::anyhow!("unsupported dtype {dtype:?}")),
6037 }
6038 }
6039
6040 fn native_nonfinite_flags_typed<T: TensorElement>(
6041 native: &EagerTensor,
6042 classify: impl Fn(T) -> NonfiniteFlags,
6043 ) -> Result<NonfiniteFlags> {
6044 let fold = |values: &[T]| {
6045 values
6046 .iter()
6047 .copied()
6048 .map(&classify)
6049 .fold(NonfiniteFlags::default(), |flags, value| NonfiniteFlags {
6050 has_nan: flags.has_nan | value.has_nan,
6051 has_infinity: flags.has_infinity | value.has_infinity,
6052 })
6053 };
6054 let value = native.value()?;
6055 if let Ok(values) = value.as_slice::<T>() {
6056 return Ok(fold(values));
6057 }
6058 drop(value);
6059 let value = IdxTensorStorage::materialize_eager_payload(native)?;
6060 Ok(fold(value.as_slice::<T>()?))
6061 }
6062
6063 fn native_nonfinite_flags(native: &EagerTensor) -> Result<NonfiniteFlags> {
6064 match native.dtype() {
6065 DType::F32 => Self::native_nonfinite_flags_typed(native, |value: f32| NonfiniteFlags {
6066 has_nan: value.is_nan(),
6067 has_infinity: value.is_infinite(),
6068 }),
6069 DType::F64 => Self::native_nonfinite_flags_typed(native, |value: f64| NonfiniteFlags {
6070 has_nan: value.is_nan(),
6071 has_infinity: value.is_infinite(),
6072 }),
6073 DType::C32 => {
6074 Self::native_nonfinite_flags_typed(native, |value: Complex32| NonfiniteFlags {
6075 has_nan: value.re.is_nan() || value.im.is_nan(),
6076 has_infinity: value.re.is_infinite() || value.im.is_infinite(),
6077 })
6078 }
6079 DType::C64 => {
6080 Self::native_nonfinite_flags_typed(native, |value: Complex64| NonfiniteFlags {
6081 has_nan: value.re.is_nan() || value.im.is_nan(),
6082 has_infinity: value.re.is_infinite() || value.im.is_infinite(),
6083 })
6084 }
6085 dtype => Err(anyhow::anyhow!("unsupported dtype {dtype:?}")),
6086 }
6087 }
6088
6089 fn compact_nonfinite_flags(&self) -> Result<NonfiniteFlags> {
6090 self.storage.nonfinite_flags()
6091 }
6092
6093 pub fn sub(&self, other: &Self) -> std::result::Result<Self, IdxTensorError> {
6103 self.axpby(AnyScalar::new_real(1.0), other, AnyScalar::new_real(-1.0))
6104 }
6105
6106 pub fn neg(&self) -> std::result::Result<Self, IdxTensorError> {
6113 self.scale(AnyScalar::new_real(-1.0))
6114 }
6115
6116 pub fn isapprox(
6128 &self,
6129 other: &Self,
6130 atol: f64,
6131 rtol: f64,
6132 ) -> std::result::Result<bool, IdxTensorError> {
6133 for (name, value) in [("atol", atol), ("rtol", rtol)] {
6134 if !value.is_finite() || value < 0.0 {
6135 return Err(IdxTensorError::InvalidTolerance { name, value });
6136 }
6137 }
6138 if self.indices.len() != other.indices.len() {
6139 return Err(IdxTensorError::ShapeMismatch {
6140 operation: "isapprox",
6141 expected: format!("indices {:?}", self.indices),
6142 actual: format!("indices {:?}", other.indices),
6143 });
6144 }
6145
6146 let other_axis_by_index = other
6147 .indices
6148 .iter()
6149 .cloned()
6150 .enumerate()
6151 .map(|(axis, index)| (index, axis))
6152 .collect::<HashMap<_, _>>();
6153 let other_positions = self
6154 .indices
6155 .iter()
6156 .map(|index| {
6157 other_axis_by_index.get(index).copied().ok_or_else(|| {
6158 IdxTensorError::ShapeMismatch {
6159 operation: "isapprox",
6160 expected: format!("indices {:?}", self.indices),
6161 actual: format!("indices {:?}", other.indices),
6162 }
6163 })
6164 })
6165 .collect::<std::result::Result<Vec<_>, _>>()?;
6166 let self_dims = self.dims();
6167 let other_dims = other.dims();
6168 for (axis, &other_axis) in other_positions.iter().enumerate() {
6169 if self_dims[axis] != other_dims[other_axis] {
6170 return Err(IdxTensorError::ShapeMismatch {
6171 operation: "isapprox",
6172 expected: format!("dims {:?}", self_dims),
6173 actual: format!("dims {:?}", other_dims),
6174 });
6175 }
6176 }
6177 for tensor in [self, other] {
6178 if tensor
6179 .compact_nonfinite_flags()
6180 .map_err(IdxTensorError::materialization)?
6181 .has_nan
6182 {
6183 return Err(IdxTensorError::NaNInput {
6184 operation: "isapprox",
6185 });
6186 }
6187 }
6188
6189 let exact = atol == 0.0 && rtol == 0.0;
6190 let lhs_payload_dims = self.storage.payload_dims().to_vec();
6191 let rhs_payload_dims = other.storage.payload_dims().to_vec();
6192 let lhs_payload_len =
6193 checked_product(&lhs_payload_dims).map_err(IdxTensorError::materialization)?;
6194 let rhs_payload_len =
6195 checked_product(&rhs_payload_dims).map_err(IdxTensorError::materialization)?;
6196 let lhs_axis_classes = self.storage.axis_classes();
6197 let rhs_axis_classes = other.storage.axis_classes();
6198 let self_to_other = other_positions.clone();
6199 let mut other_to_self = vec![0usize; self_to_other.len()];
6200 for (self_axis, &other_axis) in self_to_other.iter().enumerate() {
6201 other_to_self[other_axis] = self_axis;
6202 }
6203
6204 let mut diff = Lassq::default();
6205 let mut lhs_norm = Lassq::default();
6206 let mut rhs_norm = Lassq::default();
6207 let mut compare = |lhs: Complex64, rhs: Complex64| -> bool {
6208 if exact {
6209 return lhs == rhs;
6210 }
6211 let lhs_infinite = lhs.re.is_infinite() || lhs.im.is_infinite();
6212 let rhs_infinite = rhs.re.is_infinite() || rhs.im.is_infinite();
6213 if lhs_infinite || rhs_infinite {
6214 return lhs == rhs;
6215 }
6216 lhs_norm.add_complex(lhs);
6217 rhs_norm.add_complex(rhs);
6218 diff.add_complex_difference(lhs, rhs);
6219 true
6220 };
6221
6222 let mut lhs_coords = vec![0usize; lhs_payload_dims.len()];
6226 let mut rhs_from_lhs = vec![0usize; rhs_payload_dims.len()];
6227 let mut rhs_seen = vec![false; rhs_payload_dims.len()];
6228 for _ in 0..lhs_payload_len {
6229 let lhs = self
6230 .storage
6231 .payload_value_at(&lhs_coords)
6232 .map_err(IdxTensorError::materialization)?;
6233 if map_payload_support_coordinate(
6234 lhs_axis_classes,
6235 rhs_axis_classes,
6236 &self_to_other,
6237 &lhs_coords,
6238 &mut rhs_from_lhs,
6239 &mut rhs_seen,
6240 ) {
6241 let rhs = other
6242 .storage
6243 .payload_value_at(&rhs_from_lhs)
6244 .map_err(IdxTensorError::materialization)?;
6245 if !compare(lhs, rhs) {
6246 return Ok(false);
6247 }
6248 } else if !compare(lhs, Complex64::new(0.0, 0.0)) {
6249 return Ok(false);
6250 }
6251
6252 increment_col_major_coordinate(&mut lhs_coords, &lhs_payload_dims);
6253 }
6254
6255 let mut rhs_coords = vec![0usize; rhs_payload_dims.len()];
6259 let mut lhs_from_rhs = vec![0usize; lhs_payload_dims.len()];
6260 let mut lhs_seen = vec![false; lhs_payload_dims.len()];
6261 for _ in 0..rhs_payload_len {
6262 let rhs = other
6263 .storage
6264 .payload_value_at(&rhs_coords)
6265 .map_err(IdxTensorError::materialization)?;
6266 if !map_payload_support_coordinate(
6267 rhs_axis_classes,
6268 lhs_axis_classes,
6269 &other_to_self,
6270 &rhs_coords,
6271 &mut lhs_from_rhs,
6272 &mut lhs_seen,
6273 ) && !compare(Complex64::new(0.0, 0.0), rhs)
6274 {
6275 return Ok(false);
6276 }
6277 increment_col_major_coordinate(&mut rhs_coords, &rhs_payload_dims);
6278 }
6279
6280 if exact {
6281 return Ok(true);
6282 }
6283 let absolute_ok = if diff.is_zero() {
6284 true
6285 } else if atol == 0.0 || diff.infinite {
6286 false
6287 } else {
6288 diff.log_norm() <= atol.ln()
6289 };
6290 let relative_ok = if rtol == 0.0 || diff.is_zero() {
6291 diff.is_zero()
6292 } else if diff.infinite {
6293 lhs_norm.infinite || rhs_norm.infinite
6294 } else if lhs_norm.infinite || rhs_norm.infinite {
6295 true
6296 } else {
6297 let reference_log = lhs_norm.log_norm().max(rhs_norm.log_norm());
6298 diff.log_norm() <= rtol.ln() + reference_log
6299 };
6300 Ok(absolute_ok || relative_ok)
6301 }
6302
6303 pub fn diagonal(
6310 input_index: &DynIndex,
6311 output_index: &DynIndex,
6312 ) -> std::result::Result<Self, IdxTensorError> {
6313 <Self as TensorConstructionLike>::diagonal(input_index, output_index)
6314 }
6315
6316 pub fn delta(
6322 input_indices: &[DynIndex],
6323 output_indices: &[DynIndex],
6324 ) -> std::result::Result<Self, IdxTensorError> {
6325 <Self as TensorConstructionLike>::delta(input_indices, output_indices)
6326 }
6327
6328 pub fn scalar_one() -> std::result::Result<Self, IdxTensorError> {
6335 <Self as TensorConstructionLike>::scalar_one()
6336 }
6337
6338 pub fn ones(indices: &[DynIndex]) -> std::result::Result<Self, IdxTensorError> {
6345 <Self as TensorConstructionLike>::ones(indices)
6346 }
6347
6348 pub fn onehot(index_vals: &[(DynIndex, usize)]) -> std::result::Result<Self, IdxTensorError> {
6355 <Self as TensorConstructionLike>::onehot(index_vals)
6356 }
6357
6358 pub fn mask_index(
6397 &self,
6398 index: &DynIndex,
6399 position: usize,
6400 ) -> std::result::Result<Self, IdxTensorError> {
6401 if !(self.indices.iter().any(|candidate| candidate == index)) {
6402 return Err(anyhow::anyhow!("mask_index: index is not present in tensor").into());
6403 };
6404 if !(position < index.dim()) {
6405 return Err(anyhow::anyhow!(
6406 "mask_index: position {position} is out of range for dimension {}",
6407 index.dim()
6408 )
6409 .into());
6410 };
6411
6412 let mask = match self.scalar_dtype()? {
6417 DType::F32 => Self::from_dense(
6418 vec![index.clone()],
6419 (0..index.dim())
6420 .map(|value| if value == position { 1.0_f32 } else { 0.0 })
6421 .collect(),
6422 ),
6423 DType::F64 => Self::from_dense(
6424 vec![index.clone()],
6425 (0..index.dim())
6426 .map(|value| if value == position { 1.0_f64 } else { 0.0 })
6427 .collect(),
6428 ),
6429 DType::C32 => Self::from_dense(
6430 vec![index.clone()],
6431 (0..index.dim())
6432 .map(|value| {
6433 if value == position {
6434 num_complex::Complex32::new(1.0, 0.0)
6435 } else {
6436 num_complex::Complex32::new(0.0, 0.0)
6437 }
6438 })
6439 .collect(),
6440 ),
6441 DType::C64 => Self::from_dense(
6442 vec![index.clone()],
6443 (0..index.dim())
6444 .map(|value| {
6445 if value == position {
6446 Complex64::new(1.0, 0.0)
6447 } else {
6448 Complex64::new(0.0, 0.0)
6449 }
6450 })
6451 .collect(),
6452 ),
6453 dtype => {
6454 return Err(anyhow::anyhow!("mask_index does not support dtype {dtype:?}").into())
6455 }
6456 }?;
6457 super::contract::contract_pair_with_options(
6458 self,
6459 &mask,
6460 super::contract::ContractionOptions::new()
6461 .with_retain_indices(std::slice::from_ref(index)),
6462 )
6463 }
6464
6465 pub fn distance(&self, other: &Self) -> std::result::Result<f64, IdxTensorError> {
6500 let norm_self = self.norm()?;
6501
6502 let neg_other = other.scale(AnyScalar::new_real(-1.0))?;
6504 let diff = self.add(&neg_other)?;
6505 let norm_diff = diff.norm()?;
6506
6507 if norm_self > 0.0 {
6508 Ok(norm_diff / norm_self)
6509 } else {
6510 Ok(norm_diff)
6511 }
6512 }
6513}
6514
6515impl std::fmt::Debug for IdxTensor {
6516 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
6517 f.debug_struct("IdxTensor")
6518 .field("indices", &self.indices)
6519 .field("dims", &self.dims())
6520 .field("is_diag", &self.is_diag())
6521 .finish()
6522 }
6523}
6524
6525pub fn diag_idx_tensor(
6549 indices: Vec<DynIndex>,
6550 diag_data: Vec<f64>,
6551) -> std::result::Result<IdxTensor, IdxTensorError> {
6552 IdxTensor::from_diag(indices, diag_data)
6553}
6554
6555#[allow(clippy::type_complexity)]
6556pub(crate) type UnfoldSplitInnerResult = (
6557 EagerTensor,
6558 usize,
6559 usize,
6560 usize,
6561 Vec<DynIndex>,
6562 Vec<DynIndex>,
6563);
6564
6565#[allow(clippy::type_complexity)]
6603pub fn unfold_split(
6604 t: &IdxTensor,
6605 left_inds: &[DynIndex],
6606) -> std::result::Result<
6607 (
6608 NativeTensor,
6609 usize,
6610 usize,
6611 usize,
6612 Vec<DynIndex>,
6613 Vec<DynIndex>,
6614 ),
6615 IdxTensorError,
6616> {
6617 let (matrix_inner, left_len, m, n, left_indices, right_indices) =
6618 unfold_split_inner(t, left_inds)?;
6619
6620 Ok((
6621 with_engine(matrix_inner.runtime(), None, |engine| {
6622 matrix_inner.duplicate_value(engine)
6623 })?,
6624 left_len,
6625 m,
6626 n,
6627 left_indices,
6628 right_indices,
6629 ))
6630}
6631
6632pub(crate) fn unfold_index_sets(
6638 t: &IdxTensor,
6639 left_inds: &[DynIndex],
6640) -> Result<(Vec<DynIndex>, Vec<DynIndex>, Vec<DynIndex>)> {
6641 let rank = t.indices.len();
6642
6643 if !(rank >= 2) {
6645 return Err(anyhow::anyhow!(
6646 "Tensor must have rank >= 2, got rank {}",
6647 rank
6648 ));
6649 };
6650
6651 let left_len = left_inds.len();
6652
6653 if !(left_len > 0 && left_len < rank) {
6655 return Err(anyhow::anyhow!("Left indices must be a non-empty proper subset of tensor indices (0 < left_len < rank), got left_len={}, rank={}",
6656 left_len,
6657 rank));
6658 };
6659
6660 let tensor_set: HashSet<_> = t.indices.iter().collect();
6662 let mut left_set = HashSet::new();
6663
6664 for left_idx in left_inds {
6665 if !(tensor_set.contains(left_idx)) {
6666 return Err(anyhow::anyhow!("Index in left_inds not found in tensor"));
6667 };
6668 if !(left_set.insert(left_idx)) {
6669 return Err(anyhow::anyhow!("Duplicate index in left_inds"));
6670 };
6671 }
6672
6673 let mut right_inds = Vec::new();
6675 for idx in &t.indices {
6676 if !left_set.contains(idx) {
6677 right_inds.push(idx.clone());
6678 }
6679 }
6680
6681 let mut new_indices = Vec::with_capacity(rank);
6683 new_indices.extend_from_slice(left_inds);
6684 new_indices.extend_from_slice(&right_inds);
6685
6686 Ok((left_inds.to_vec(), right_inds, new_indices))
6687}
6688
6689pub(crate) fn unfold_split_inner(
6690 t: &IdxTensor,
6691 left_inds: &[DynIndex],
6692) -> Result<UnfoldSplitInnerResult> {
6693 unfold_split_inner_scoped(t, left_inds, None)
6694}
6695
6696pub(crate) fn unfold_split_inner_in(
6702 t: &IdxTensor,
6703 left_inds: &[DynIndex],
6704 context: &ExecutionContext,
6705) -> Result<UnfoldSplitInnerResult> {
6706 unfold_split_inner_scoped(t, left_inds, Some(context))
6707}
6708
6709fn unfold_split_inner_scoped(
6710 t: &IdxTensor,
6711 left_inds: &[DynIndex],
6712 context: Option<&ExecutionContext>,
6713) -> Result<UnfoldSplitInnerResult> {
6714 let (left_indices, right_indices, new_indices) = unfold_index_sets(t, left_inds)?;
6715 let left_len = left_indices.len();
6716
6717 let adopted;
6720 let tensor = match context {
6721 Some(context) if t.eager_runtime().is_none() => {
6722 adopted = IdxTensor::adopt_host_into_context(t, context)?;
6723 &adopted
6724 }
6725 _ => t,
6726 };
6727
6728 let unfolded = match context {
6730 Some(context) => {
6731 let perm = compute_permutation_from_indices(&tensor.indices, &new_indices)?;
6732 if perm.iter().copied().eq(0..perm.len()) {
6733 IdxTensor {
6734 indices: new_indices,
6735 storage: tensor.storage.clone(),
6736 eager_cache: Arc::clone(&tensor.eager_cache),
6737 }
6738 } else {
6739 let inner = tensor.try_materialized_inner()?;
6740 let permuted = with_eager_session(inner.runtime(), Some(context), |session| {
6741 session.transpose(inner, &perm)
6742 })?;
6743 let axis_classes = tensor.permute_axis_classes(&perm);
6744 IdxTensor::from_inner_with_axis_classes(new_indices, permuted, axis_classes)?
6745 }
6746 }
6747 None => tensor.permute_indices(&new_indices)?,
6748 };
6749
6750 let unfolded_dims = unfolded.dims();
6752 let m = checked_product(&unfolded_dims[..left_len])?;
6753 let n = checked_product(&unfolded_dims[left_len..])?;
6754
6755 let unfolded_inner = unfolded.try_materialized_inner()?;
6756 let matrix_tensor = with_eager_session(unfolded_inner.runtime(), context, |session| {
6757 session.reshape(unfolded_inner, [m, n])
6758 })?;
6759
6760 Ok((matrix_tensor, left_len, m, n, left_indices, right_indices))
6761}
6762
6763use crate::tensor_index::TensorIndex;
6768
6769impl TensorIndex for IdxTensor {
6770 type Index = DynIndex;
6771 type Error = IdxTensorError;
6772
6773 fn external_indices(&self) -> Vec<DynIndex> {
6774 self.indices.clone()
6776 }
6777
6778 fn num_external_indices(&self) -> usize {
6779 self.indices.len()
6780 }
6781
6782 fn replaceind(
6783 &self,
6784 old_index: &DynIndex,
6785 new_index: &DynIndex,
6786 ) -> std::result::Result<Self, Self::Error> {
6787 IdxTensor::replaceind(self, old_index, new_index)
6789 }
6790
6791 fn replace_indices(
6792 &self,
6793 old_indices: &[DynIndex],
6794 new_indices: &[DynIndex],
6795 ) -> std::result::Result<Self, Self::Error> {
6796 IdxTensor::replace_indices(self, old_indices, new_indices)
6798 }
6799}
6800
6801use crate::tensor_like::{
6806 FactorizeError, FactorizeOptions, FactorizeResult, IncrementalQrState, TensorConstructionLike,
6807 TensorContractionLike, TensorFactorizationLike, TensorVectorSpace,
6808};
6809
6810fn modulus_lower_bound(hypot: f64) -> f64 {
6815 if !hypot.is_finite() || hypot <= 0.0 {
6816 return hypot;
6817 }
6818 let below = |x: f64| {
6819 if x > 0.0 {
6820 f64::from_bits(x.to_bits() - 1)
6821 } else {
6822 x
6823 }
6824 };
6825 below(below(hypot))
6826}
6827
6828impl TensorVectorSpace for IdxTensor {
6829 fn norm_squared(&self) -> std::result::Result<f64, Self::Error> {
6830 IdxTensor::norm_squared(self)
6831 }
6832
6833 fn is_single_precision(&self) -> bool {
6834 self.is_f32() || self.is_c32()
6835 }
6836
6837 fn is_complex(&self) -> bool {
6838 IdxTensor::is_complex(self)
6839 }
6840
6841 fn norm(&self) -> std::result::Result<f64, Self::Error> {
6842 IdxTensor::norm(self)
6843 }
6844
6845 fn maxabs(&self) -> std::result::Result<f64, Self::Error> {
6846 IdxTensor::maxabs(self)
6847 }
6848
6849 fn min_nonzero_abs(&self) -> std::result::Result<Option<f64>, Self::Error> {
6850 IdxTensor::min_nonzero_abs(self)
6851 }
6852
6853 fn is_all_finite(&self) -> std::result::Result<bool, Self::Error> {
6854 if let Some(error) = self.storage.deferred_error() {
6855 return Err(IdxTensorError::Storage {
6856 source: error.clone(),
6857 });
6858 }
6859 let flags = self
6860 .compact_nonfinite_flags()
6861 .map_err(IdxTensorError::materialization)?;
6862 Ok(!(flags.has_nan || flags.has_infinity))
6863 }
6864
6865 fn isapprox(
6866 &self,
6867 other: &Self,
6868 atol: f64,
6869 rtol: f64,
6870 ) -> std::result::Result<bool, Self::Error> {
6871 IdxTensor::isapprox(self, other, atol, rtol)
6872 }
6873
6874 fn axpby(
6875 &self,
6876 a: crate::AnyScalar,
6877 other: &Self,
6878 b: crate::AnyScalar,
6879 ) -> std::result::Result<Self, Self::Error> {
6880 IdxTensor::axpby(self, a, other, b)
6881 }
6882
6883 fn scale(&self, scalar: crate::AnyScalar) -> std::result::Result<Self, Self::Error> {
6884 IdxTensor::scale(self, scalar)
6885 }
6886
6887 fn scale_in(
6888 &self,
6889 factor: f64,
6890 context: &ExecutionContext,
6891 ) -> std::result::Result<Self, Self::Error> {
6892 IdxTensor::scale_in(self, factor, context)
6893 }
6894
6895 fn norm_in(&self, context: &ExecutionContext) -> std::result::Result<f64, Self::Error> {
6896 IdxTensor::norm_in(self, context)
6897 }
6898
6899 fn inner_product(&self, other: &Self) -> std::result::Result<crate::AnyScalar, Self::Error> {
6900 IdxTensor::inner_product(self, other)
6901 }
6902}
6903
6904impl TensorFactorizationLike for IdxTensor {
6905 fn factorize(
6906 &self,
6907 left_inds: &[DynIndex],
6908 options: &FactorizeOptions,
6909 ) -> std::result::Result<FactorizeResult<Self>, FactorizeError> {
6910 crate::factorize::factorize(self, left_inds, options)
6911 }
6912
6913 fn factorize_with_rng(
6914 &self,
6915 left_inds: &[DynIndex],
6916 options: &FactorizeOptions,
6917 rng: &mut dyn rand::RngCore,
6918 ) -> std::result::Result<FactorizeResult<Self>, FactorizeError> {
6919 crate::factorize::factorize_with_rng(self, left_inds, options, rng)
6920 }
6921
6922 fn factorize_in(
6923 &self,
6924 left_inds: &[DynIndex],
6925 options: &FactorizeOptions,
6926 context: &ExecutionContext,
6927 ) -> std::result::Result<FactorizeResult<Self>, FactorizeError> {
6928 crate::factorize::factorize_in(self, left_inds, options, context)
6929 }
6930
6931 fn factorize_auto(
6932 &self,
6933 left_inds: &[DynIndex],
6934 options: &FactorizeOptions,
6935 ) -> std::result::Result<FactorizeResult<Self>, FactorizeError> {
6936 crate::factorize::factorize_auto(self, left_inds, options)
6937 }
6938
6939 fn factorize_full_rank(
6940 &self,
6941 left_inds: &[DynIndex],
6942 alg: crate::FactorizeAlg,
6943 canonical: crate::Canonical,
6944 ) -> std::result::Result<FactorizeResult<Self>, FactorizeError> {
6945 crate::factorize::factorize_full_rank(self, left_inds, alg, canonical)
6946 }
6947
6948 fn factorize_full_rank_in(
6949 &self,
6950 left_inds: &[DynIndex],
6951 alg: crate::FactorizeAlg,
6952 canonical: crate::Canonical,
6953 context: &ExecutionContext,
6954 ) -> std::result::Result<FactorizeResult<Self>, FactorizeError> {
6955 crate::factorize::factorize_full_rank_in(self, left_inds, alg, canonical, context)
6956 }
6957
6958 fn factorize_probe_columns_incremental(
6959 previous: Option<&FactorizeResult<Self>>,
6960 all_columns: &[&Self],
6961 appended_columns: &[&Self],
6962 left_inds: &[DynIndex],
6963 ) -> std::result::Result<FactorizeResult<Self>, FactorizeError> {
6964 factorize_probe_columns_incremental(previous, all_columns, appended_columns, left_inds)
6965 }
6966
6967 fn factorize_probe_batch_incremental(
6968 previous: Option<&FactorizeResult<IdxTensor>>,
6969 batch_tensor: &IdxTensor,
6970 batch_index: &DynIndex,
6971 left_inds: &[DynIndex],
6972 ) -> std::result::Result<FactorizeResult<IdxTensor>, FactorizeError> {
6973 factorize_probe_batch_incremental_impl(previous, batch_tensor, batch_index, left_inds)
6974 }
6975
6976 fn factorize_probe_batch_incremental_in(
6977 previous: Option<&FactorizeResult<IdxTensor>>,
6978 batch_tensor: &IdxTensor,
6979 batch_index: &DynIndex,
6980 left_inds: &[DynIndex],
6981 context: &ExecutionContext,
6982 ) -> std::result::Result<FactorizeResult<IdxTensor>, FactorizeError> {
6983 if let Some(previous) = previous {
6984 previous
6985 .left
6986 .validate_context(context)
6987 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
6988 previous
6989 .right
6990 .validate_context(context)
6991 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
6992 }
6993 batch_tensor
6994 .validate_context(context)
6995 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
6996 if context.is_global_default_cpu() {
7001 return factorize_probe_batch_incremental_impl(
7002 previous,
7003 batch_tensor,
7004 batch_index,
7005 left_inds,
7006 );
7007 }
7008 Self::resident_probe_batch_qr(previous, batch_tensor, batch_index, left_inds, context)
7009 }
7010
7011 fn src_error_estimate(
7012 &self,
7013 ) -> std::result::Result<tensor4all_tensorbackend::SrcErrorEstimate, FactorizeError> {
7014 if self.is_cuda_resident() {
7015 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7016 "CUDA-resident QR factor requires src_error_estimate_in with the owning execution context"
7017 )));
7018 }
7019 let indices = self.indices();
7020 if indices.len() != 2 {
7021 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7022 "SRC QR factor must have rank 2, got {}",
7023 indices.len()
7024 )));
7025 }
7026 let nrows = indices[0].dim();
7027 let ncols = indices[1].dim();
7028 let result = if self.is_f64() {
7029 let matrix = Matrix::from_col_major_vec(
7030 nrows,
7031 ncols,
7032 self.to_vec::<f64>()
7033 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7034 );
7035 backend_src_error_estimate(&matrix)
7036 } else if self.is_c64() {
7037 let matrix = Matrix::from_col_major_vec(
7038 nrows,
7039 ncols,
7040 self.to_vec::<Complex64>()
7041 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7042 );
7043 backend_src_error_estimate(&matrix)
7044 } else {
7045 return Err(FactorizeError::UnsupportedStorage(
7046 "SRC adaptive estimation currently supports f64 and Complex64 QR factors",
7047 ));
7048 };
7049 result.map_err(|error| FactorizeError::ComputationError(anyhow::anyhow!(error)))
7050 }
7051
7052 fn src_error_estimate_in(
7053 &self,
7054 context: &ExecutionContext,
7055 ) -> std::result::Result<tensor4all_tensorbackend::SrcErrorEstimate, FactorizeError> {
7056 self.validate_context(context)
7057 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7058 #[cfg(feature = "tenferro-cuda")]
7059 if matches!(context, ExecutionContext::Cuda(_)) {
7060 return self.resident_src_error_estimate(context);
7061 }
7062 if context.is_global_default_cpu() {
7063 return TensorFactorizationLike::src_error_estimate(self);
7064 }
7065 self.host_src_error_estimate_general()
7066 }
7067}
7068
7069impl IdxTensor {
7070 fn host_src_error_estimate_general(
7072 &self,
7073 ) -> std::result::Result<tensor4all_tensorbackend::SrcErrorEstimate, FactorizeError> {
7074 let indices = self.indices();
7075 if indices.len() != 2 {
7076 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7077 "SRC QR factor must have rank 2, got {}",
7078 indices.len()
7079 )));
7080 }
7081 let nrows = indices[0].dim();
7082 let ncols = indices[1].dim();
7083 let result = if self.is_f64() {
7084 let matrix = Matrix::from_col_major_vec(
7085 nrows,
7086 ncols,
7087 self.to_vec::<f64>()
7088 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7089 );
7090 backend_src_error_estimate_general(&matrix)
7091 } else if self.is_c64() {
7092 let matrix = Matrix::from_col_major_vec(
7093 nrows,
7094 ncols,
7095 self.to_vec::<Complex64>()
7096 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7097 );
7098 backend_src_error_estimate_general(&matrix)
7099 } else {
7100 return Err(FactorizeError::UnsupportedStorage(
7101 "SRC adaptive estimation currently supports f64 and Complex64 QR factors",
7102 ));
7103 };
7104 result.map_err(|error| FactorizeError::ComputationError(anyhow::anyhow!(error)))
7105 }
7106
7107 fn resident_probe_batch_qr(
7118 previous: Option<&FactorizeResult<IdxTensor>>,
7119 batch_tensor: &IdxTensor,
7120 batch_index: &DynIndex,
7121 left_inds: &[DynIndex],
7122 context: &ExecutionContext,
7123 ) -> std::result::Result<FactorizeResult<IdxTensor>, FactorizeError> {
7124 if !batch_tensor.indices().contains(batch_index) {
7128 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7129 "resident probe batch is missing its batch index"
7130 )));
7131 }
7132 let (appended_inner, _, m, b, _, _) =
7133 unfold_split_inner_in(batch_tensor, left_inds, context)
7134 .context("resident probe batch unfold failed")
7135 .map_err(FactorizeError::ComputationError)?;
7136 if !matches!(appended_inner.dtype(), DType::F64 | DType::C64) {
7137 return Err(FactorizeError::UnsupportedStorage(
7138 "incremental SRC factorization currently supports f64 and Complex64 tensors",
7139 ));
7140 }
7141 if b == 0 {
7142 if let Some(previous) = previous {
7143 return Ok(previous.clone());
7144 }
7145 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7146 "incremental SRC factorization requires at least one column"
7147 )));
7148 }
7149 let full_inner = if let Some(previous) = previous {
7150 let (q_inner, _, previous_rows, _, _, _) =
7151 unfold_split_inner_in(&previous.left, left_inds, context)
7152 .context("resident previous Q unfold failed")
7153 .map_err(FactorizeError::ComputationError)?;
7154 if previous_rows != m {
7155 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7156 "previous and appended sketch row dimensions differ: {previous_rows} vs {m}"
7157 )));
7158 }
7159 let (r_inner, _, _, _, _, _) = unfold_split_inner_in(
7160 &previous.right,
7161 std::slice::from_ref(&previous.bond_index),
7162 context,
7163 )
7164 .context("resident previous R unfold failed")
7165 .map_err(FactorizeError::ComputationError)?;
7166 let reconstructed = with_eager_session(q_inner.runtime(), Some(context), |session| {
7167 session.matmul(&q_inner, &r_inner)
7168 })
7169 .map_err(|error| {
7170 FactorizeError::ComputationError(
7171 anyhow::Error::new(error).context("resident sketch reconstruction failed"),
7172 )
7173 })?;
7174 with_eager_session(reconstructed.runtime(), Some(context), |session| {
7175 session.concatenate(&[&reconstructed, &appended_inner], 1)
7176 })
7177 .map_err(|error| {
7178 FactorizeError::ComputationError(
7179 anyhow::Error::new(error).context("resident sketch concatenation failed"),
7180 )
7181 })?
7182 } else {
7183 appended_inner
7184 };
7185 let total_width = *full_inner.shape().get(1).ok_or_else(|| {
7186 FactorizeError::ComputationError(anyhow::anyhow!(
7187 "resident sketch factor is not rank-2"
7188 ))
7189 })?;
7190 let rank_tolerance = 32.0 * f64::EPSILON * m.max(total_width) as f64;
7191 let decomposition = with_eager_session(full_inner.runtime(), Some(context), |session| {
7192 session.rank_revealing_qr(
7193 &full_inner,
7194 RankRevealingQrOptions::default().rtol(rank_tolerance),
7195 )
7196 })
7197 .map_err(|error| {
7198 FactorizeError::ComputationError(
7199 anyhow::Error::new(error).context("resident probe batch RRQR failed"),
7200 )
7201 })?;
7202 let rank = Self::read_resident_rank(&decomposition.rank, context)?;
7203 let maximum_rank = m.min(total_width);
7204 if rank > maximum_rank {
7205 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7206 "resident RRQR returned rank {rank}, exceeding maximum {maximum_rank}"
7207 )));
7208 }
7209 if let Some(previous) = previous {
7210 if rank < previous.rank {
7211 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7212 "resident SRC RRQR rank decreased from {} to {rank}",
7213 previous.rank
7214 )));
7215 }
7216 }
7217 let q_full = with_eager_session(decomposition.q.runtime(), Some(context), |session| {
7218 session.slice_axis(&decomposition.q, 1, 0..rank)
7219 })
7220 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7221 let transposed = with_eager_session(q_full.runtime(), Some(context), |session| {
7225 session.transpose(&q_full, &[1, 0])
7226 })
7227 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7228 let q_adjoint = with_eager_session(transposed.runtime(), Some(context), |session| {
7229 session.conj(&transposed)
7230 })
7231 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7232 let r_full = with_eager_session(q_adjoint.runtime(), Some(context), |session| {
7233 session.matmul(&q_adjoint, &full_inner)
7234 })
7235 .map_err(|error| {
7236 FactorizeError::ComputationError(
7237 anyhow::Error::new(error).context("resident RRQR right-factor restoration failed"),
7238 )
7239 })?;
7240 let cap = DynIndex::new_bond(rank)
7241 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7242 let batch = DynIndex::new_link(total_width)
7243 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7244 let mut q_indices = left_inds.to_vec();
7245 q_indices.push(cap.clone());
7246 let q_dims: Vec<usize> = q_indices.iter().map(|index| index.dim()).collect();
7247 let left = Self::from_inner(
7248 q_indices,
7249 with_eager_session(q_full.runtime(), Some(context), |session| {
7250 session.reshape(&q_full, &q_dims)
7251 })
7252 .map_err(|error| {
7253 FactorizeError::ComputationError(
7254 anyhow::Error::new(error).context("resident probe batch Q reshape failed"),
7255 )
7256 })?,
7257 )
7258 .map_err(FactorizeError::ComputationError)?;
7259 let right = Self::from_inner(
7260 vec![cap.clone(), batch],
7261 with_eager_session(r_full.runtime(), Some(context), |session| {
7262 session.reshape(&r_full, [rank, total_width])
7263 })
7264 .map_err(|error| {
7265 FactorizeError::ComputationError(
7266 anyhow::Error::new(error).context("resident probe batch R reshape failed"),
7267 )
7268 })?,
7269 )
7270 .map_err(FactorizeError::ComputationError)?;
7271 Ok(FactorizeResult::new(left, right, cap, None, rank))
7272 }
7273
7274 #[cfg(feature = "tenferro-cuda")]
7281 fn resident_src_error_estimate(
7282 &self,
7283 context: &ExecutionContext,
7284 ) -> std::result::Result<tensor4all_tensorbackend::SrcErrorEstimate, FactorizeError> {
7285 let indices = self.indices();
7286 if indices.len() != 2 {
7287 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7288 "SRC QR factor must have rank 2, got {}",
7289 indices.len()
7290 )));
7291 }
7292 let nrows = indices[0].dim();
7293 let ncols = indices[1].dim();
7294 if nrows != ncols {
7295 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7296 "SRC estimator requires a square R, got {nrows}x{ncols}"
7297 )));
7298 }
7299 if nrows == 0 {
7300 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7301 "SRC estimator requires a non-empty R"
7302 )));
7303 }
7304 let inner = self.cuda_eager_inner().ok_or_else(|| {
7305 FactorizeError::ComputationError(anyhow::anyhow!(
7306 "SRC estimator requires a resident eager QR factor"
7307 ))
7308 })?;
7309 let ExecutionContext::Cuda(cuda) = context else {
7310 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7311 "SRC device estimator requires a CUDA execution context"
7312 )));
7313 };
7314 let native = with_engine(inner.runtime(), Some(context), |engine| {
7319 inner.to_tensor(engine)
7320 })
7321 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7322 let host_native = cuda
7323 .download(&native)
7324 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7325 let restored = Self::from_native(self.indices().to_vec(), host_native)
7326 .map_err(FactorizeError::ComputationError)?;
7327 restored.host_src_error_estimate_general()
7328 }
7329
7330 fn read_resident_rank(
7332 rank: &EagerTensor,
7333 context: &ExecutionContext,
7334 ) -> std::result::Result<usize, FactorizeError> {
7335 if !rank.shape().is_empty() {
7336 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7337 "resident RRQR rank metadata has shape {:?}, expected []",
7338 rank.shape()
7339 )));
7340 }
7341 #[cfg(not(feature = "tenferro-cuda"))]
7342 ensure_supported_execution_context(context)
7343 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7344 let resident = with_engine(rank.runtime(), Some(context), |engine| {
7345 rank.to_tensor(engine)
7346 })
7347 .map_err(|error| {
7348 FactorizeError::ComputationError(
7349 anyhow::Error::new(error).context("resident RRQR rank materialization failed"),
7350 )
7351 })?;
7352 #[cfg(feature = "tenferro-cuda")]
7353 let host = match context {
7354 ExecutionContext::Cpu(_) => resident,
7355 ExecutionContext::Cuda(cuda) => cuda.download(&resident).map_err(|error| {
7356 FactorizeError::ComputationError(
7357 anyhow::Error::new(error).context("resident RRQR rank download failed"),
7358 )
7359 })?,
7360 };
7361 #[cfg(not(feature = "tenferro-cuda"))]
7362 let host = match context {
7363 ExecutionContext::Cpu(_) => resident,
7364 #[allow(unreachable_patterns)]
7366 _ => return Err(unsupported_factorize_context_error()),
7367 };
7368 let values = host.as_slice::<i64>().map_err(|error| {
7369 FactorizeError::ComputationError(
7370 anyhow::Error::new(error).context("resident RRQR rank decode failed"),
7371 )
7372 })?;
7373 let value = *values.first().ok_or_else(|| {
7374 FactorizeError::ComputationError(anyhow::anyhow!(
7375 "resident RRQR rank metadata is empty"
7376 ))
7377 })?;
7378 usize::try_from(value).map_err(|error| {
7379 FactorizeError::ComputationError(
7380 anyhow::Error::new(error).context("resident RRQR rank is negative"),
7381 )
7382 })
7383 }
7384
7385 #[cfg(feature = "tenferro-cuda")]
7387 fn read_resident_vector(
7388 eager: &EagerTensor,
7389 len: usize,
7390 context: &ExecutionContext,
7391 ) -> std::result::Result<Vec<f64>, FactorizeError> {
7392 if eager.shape() != [len] {
7393 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7394 "device SRC decision vector has shape {:?}, expected [{len}]",
7395 eager.shape()
7396 )));
7397 }
7398 let decision = Self::from_inner(vec![DynIndex::new_dyn(len)], eager.clone())
7399 .map_err(FactorizeError::ComputationError)?;
7400 decision
7401 .read_decision_data(context)
7402 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))
7403 }
7404}
7405
7406fn factorize_probe_columns_incremental(
7411 previous: Option<&FactorizeResult<IdxTensor>>,
7412 all_columns: &[&IdxTensor],
7413 appended_columns: &[&IdxTensor],
7414 left_inds: &[DynIndex],
7415) -> std::result::Result<FactorizeResult<IdxTensor>, FactorizeError> {
7416 let first = all_columns.first().ok_or_else(|| {
7417 FactorizeError::ComputationError(anyhow::anyhow!(
7418 "incremental SRC factorization requires at least one column"
7419 ))
7420 })?;
7421 if first.is_f64() {
7422 incremental_probe_factorize_typed::<f64>(previous, all_columns, appended_columns, left_inds)
7423 } else if first.is_c64() {
7424 incremental_probe_factorize_typed::<Complex64>(
7425 previous,
7426 all_columns,
7427 appended_columns,
7428 left_inds,
7429 )
7430 } else {
7431 Err(FactorizeError::UnsupportedStorage(
7432 "incremental SRC factorization currently supports f64 and Complex64 tensors",
7433 ))
7434 }
7435}
7436
7437trait IncrementalQrStateScalar: IncrementalQrScalar + TensorElement {
7438 fn resume(previous: &FactorizeResult<IdxTensor>) -> Option<IncrementalQr<Self>>;
7439
7440 fn store(state: IncrementalQr<Self>) -> IncrementalQrState;
7441}
7442
7443impl IncrementalQrStateScalar for f64 {
7444 fn resume(previous: &FactorizeResult<IdxTensor>) -> Option<IncrementalQr<Self>> {
7445 match previous.incremental_qr_state()? {
7446 IncrementalQrState::F64(state) => Some(state.clone()),
7447 IncrementalQrState::C64(_) => None,
7448 }
7449 }
7450
7451 fn store(state: IncrementalQr<Self>) -> IncrementalQrState {
7452 IncrementalQrState::F64(state)
7453 }
7454}
7455
7456impl IncrementalQrStateScalar for Complex64 {
7457 fn resume(previous: &FactorizeResult<IdxTensor>) -> Option<IncrementalQr<Self>> {
7458 match previous.incremental_qr_state()? {
7459 IncrementalQrState::F64(_) => None,
7460 IncrementalQrState::C64(state) => Some(state.clone()),
7461 }
7462 }
7463
7464 fn store(state: IncrementalQr<Self>) -> IncrementalQrState {
7465 IncrementalQrState::C64(state)
7466 }
7467}
7468
7469fn incremental_probe_factorize_from_matrices<S>(
7470 previous: Option<&FactorizeResult<IdxTensor>>,
7471 appended_is_empty: bool,
7472 left_inds: &[DynIndex],
7473 initial_matrix: impl FnOnce() -> std::result::Result<Matrix<S>, FactorizeError>,
7474 appended_matrix: impl FnOnce() -> std::result::Result<Matrix<S>, FactorizeError>,
7475) -> std::result::Result<FactorizeResult<IdxTensor>, FactorizeError>
7476where
7477 S: IncrementalQrStateScalar,
7478{
7479 let (mut state, previous_left, previous_cap, previous_rank) = if let Some(previous) = previous {
7480 if appended_is_empty {
7481 return Ok(previous.clone());
7482 }
7483 let previous_rank = previous.rank;
7484 let resumed = S::resume(previous);
7485 let reused_previous_q = resumed.is_some();
7486 let state = if let Some(state) = resumed {
7487 state
7488 } else {
7489 let q = previous.left.clone();
7490 let cap = previous.bond_index.clone();
7491 let mut q_indices = left_inds.to_vec();
7492 q_indices.push(cap.clone());
7493 let q = q
7494 .permute_indices(&q_indices)
7495 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7496 let q_dims = q_indices.iter().map(IndexLike::dim).collect::<Vec<_>>();
7497 let q_rows = q_dims[..q_dims.len() - 1]
7498 .iter()
7499 .try_fold(1usize, |size, &dim| size.checked_mul(dim))
7500 .ok_or_else(|| {
7501 FactorizeError::ComputationError(anyhow::anyhow!(
7502 "incremental SRC Q row dimension overflows usize"
7503 ))
7504 })?;
7505 let q_matrix = Matrix::from_col_major_vec(
7506 q_rows,
7507 cap.dim(),
7508 q.to_vec::<S>()
7509 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7510 );
7511
7512 let batch = previous.right.indices().last().cloned().ok_or_else(|| {
7513 FactorizeError::ComputationError(anyhow::anyhow!(
7514 "incremental SRC R factor has no batch index"
7515 ))
7516 })?;
7517 let r = previous
7518 .right
7519 .permute_indices(&[cap.clone(), batch.clone()])
7520 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7521 let r_matrix = Matrix::from_col_major_vec(
7522 cap.dim(),
7523 batch.dim(),
7524 r.to_vec::<S>()
7525 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7526 );
7527 IncrementalQr::from_factors(q_matrix, r_matrix)
7528 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?
7529 };
7530 (
7531 state,
7532 if reused_previous_q {
7533 Some(previous.left.clone())
7534 } else {
7535 None
7536 },
7537 if reused_previous_q {
7538 Some(previous.bond_index.clone())
7539 } else {
7540 None
7541 },
7542 previous_rank,
7543 )
7544 } else {
7545 let initial_matrix = initial_matrix()?;
7546 (
7547 IncrementalQr::new(initial_matrix)
7548 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7549 None,
7550 None,
7551 0,
7552 )
7553 };
7554 if !appended_is_empty && previous.is_some() {
7555 let appended_matrix = appended_matrix()?;
7556 state
7557 .append(&appended_matrix)
7558 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7559 }
7560
7561 let rank = state.rank();
7562 if previous_left.is_some() && rank < previous_rank {
7563 return Err(FactorizeError::ComputationError(anyhow::anyhow!(
7564 "incremental SRC QR rank decreased from {previous_rank} to {rank}"
7565 )));
7566 }
7567 let r = state.r();
7568 let cap = if rank == previous_rank {
7569 match previous_cap.as_ref() {
7570 Some(cap) => cap.clone(),
7571 None => DynIndex::new_bond(rank)
7572 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7573 }
7574 } else {
7575 DynIndex::new_bond(rank)
7576 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?
7577 };
7578 let left = if let Some(previous_left) = previous_left {
7579 if rank == previous_rank {
7580 previous_left
7581 } else {
7582 let appended_rank = rank - previous_rank;
7583 let appended_cap = DynIndex::new_bond(appended_rank)
7584 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7585 let appended_q = state
7586 .q_columns(previous_rank, appended_rank)
7587 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7588 let mut appended_indices = left_inds.to_vec();
7589 appended_indices.push(appended_cap.clone());
7590 let appended_left =
7591 IdxTensor::from_dense(appended_indices, appended_q.as_col_major_slice().to_vec())
7592 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7593 IdxTensor::concatenate_along_new_index(
7594 &[&previous_left, &appended_left],
7595 &[
7596 previous_cap.ok_or_else(|| {
7597 FactorizeError::ComputationError(anyhow::anyhow!(
7598 "incremental SRC previous QR state has no bond index"
7599 ))
7600 })?,
7601 appended_cap,
7602 ],
7603 cap.clone(),
7604 )
7605 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?
7606 }
7607 } else {
7608 let q = state.q();
7609 let mut q_indices = left_inds.to_vec();
7610 q_indices.push(cap.clone());
7611 IdxTensor::from_dense(q_indices, q.as_col_major_slice().to_vec())
7612 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?
7613 };
7614 let batch = DynIndex::new_link(r.ncols())
7615 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7616 let right = IdxTensor::from_dense(vec![cap.clone(), batch], r.as_col_major_slice().to_vec())
7617 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7618 Ok(FactorizeResult::new(left, right, cap, None, rank)
7619 .with_incremental_qr_state(S::store(state)))
7620}
7621
7622fn incremental_probe_factorize_typed<S>(
7623 previous: Option<&FactorizeResult<IdxTensor>>,
7624 all_columns: &[&IdxTensor],
7625 appended_columns: &[&IdxTensor],
7626 left_inds: &[DynIndex],
7627) -> std::result::Result<FactorizeResult<IdxTensor>, FactorizeError>
7628where
7629 S: IncrementalQrStateScalar,
7630{
7631 incremental_probe_factorize_from_matrices::<S>(
7632 previous,
7633 appended_columns.is_empty(),
7634 left_inds,
7635 || probe_columns_matrix::<S>(all_columns, left_inds),
7636 || probe_columns_matrix::<S>(appended_columns, left_inds),
7637 )
7638}
7639
7640fn incremental_probe_factorize_batch_typed<S>(
7641 previous: Option<&FactorizeResult<IdxTensor>>,
7642 batch_tensor: &IdxTensor,
7643 batch_index: &DynIndex,
7644 left_inds: &[DynIndex],
7645) -> std::result::Result<FactorizeResult<IdxTensor>, FactorizeError>
7646where
7647 S: IncrementalQrStateScalar,
7648{
7649 incremental_probe_factorize_from_matrices::<S>(
7650 previous,
7651 batch_index.dim() == 0,
7652 left_inds,
7653 || probe_batch_matrix::<S>(batch_tensor, batch_index, left_inds),
7654 || probe_batch_matrix::<S>(batch_tensor, batch_index, left_inds),
7655 )
7656}
7657
7658fn factorize_probe_batch_incremental_impl(
7659 previous: Option<&FactorizeResult<IdxTensor>>,
7660 batch_tensor: &IdxTensor,
7661 batch_index: &DynIndex,
7662 left_inds: &[DynIndex],
7663) -> std::result::Result<FactorizeResult<IdxTensor>, FactorizeError> {
7664 if batch_tensor.is_f64() {
7665 incremental_probe_factorize_batch_typed::<f64>(
7666 previous,
7667 batch_tensor,
7668 batch_index,
7669 left_inds,
7670 )
7671 } else if batch_tensor.is_c64() {
7672 incremental_probe_factorize_batch_typed::<Complex64>(
7673 previous,
7674 batch_tensor,
7675 batch_index,
7676 left_inds,
7677 )
7678 } else {
7679 Err(FactorizeError::UnsupportedStorage(
7680 "incremental SRC factorization currently supports f64 and Complex64 tensors",
7681 ))
7682 }
7683}
7684
7685fn probe_columns_matrix<S>(
7686 columns: &[&IdxTensor],
7687 left_inds: &[DynIndex],
7688) -> std::result::Result<Matrix<S>, FactorizeError>
7689where
7690 S: TensorElement,
7691{
7692 let batch = DynIndex::new_link(columns.len())
7693 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7694 let stacked = IdxTensor::stack_along_new_index(columns, batch.clone(), -1)
7695 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7696 let mut ordered_indices = left_inds.to_vec();
7697 ordered_indices.push(batch);
7698 let ordered = stacked
7699 .permute_indices(&ordered_indices)
7700 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7701 let nrows = left_inds
7702 .iter()
7703 .try_fold(1usize, |size, index| size.checked_mul(index.dim()))
7704 .ok_or_else(|| {
7705 FactorizeError::ComputationError(anyhow::anyhow!(
7706 "incremental SRC sketch row dimension overflows usize"
7707 ))
7708 })?;
7709 Ok(Matrix::from_col_major_vec(
7710 nrows,
7711 columns.len(),
7712 ordered
7713 .to_vec::<S>()
7714 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7715 ))
7716}
7717
7718fn probe_batch_matrix<S>(
7723 batch_tensor: &IdxTensor,
7724 batch_index: &DynIndex,
7725 left_inds: &[DynIndex],
7726) -> std::result::Result<Matrix<S>, FactorizeError>
7727where
7728 S: TensorElement,
7729{
7730 let mut ordered_indices = left_inds.to_vec();
7731 ordered_indices.push(batch_index.clone());
7732 let ordered = batch_tensor
7733 .permute_indices(&ordered_indices)
7734 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?;
7735 let nrows = left_inds
7736 .iter()
7737 .try_fold(1usize, |size, index| size.checked_mul(index.dim()))
7738 .ok_or_else(|| {
7739 FactorizeError::ComputationError(anyhow::anyhow!(
7740 "incremental SRC sketch row dimension overflows usize"
7741 ))
7742 })?;
7743 Ok(Matrix::from_col_major_vec(
7744 nrows,
7745 batch_index.dim(),
7746 ordered
7747 .to_vec::<S>()
7748 .map_err(|error| FactorizeError::ComputationError(anyhow::Error::new(error)))?,
7749 ))
7750}
7751
7752impl TensorContractionLike for IdxTensor {
7753 fn conj(&self) -> Self {
7754 IdxTensor::conj(self)
7756 }
7757
7758 fn direct_sum(
7759 &self,
7760 other: &Self,
7761 pairs: &[(DynIndex, DynIndex)],
7762 ) -> std::result::Result<crate::tensor_like::DirectSumResult<Self>, Self::Error> {
7763 let (tensor, new_indices) = crate::direct_sum::direct_sum(self, other, pairs)?;
7764 Ok(crate::tensor_like::DirectSumResult {
7765 tensor,
7766 new_indices,
7767 })
7768 }
7769
7770 fn direct_sum_many(
7772 operands: &[&Self],
7773 groups: &[Vec<DynIndex>],
7774 ) -> std::result::Result<crate::tensor_like::DirectSumResult<Self>, Self::Error> {
7775 let (tensor, new_indices) = crate::direct_sum::direct_sum_many(operands, groups)?;
7776 Ok(crate::tensor_like::DirectSumResult {
7777 tensor,
7778 new_indices,
7779 })
7780 }
7781
7782 fn outer_product(&self, other: &Self) -> std::result::Result<Self, Self::Error> {
7783 super::contract::outer_product(self, other)
7784 }
7785
7786 fn permuteinds(&self, new_order: &[DynIndex]) -> std::result::Result<Self, Self::Error> {
7787 IdxTensor::permute_indices(self, new_order)
7789 }
7790
7791 fn fuse_indices(
7792 &self,
7793 old_indices: &[DynIndex],
7794 new_index: DynIndex,
7795 order: LinearizationOrder,
7796 ) -> std::result::Result<Self, Self::Error> {
7797 IdxTensor::fuse_indices(self, old_indices, new_index, order)
7798 }
7799
7800 fn contract(tensors: &[&Self]) -> std::result::Result<Self, Self::Error> {
7801 super::contract::contract(tensors)
7802 }
7803
7804 fn contract_retaining_indices(
7805 tensors: &[&Self],
7806 retained_indices: &[DynIndex],
7807 ) -> std::result::Result<Self, Self::Error> {
7808 if tensors.len() == 2 {
7809 return tensors[0]
7810 .try_contract_pairwise_retaining(tensors[1], retained_indices)
7811 .map_err(IdxTensorError::from);
7812 }
7813 let options = ContractionOptions::new().with_retain_indices(retained_indices);
7814 super::contract::contract_with_options(tensors, options)
7815 }
7816
7817 fn contract_pair(&self, other: &Self) -> std::result::Result<Self, Self::Error> {
7818 super::contract::contract_pair(self, other)
7819 }
7820}
7821
7822impl TensorConstructionLike for IdxTensor {
7823 fn select_indices(
7824 &self,
7825 selected_indices: &[DynIndex],
7826 positions: &[usize],
7827 ) -> std::result::Result<Self, Self::Error> {
7828 IdxTensor::select_indices(self, selected_indices, positions)
7829 }
7830
7831 fn diagonal(
7832 input_index: &DynIndex,
7833 output_index: &DynIndex,
7834 ) -> std::result::Result<Self, Self::Error> {
7835 let dim = input_index.dim();
7836 if dim != output_index.dim() {
7837 return Err(anyhow::anyhow!(
7838 "Dimension mismatch: input index has dim {}, output has dim {}",
7839 dim,
7840 output_index.dim(),
7841 )
7842 .into());
7843 }
7844
7845 IdxTensor::from_diag(
7846 vec![input_index.clone(), output_index.clone()],
7847 vec![1.0_f64; dim],
7848 )
7849 }
7850
7851 fn scalar_one() -> std::result::Result<Self, Self::Error> {
7852 IdxTensor::from_dense(vec![], vec![1.0_f64])
7853 }
7854
7855 fn ones(indices: &[DynIndex]) -> std::result::Result<Self, Self::Error> {
7856 if indices.is_empty() {
7857 return <Self as TensorConstructionLike>::scalar_one();
7858 }
7859 let dims: Vec<usize> = indices.iter().map(|idx| idx.size()).collect();
7860 let total_size = checked_total_size(&dims)?;
7861 IdxTensor::from_dense(indices.to_vec(), vec![1.0_f64; total_size])
7862 }
7863
7864 fn ones_in(
7865 context: &tensor4all_tensorbackend::ExecutionContext,
7866 indices: &[DynIndex],
7867 ) -> std::result::Result<Self, Self::Error> {
7868 IdxTensor::ones_in(context, indices)
7869 }
7870
7871 fn validate_context(
7872 &self,
7873 context: &tensor4all_tensorbackend::ExecutionContext,
7874 ) -> std::result::Result<(), Self::Error> {
7875 IdxTensor::validate_context(self, context)
7876 }
7877
7878 fn adopt_into(
7879 &self,
7880 context: &tensor4all_tensorbackend::ExecutionContext,
7881 ) -> std::result::Result<Self, Self::Error> {
7882 IdxTensor::adopt_into(self, context)
7883 }
7884
7885 fn from_dense_any(
7886 indices: Vec<DynIndex>,
7887 data: Vec<AnyScalar>,
7888 ) -> std::result::Result<Self, Self::Error> {
7889 IdxTensor::from_dense_any(indices, data)
7890 }
7891
7892 fn from_dense<T: TensorElement + Into<AnyScalar>>(
7893 indices: Vec<DynIndex>,
7894 data: Vec<T>,
7895 ) -> std::result::Result<Self, Self::Error> {
7896 IdxTensor::from_dense(indices, data)
7897 }
7898
7899 fn from_dense_in<T: TensorElement + Into<AnyScalar>>(
7900 context: &tensor4all_tensorbackend::ExecutionContext,
7901 indices: Vec<DynIndex>,
7902 data: Vec<T>,
7903 ) -> std::result::Result<Self, Self::Error> {
7904 IdxTensor::from_dense_in(context, indices, data)
7905 }
7906
7907 fn stack_along_new_index(
7908 tensors: &[&Self],
7909 new_index: DynIndex,
7910 axis: isize,
7911 ) -> std::result::Result<Self, Self::Error> {
7912 IdxTensor::stack_along_new_index(tensors, new_index, axis)
7913 }
7914
7915 fn concatenate_along_new_index(
7916 tensors: &[&Self],
7917 source_indices: &[DynIndex],
7918 new_index: DynIndex,
7919 ) -> std::result::Result<Self, Self::Error> {
7920 let first = tensors
7921 .first()
7922 .ok_or_else(|| anyhow::anyhow!("concatenate requires at least one tensor"))?;
7923 if tensors.len() != source_indices.len() {
7924 return Err(anyhow::anyhow!(
7925 "concatenate tensor count {} does not match source-index count {}",
7926 tensors.len(),
7927 source_indices.len()
7928 )
7929 .into());
7930 }
7931 let first_axis = first
7932 .indices
7933 .iter()
7934 .position(|index| index == &source_indices[0])
7935 .ok_or_else(|| anyhow::anyhow!("concatenate source index is not present"))?;
7936 let mut total_dim = 0usize;
7937 let first_indices = first.indices();
7938 let mut inners = Vec::with_capacity(tensors.len());
7939 for (tensor, source) in tensors.iter().zip(source_indices) {
7940 let axis = tensor
7941 .indices
7942 .iter()
7943 .position(|index| index == source)
7944 .ok_or_else(|| anyhow::anyhow!("concatenate source index is not present"))?;
7945 if axis != first_axis {
7946 return Err(
7947 anyhow::anyhow!("concatenate source axes must have the same position").into(),
7948 );
7949 }
7950 if tensor
7951 .indices()
7952 .iter()
7953 .zip(first_indices)
7954 .enumerate()
7955 .any(|(position, (actual, expected))| position != first_axis && actual != expected)
7956 {
7957 return Err(anyhow::anyhow!(
7958 "concatenate tensors must match away from the source axis"
7959 )
7960 .into());
7961 }
7962 total_dim = total_dim
7963 .checked_add(source.dim())
7964 .ok_or_else(|| anyhow::anyhow!("concatenate dimension overflow"))?;
7965 tensor.ensure_shape_packing_preserves_ad("concatenate_along_new_index")?;
7966 inners.push(tensor.try_materialized_inner()?);
7967 }
7968 if new_index.dim() != total_dim {
7969 return Err(anyhow::anyhow!(
7970 "concatenate output dimension {} does not match source dimension sum {}",
7971 new_index.dim(),
7972 total_dim
7973 )
7974 .into());
7975 }
7976 let runtime = inners
7977 .first()
7978 .map(|tensor| tensor.runtime())
7979 .ok_or_else(|| anyhow::anyhow!("concatenate requires at least one tensor"))?;
7980 let concatenated = with_eager_session(runtime, None, |session| {
7981 session.concatenate(&inners, first_axis)
7982 })?;
7983 let mut indices = first_indices.to_vec();
7984 indices[first_axis] = new_index;
7985 Self::from_inner(indices, concatenated).map_err(IdxTensorError::from)
7986 }
7987
7988 fn onehot(index_vals: &[(DynIndex, usize)]) -> std::result::Result<Self, Self::Error> {
7989 if index_vals.is_empty() {
7990 return <Self as TensorConstructionLike>::scalar_one();
7991 }
7992 let indices: Vec<DynIndex> = index_vals.iter().map(|(idx, _)| idx.clone()).collect();
7993 let vals: Vec<usize> = index_vals.iter().map(|(_, v)| *v).collect();
7994 let dims: Vec<usize> = indices.iter().map(|idx| idx.size()).collect();
7995
7996 for (k, (&v, &d)) in vals.iter().zip(dims.iter()).enumerate() {
7997 if v >= d {
7998 return Err(anyhow::anyhow!(
7999 "onehot: value {} at position {} is >= dimension {}",
8000 v,
8001 k,
8002 d
8003 )
8004 .into());
8005 }
8006 }
8007
8008 let total_size = checked_total_size(&dims).map_err(Self::Error::from)?;
8009 let mut data = vec![0.0_f64; total_size];
8010
8011 let offset = column_major_offset(&dims, &vals).map_err(Self::Error::from)?;
8012 data[offset] = 1.0;
8013
8014 Self::from_dense(indices, data)
8015 }
8016
8017 }
8019
8020fn checked_total_size(dims: &[usize]) -> Result<usize> {
8021 dims.iter().try_fold(1_usize, |acc, &d| {
8022 if d == 0 {
8023 return Err(anyhow::anyhow!("invalid dimension 0"));
8024 }
8025 acc.checked_mul(d)
8026 .ok_or_else(|| anyhow::anyhow!("tensor size overflow"))
8027 })
8028}
8029
8030fn column_major_offset(dims: &[usize], vals: &[usize]) -> Result<usize> {
8031 if dims.len() != vals.len() {
8032 return Err(anyhow::anyhow!(
8033 "column_major_offset: dims.len() != vals.len()"
8034 ));
8035 }
8036 checked_total_size(dims)?;
8037
8038 let mut offset = 0usize;
8039 let mut stride = 1usize;
8040 for (k, (&v, &d)) in vals.iter().zip(dims.iter()).enumerate() {
8041 if d == 0 {
8042 return Err(anyhow::anyhow!("invalid dimension 0 at position {}", k));
8043 }
8044 if v >= d {
8045 return Err(anyhow::anyhow!(
8046 "column_major_offset: value {} at position {} is >= dimension {}",
8047 v,
8048 k,
8049 d
8050 ));
8051 }
8052 let term = v
8053 .checked_mul(stride)
8054 .ok_or_else(|| anyhow::anyhow!("column_major_offset: overflow"))?;
8055 offset = offset
8056 .checked_add(term)
8057 .ok_or_else(|| anyhow::anyhow!("column_major_offset: overflow"))?;
8058 stride = stride
8059 .checked_mul(d)
8060 .ok_or_else(|| anyhow::anyhow!("column_major_offset: overflow"))?;
8061 }
8062 Ok(offset)
8063}
8064
8065impl IdxTensor {
8070 fn any_scalar_payload_to_complex(data: Vec<AnyScalar>) -> Vec<Complex64> {
8071 data.into_iter()
8072 .map(|value| {
8073 value
8074 .as_c64()
8075 .unwrap_or_else(|| Complex64::new(value.real(), 0.0))
8076 })
8077 .collect()
8078 }
8079
8080 fn any_scalar_payload_to_real(data: Vec<AnyScalar>) -> Vec<f64> {
8081 data.into_iter().map(|value| value.real()).collect()
8082 }
8083
8084 fn validate_dense_payload_len(data_len: usize, dims: &[usize]) -> Result<()> {
8085 let expected_len = checked_total_size(dims)?;
8086 if !(data_len == expected_len) {
8087 return Err(anyhow::anyhow!(
8088 "dense payload length {} does not match dims {:?} (expected {})",
8089 data_len,
8090 dims,
8091 expected_len
8092 ));
8093 };
8094 Ok(())
8095 }
8096
8097 fn validate_diag_payload_len(data_len: usize, dims: &[usize]) -> Result<()> {
8098 if !(!dims.is_empty()) {
8099 return Err(anyhow::anyhow!(
8100 "diagonal tensor construction requires at least one index"
8101 ));
8102 };
8103 Self::validate_diag_dims(dims)?;
8104 if !(data_len == dims[0]) {
8105 return Err(anyhow::anyhow!(
8106 "diagonal payload length {} does not match diagonal dimension {}",
8107 data_len,
8108 dims[0]
8109 ));
8110 };
8111 Ok(())
8112 }
8113
8114 pub fn from_dense<T: TensorElement>(
8141 indices: Vec<DynIndex>,
8142 data: Vec<T>,
8143 ) -> std::result::Result<Self, IdxTensorError> {
8144 let dims = Self::expected_dims_from_indices(&indices);
8145 Self::validate_indices(&indices)?;
8146 Self::validate_dense_payload_len(data.len(), &dims)?;
8147 let native = dense_native_tensor_from_col_major_owned(data, &dims)?;
8148 Self::from_native(indices, native).map_err(IdxTensorError::from)
8149 }
8150
8151 pub fn from_dense_any(
8180 indices: Vec<DynIndex>,
8181 data: Vec<AnyScalar>,
8182 ) -> std::result::Result<Self, IdxTensorError> {
8183 if data.iter().any(AnyScalar::is_complex) {
8184 Self::from_dense(indices, Self::any_scalar_payload_to_complex(data))
8185 } else {
8186 Self::from_dense(indices, Self::any_scalar_payload_to_real(data))
8187 }
8188 }
8189
8190 pub fn from_diag<T: TensorElement>(
8223 indices: Vec<DynIndex>,
8224 data: Vec<T>,
8225 ) -> std::result::Result<Self, IdxTensorError> {
8226 let dims = Self::expected_dims_from_indices(&indices);
8227 Self::validate_indices(&indices)?;
8228 Self::validate_diag_payload_len(data.len(), &dims)?;
8229 let native = diag_native_tensor_from_col_major(&data, dims.len())?;
8230 Self::from_native_with_axis_classes(indices, native, Self::diag_axis_classes(dims.len()))
8231 .map_err(IdxTensorError::from)
8232 }
8233
8234 pub fn from_diag_any(
8260 indices: Vec<DynIndex>,
8261 data: Vec<AnyScalar>,
8262 ) -> std::result::Result<Self, IdxTensorError> {
8263 if data.iter().any(AnyScalar::is_complex) {
8264 Self::from_diag(indices, Self::any_scalar_payload_to_complex(data))
8265 } else {
8266 Self::from_diag(indices, Self::any_scalar_payload_to_real(data))
8267 }
8268 }
8269
8270 pub fn copy_tensor(
8294 indices: Vec<DynIndex>,
8295 value: AnyScalar,
8296 ) -> std::result::Result<Self, IdxTensorError> {
8297 if indices.is_empty() {
8298 return Self::from_dense_any(vec![], vec![value]);
8299 }
8300 let dim = indices[0].dim();
8301 let data = vec![value; dim];
8302 Self::from_diag_any(indices, data)
8303 }
8304
8305 pub fn fuse_indices(
8362 &self,
8363 old_indices: &[DynIndex],
8364 new_index: DynIndex,
8365 order: LinearizationOrder,
8366 ) -> std::result::Result<Self, IdxTensorError> {
8367 if !(!old_indices.is_empty()) {
8368 return Err(anyhow::anyhow!("fuse_indices requires at least one index to fuse").into());
8369 };
8370
8371 let old_dims = self.dims();
8372 let mut seen_indices = HashSet::new();
8373 let mut old_axes = Vec::with_capacity(old_indices.len());
8374 for old_index in old_indices {
8375 if !(seen_indices.insert(old_index)) {
8376 return Err(anyhow::anyhow!("duplicate index in old_indices").into());
8377 };
8378 let axis = self
8379 .indices
8380 .iter()
8381 .position(|idx| idx == old_index)
8382 .ok_or_else(|| anyhow::anyhow!("index {:?} not found in tensor", old_index))?;
8383 if !(old_index.dim() == old_dims[axis]) {
8384 return Err(anyhow::anyhow!(
8385 "old index dimension does not match tensor axis dimension"
8386 )
8387 .into());
8388 };
8389 old_axes.push(axis);
8390 }
8391
8392 let fused_dims: Vec<usize> = old_axes.iter().map(|&axis| old_dims[axis]).collect();
8393 let fused_product = checked_product(&fused_dims)?;
8394 if !(fused_product == new_index.dim()) {
8395 return Err(anyhow::anyhow!(
8396 "product of old index dimensions must match the replacement index dimension"
8397 )
8398 .into());
8399 };
8400
8401 let insertion_axis =
8402 old_axes.iter().copied().min().ok_or_else(|| {
8403 anyhow::anyhow!("fuse_indices requires at least one index to fuse")
8404 })?;
8405 let old_axis_set: HashSet<usize> = old_axes.iter().copied().collect();
8406
8407 let mut result_indices =
8408 Vec::with_capacity(self.indices.len() - old_indices.len() + 1usize);
8409 for (axis, index) in self.indices.iter().enumerate() {
8410 if axis == insertion_axis {
8411 result_indices.push(new_index.clone());
8412 }
8413 if !old_axis_set.contains(&axis) {
8414 result_indices.push(index.clone());
8415 }
8416 }
8417 let mut result_seen = HashSet::new();
8418 for index in &result_indices {
8419 if !(result_seen.insert(index)) {
8420 return Err(
8421 anyhow::anyhow!("fuse_indices result would contain duplicate index").into(),
8422 );
8423 };
8424 }
8425 Self::validate_indices(&result_indices)?;
8426
8427 let mut new_dims = Vec::with_capacity(old_dims.len() - old_indices.len() + 1usize);
8428 for (axis, dim) in old_dims.iter().copied().enumerate() {
8429 if axis == insertion_axis {
8430 new_dims.push(new_index.dim());
8431 }
8432 if !old_axis_set.contains(&axis) {
8433 new_dims.push(dim);
8434 }
8435 }
8436
8437 self.ensure_shape_packing_preserves_ad("fuse_indices")?;
8438
8439 let mut grouped_axes = old_axes.clone();
8440 if matches!(order, LinearizationOrder::RowMajor) {
8441 grouped_axes.reverse();
8442 }
8443 let mut perm = Vec::with_capacity(self.indices.len());
8444 perm.extend((0..insertion_axis).filter(|axis| !old_axis_set.contains(axis)));
8445 perm.extend(grouped_axes);
8446 perm.extend(
8447 ((insertion_axis + 1)..self.indices.len()).filter(|axis| !old_axis_set.contains(axis)),
8448 );
8449 debug_assert_eq!(perm.len(), self.indices.len());
8450
8451 let packed = self.permute(&perm)?;
8452 let packed_inner = packed.try_materialized_inner()?;
8453 let reshaped = with_eager_session(packed_inner.runtime(), None, |session| {
8454 session.reshape(packed_inner, &new_dims)
8455 })?;
8456 Self::from_inner(result_indices, reshaped).map_err(IdxTensorError::from)
8457 }
8458
8459 pub fn unfuse_index(
8485 &self,
8486 old_index: &DynIndex,
8487 new_indices: &[DynIndex],
8488 order: LinearizationOrder,
8489 ) -> std::result::Result<Self, IdxTensorError> {
8490 if !(!new_indices.is_empty()) {
8491 return Err(
8492 anyhow::anyhow!("unfuse_index requires at least one replacement index").into(),
8493 );
8494 };
8495
8496 let axis = self
8497 .indices
8498 .iter()
8499 .position(|idx| idx == old_index)
8500 .ok_or_else(|| anyhow::anyhow!("index {:?} not found in tensor", old_index))?;
8501
8502 let replacement_dims: Vec<usize> = new_indices.iter().map(DynIndex::dim).collect();
8503 let replacement_product = checked_product(&replacement_dims)?;
8504 if !(replacement_product == old_index.dim()) {
8505 return Err(anyhow::anyhow!(
8506 "product of new index dimensions must match the replaced index dimension"
8507 )
8508 .into());
8509 };
8510
8511 let mut result_indices =
8512 Vec::with_capacity(self.indices.len() - 1usize + new_indices.len());
8513 result_indices.extend_from_slice(&self.indices[..axis]);
8514 result_indices.extend(new_indices.iter().cloned());
8515 result_indices.extend_from_slice(&self.indices[axis + 1..]);
8516 Self::validate_indices(&result_indices)?;
8517
8518 let old_dims = self.dims();
8519 let mut new_dims = Vec::with_capacity(old_dims.len() - 1usize + replacement_dims.len());
8520 new_dims.extend_from_slice(&old_dims[..axis]);
8521 new_dims.extend_from_slice(&replacement_dims);
8522 new_dims.extend_from_slice(&old_dims[axis + 1..]);
8523
8524 self.ensure_shape_packing_preserves_ad("unfuse_index")?;
8525
8526 let mut grouped_indices = new_indices.to_vec();
8527 let mut grouped_dims = replacement_dims.clone();
8528 if matches!(order, LinearizationOrder::RowMajor) {
8529 grouped_indices.reverse();
8530 grouped_dims.reverse();
8531 }
8532 let mut packed_indices =
8533 Vec::with_capacity(self.indices.len() - 1usize + grouped_indices.len());
8534 packed_indices.extend_from_slice(&self.indices[..axis]);
8535 packed_indices.extend(grouped_indices);
8536 packed_indices.extend_from_slice(&self.indices[axis + 1..]);
8537
8538 let mut packed_dims = Vec::with_capacity(old_dims.len() - 1usize + grouped_dims.len());
8539 packed_dims.extend_from_slice(&old_dims[..axis]);
8540 packed_dims.extend_from_slice(&grouped_dims);
8541 packed_dims.extend_from_slice(&old_dims[axis + 1..]);
8542
8543 let inner = self.try_materialized_inner()?;
8544 let reshaped = with_eager_session(inner.runtime(), None, |session| {
8545 session.reshape(inner, &packed_dims)
8546 })?;
8547 let packed = Self::from_inner(packed_indices, reshaped)?;
8548 if matches!(order, LinearizationOrder::ColumnMajor) {
8549 Ok(packed)
8550 } else {
8551 packed.permute_indices(&result_indices)
8552 }
8553 }
8554
8555 pub fn scalar<T: TensorElement>(value: T) -> std::result::Result<Self, IdxTensorError> {
8569 Self::from_dense(vec![], vec![value])
8570 }
8571
8572 pub fn zeros<T: TensorElement + Zero + Clone>(
8588 indices: Vec<DynIndex>,
8589 ) -> std::result::Result<Self, IdxTensorError> {
8590 let dims: Vec<usize> = indices.iter().map(|idx| idx.dim()).collect();
8591 let size = checked_product(&dims)?;
8592 Self::from_dense(indices, vec![T::zero(); size])
8593 }
8594}
8595
8596impl IdxTensor {
8601 pub fn to_vec<T: TensorElement>(&self) -> std::result::Result<Vec<T>, IdxTensorError> {
8630 let inner = self.as_inner()?;
8631 let values = with_engine(inner.runtime(), None, |engine| {
8632 inner.duplicate_value(engine)
8633 })?;
8634 values
8635 .as_slice::<T>()
8636 .map(|values| values.to_vec())
8637 .map_err(|source| {
8638 if matches!(
8642 source.kind(),
8643 tenferro_tensor::ErrorKind::Validation(
8644 tenferro_tensor::ValidationKind::DTypeMismatch
8645 )
8646 ) {
8647 IdxTensorError::PayloadScalarTypeMismatch
8648 } else {
8649 IdxTensorError::materialization(anyhow::Error::new(source))
8650 }
8651 })
8652 }
8653
8654 pub fn with_dense_slice<T: TensorElement, R>(
8693 &self,
8694 read: impl FnOnce(&[T]) -> R,
8695 ) -> std::result::Result<R, IdxTensorError> {
8696 let inner = self.as_inner()?;
8697 let value = inner.value()?;
8698 if let Ok(values) = value.as_slice::<T>() {
8699 return Ok(read(values));
8700 }
8701
8702 drop(value);
8707 let values = with_engine(inner.runtime(), None, |engine| {
8708 inner.duplicate_value(engine)
8709 })?;
8710 let values = values
8711 .as_slice::<T>()
8712 .map_err(|source| IdxTensorError::materialization(anyhow::Error::new(source)))?;
8713 Ok(read(values))
8714 }
8715
8716 pub fn with_dense_host_slice<T: tensor4all_tensorbackend::StorageScalar, R>(
8750 &self,
8751 read: impl FnOnce(&[T]) -> R,
8752 ) -> std::result::Result<R, IdxTensorError> {
8753 if let IdxTensorStorage::Materialized(storage) = &self.storage {
8754 let dims = self.dims();
8755 if storage.storage_kind() == StorageKind::Dense {
8756 let view = T::payload_view(storage)
8757 .map_err(|error| IdxTensorError::materialization(anyhow::Error::new(error)))?;
8758 if let Some(values) =
8762 view.filter(|values| values.len() == dims.iter().product::<usize>())
8763 {
8764 return Ok(read(values));
8765 }
8766 }
8767 let values = T::dense_values(storage, &dims)
8768 .map_err(|error| IdxTensorError::materialization(anyhow::Error::new(error)))?;
8769 return Ok(read(&values));
8770 }
8771 self.with_dense_slice::<T, _>(read)
8772 }
8773
8774 pub fn into_dense_col_major_parts<T: TensorElement>(
8809 self,
8810 ) -> std::result::Result<(Vec<DynIndex>, Vec<T>), IdxTensorError> {
8811 if !(!self.tracks_grad()) {
8812 return Err(anyhow::anyhow!("IdxTensor::into_dense_col_major_parts cannot consume tensors with tracked autodiff state").into());
8813 };
8814 let data = self.to_vec::<T>()?;
8815 Ok((self.indices, data))
8816 }
8817
8818 pub fn is_f64(&self) -> bool {
8831 self.storage.dtype() == Some(DType::F64)
8832 }
8833
8834 pub fn is_f32(&self) -> bool {
8849 self.storage.dtype() == Some(DType::F32)
8850 }
8851
8852 pub fn is_c32(&self) -> bool {
8868 self.storage.dtype() == Some(DType::C32)
8869 }
8870
8871 pub fn is_c64(&self) -> bool {
8887 self.storage.is_c64()
8888 }
8889
8890 pub fn is_diag(&self) -> bool {
8917 self.storage.is_diag()
8918 }
8919
8920 pub fn is_complex(&self) -> bool {
8939 self.storage.is_complex()
8940 }
8941 pub fn from_dense_in<T: TensorElement>(
8970 context: &ExecutionContext,
8971 indices: Vec<DynIndex>,
8972 data: Vec<T>,
8973 ) -> std::result::Result<Self, IdxTensorError> {
8974 let dims = Self::expected_dims_from_indices(&indices);
8975 Self::validate_indices(&indices)?;
8976 Self::validate_dense_payload_len(data.len(), &dims)?;
8977 #[cfg(not(feature = "tenferro-cuda"))]
8978 ensure_supported_execution_context(context)?;
8979 let native = dense_native_tensor_from_col_major(&data, &dims)
8980 .map_err(|error| IdxTensorError::operation("context-scoped construction", error))?;
8981 let inner = match context {
8982 ExecutionContext::Cpu(context) => {
8983 let runtime = context.eager_runtime().map_err(|error| {
8984 IdxTensorError::operation("CPU context construction", anyhow::Error::new(error))
8985 })?;
8986 IdxTensor::untracked_inner(native, runtime).map_err(|error| {
8987 IdxTensorError::operation(
8988 "CPU context eager wrapping",
8989 anyhow::Error::new(error),
8990 )
8991 })?
8992 }
8993 #[cfg(feature = "tenferro-cuda")]
8994 ExecutionContext::Cuda(context) => {
8995 let uploaded = context.upload_cuda(&native).map_err(|error| {
8996 IdxTensorError::operation(
8997 "CUDA context construction",
8998 anyhow::Error::new(error),
8999 )
9000 })?;
9001 IdxTensor::untracked_inner(
9002 uploaded,
9003 context.eager_runtime().map_err(|error| {
9004 IdxTensorError::operation(
9005 "CUDA context eager runtime",
9006 anyhow::Error::new(error),
9007 )
9008 })?,
9009 )
9010 .map_err(|error| {
9011 IdxTensorError::operation(
9012 "CUDA context eager wrapping",
9013 anyhow::Error::new(error),
9014 )
9015 })?
9016 }
9017 #[cfg(not(feature = "tenferro-cuda"))]
9019 #[allow(unreachable_patterns)]
9020 _ => return Err(unsupported_execution_context_error()),
9021 };
9022 Self::from_inner(indices, inner).map_err(IdxTensorError::from)
9023 }
9024
9025 pub fn ones_in(
9051 context: &ExecutionContext,
9052 indices: &[DynIndex],
9053 ) -> std::result::Result<Self, IdxTensorError> {
9054 let dims = Self::expected_dims_from_indices(indices);
9055 let total_size = checked_total_size(&dims)?;
9056 #[cfg(not(feature = "tenferro-cuda"))]
9057 ensure_supported_execution_context(context)?;
9058 Self::from_dense_in(context, indices.to_vec(), vec![1.0_f64; total_size])
9059 }
9060
9061 pub fn validate_context(
9087 &self,
9088 context: &ExecutionContext,
9089 ) -> std::result::Result<(), IdxTensorError> {
9090 #[cfg(not(feature = "tenferro-cuda"))]
9091 ensure_supported_execution_context(context)?;
9092 let Some(inner) = self.storage.eager() else {
9093 if matches!(context, ExecutionContext::Cpu(_)) {
9094 return Ok(());
9095 }
9096 return Err(IdxTensorError::operation(
9097 "context validation",
9098 anyhow::anyhow!("tensor has no CUDA eager runtime"),
9099 ));
9100 };
9101 match context {
9102 ExecutionContext::Cpu(context) => {
9103 let expected = context.eager_runtime().map_err(|error| {
9104 IdxTensorError::operation("CPU context validation", anyhow::Error::new(error))
9105 })?;
9106 if inner.ctx_id() != expected.id() {
9107 return Err(IdxTensorError::operation(
9108 "context validation",
9109 anyhow::anyhow!("tensor belongs to a different CPU eager context"),
9110 ));
9111 }
9112 }
9113 #[cfg(feature = "tenferro-cuda")]
9114 ExecutionContext::Cuda(context) => {
9115 self.validate_cuda_residency(context).map_err(|error| {
9116 IdxTensorError::operation("CUDA context validation", anyhow::Error::new(error))
9117 })?;
9118 }
9119 #[cfg(not(feature = "tenferro-cuda"))]
9121 #[allow(unreachable_patterns)]
9122 _ => return Err(unsupported_execution_context_error()),
9123 }
9124 Ok(())
9125 }
9126
9127 pub fn adopt_into(
9180 &self,
9181 context: &ExecutionContext,
9182 ) -> std::result::Result<Self, IdxTensorError> {
9183 #[cfg(not(feature = "tenferro-cuda"))]
9184 ensure_supported_execution_context(context)?;
9185 let runtime = self.eager_runtime();
9186 if let Some(runtime) = &runtime {
9187 if runtime.id() == Self::context_eager_runtime(context)?.id() {
9188 return Ok(self.clone());
9189 }
9190 }
9191 if self
9194 .axis_classes()
9195 .iter()
9196 .copied()
9197 .enumerate()
9198 .any(|(axis, class)| axis != class)
9199 {
9200 return Err(IdxTensorError::operation(
9201 "context adoption",
9202 anyhow::anyhow!(
9203 "structured storage cannot be adopted; reading it back would expand it \
9204 densely, so publish a dense payload with IdxTensor::from_dense_in"
9205 ),
9206 ));
9207 }
9208 if self.tracks_grad() {
9209 return Err(IdxTensorError::operation(
9210 "context adoption",
9211 anyhow::anyhow!(
9212 "tensor tracks gradients; re-publishing it in another context would drop \
9213 the recorded graph, so rebuild it there instead"
9214 ),
9215 ));
9216 }
9217 match &runtime {
9218 None => Self::adopt_host_into_context(self, context),
9220 Some(runtime) if runtime.id() == default_eager_ctx()?.id() => {
9222 Self::adopt_host_into_context(self, context)
9223 }
9224 Some(_) => Err(IdxTensorError::operation(
9225 "context adoption",
9226 anyhow::anyhow!(
9227 "tensor belongs to a different execution context; re-binding it would copy \
9228 its storage into the target context"
9229 ),
9230 )),
9231 }
9232 }
9233
9234 pub fn read_decision_data(
9278 &self,
9279 context: &ExecutionContext,
9280 ) -> std::result::Result<Vec<f64>, IdxTensorError> {
9281 self.validate_context(context)?;
9282 #[cfg(feature = "tenferro-cuda")]
9283 if let ExecutionContext::Cuda(cuda) = context {
9284 let host = self.download(cuda).map_err(|error| {
9285 IdxTensorError::operation("decision readback download", anyhow::Error::new(error))
9286 })?;
9287 return Self::decision_values(&host);
9288 }
9289 Self::decision_values(self)
9290 }
9291
9292 fn decision_values(host: &Self) -> std::result::Result<Vec<f64>, IdxTensorError> {
9293 if host.is_f64() {
9298 host.dense_host_values::<f64>().map_err(|error| {
9299 IdxTensorError::operation("decision readback", anyhow::Error::new(error))
9300 })
9301 } else if host.is_c64() {
9302 host.dense_host_values::<Complex64>()
9303 .map(|values| values.into_iter().map(|value| value.re).collect())
9304 .map_err(|error| {
9305 IdxTensorError::operation("decision readback", anyhow::Error::new(error))
9306 })
9307 } else {
9308 Err(IdxTensorError::operation(
9309 "decision readback",
9310 anyhow::anyhow!("decision payloads support f64 and Complex64 storage only"),
9311 ))
9312 }
9313 }
9314
9315 pub fn scale_in(
9357 &self,
9358 factor: f64,
9359 context: &ExecutionContext,
9360 ) -> std::result::Result<Self, IdxTensorError> {
9361 self.validate_context(context)?;
9362 let adopted_storage;
9363 let operand: &EagerTensor = match self.eager_runtime() {
9364 Some(_) => self.as_inner()?,
9365 None => {
9368 adopted_storage = Self::adopt_host_into_context(self, context)?;
9369 adopted_storage.as_inner()?
9370 }
9371 };
9372 let scalar = Self::context_scalar_in(operand, factor, context)?;
9373 let scaled = with_eager_session(operand.runtime(), Some(context), |session| {
9374 session.mul(operand, &scalar)
9375 })
9376 .map_err(|error| {
9377 IdxTensorError::operation(
9378 "context-scoped scaling",
9379 anyhow::Error::new(error).context("eager multiplication failed"),
9380 )
9381 })?;
9382 Self::from_inner(self.indices.clone(), scaled).map_err(IdxTensorError::from)
9383 }
9384
9385 fn adopt_host_into_context(
9391 tensor: &Self,
9392 context: &ExecutionContext,
9393 ) -> std::result::Result<Self, IdxTensorError> {
9394 if tensor.is_f64() {
9395 let values = tensor.dense_host_values::<f64>().map_err(|error| {
9396 IdxTensorError::operation("context adoption", anyhow::Error::new(error))
9397 })?;
9398 Self::from_dense_in(context, tensor.indices.clone(), values)
9399 } else if tensor.is_c64() {
9400 let values = tensor.dense_host_values::<Complex64>().map_err(|error| {
9401 IdxTensorError::operation("context adoption", anyhow::Error::new(error))
9402 })?;
9403 Self::from_dense_in(context, tensor.indices.clone(), values)
9404 } else {
9405 Err(IdxTensorError::operation(
9406 "context adoption",
9407 anyhow::anyhow!("adoption supports f64 and Complex64 storage only"),
9408 ))
9409 }
9410 }
9411
9412 fn context_scalar_in(
9414 operand: &EagerTensor,
9415 factor: f64,
9416 context: &ExecutionContext,
9417 ) -> std::result::Result<EagerTensor, IdxTensorError> {
9418 #[cfg(not(feature = "tenferro-cuda"))]
9419 ensure_supported_execution_context(context)?;
9420 let native: NativeTensor = match operand.dtype() {
9421 DType::F64 => NativeTensor::from_vec_col_major(vec![], vec![factor]),
9422 DType::C64 => {
9423 NativeTensor::from_vec_col_major(vec![], vec![Complex64::new(factor, 0.0)])
9424 }
9425 other => {
9426 return Err(IdxTensorError::operation(
9427 "context-scoped scaling",
9428 anyhow::anyhow!("unsupported scalar dtype {other:?}"),
9429 ));
9430 }
9431 }
9432 .map_err(|error| {
9433 IdxTensorError::operation("context-scoped scaling", anyhow::anyhow!("{error}"))
9434 })?;
9435 let runtime = Self::context_eager_runtime(context)?;
9436 let resident = match context {
9437 ExecutionContext::Cpu(_) => native,
9438 #[cfg(feature = "tenferro-cuda")]
9439 ExecutionContext::Cuda(cuda) => cuda.upload_cuda(&native).map_err(|error| {
9440 IdxTensorError::operation(
9441 "context-scoped scaling",
9442 anyhow::Error::new(error).context("scalar upload failed"),
9443 )
9444 })?,
9445 #[cfg(not(feature = "tenferro-cuda"))]
9447 #[allow(unreachable_patterns)]
9448 _ => return Err(unsupported_execution_context_error()),
9449 };
9450 IdxTensor::untracked_inner(resident, runtime).map_err(|error| {
9451 IdxTensorError::operation(
9452 "context-scoped scaling",
9453 anyhow::Error::new(error).context("scalar wrapping failed"),
9454 )
9455 })
9456 }
9457
9458 fn context_eager_runtime(
9460 context: &ExecutionContext,
9461 ) -> std::result::Result<Arc<EagerRuntime>, IdxTensorError> {
9462 match context {
9463 ExecutionContext::Cpu(context) => context.eager_runtime().map_err(|error| {
9464 IdxTensorError::operation("context-scoped scaling", anyhow::Error::new(error))
9465 }),
9466 #[cfg(feature = "tenferro-cuda")]
9467 ExecutionContext::Cuda(context) => context.eager_runtime().map_err(|error| {
9468 IdxTensorError::operation("context-scoped scaling", anyhow::Error::new(error))
9469 }),
9470 #[cfg(not(feature = "tenferro-cuda"))]
9472 #[allow(unreachable_patterns)]
9473 _ => Err(unsupported_execution_context_error()),
9474 }
9475 }
9476
9477 pub fn norm_in(&self, context: &ExecutionContext) -> std::result::Result<f64, IdxTensorError> {
9491 self.validate_context(context)?;
9492 #[cfg(feature = "tenferro-cuda")]
9493 if matches!(context, ExecutionContext::Cuda(_)) {
9494 let inner = self.cuda_eager_inner().ok_or_else(|| {
9495 IdxTensorError::operation(
9496 "context-scoped norm",
9497 anyhow::anyhow!("tensor has no resident eager value"),
9498 )
9499 })?;
9500 let rank = inner.shape().len();
9501 let axes: Vec<usize> = (0..rank).collect();
9502 let scalar = with_eager_session(inner.runtime(), Some(context), |session| {
9504 let magnitudes = session.abs(inner)?;
9505 let squared = session.reduce_sum_squares(&magnitudes, Some(&axes))?;
9506 session.reshape(&squared, [1])
9507 })
9508 .map_err(|error| {
9509 IdxTensorError::operation("context-scoped norm", anyhow::anyhow!("{error}"))
9510 })?;
9511 let values = Self::read_resident_vector(&scalar, 1, context).map_err(|error| {
9512 IdxTensorError::operation("context-scoped norm", anyhow::Error::new(error))
9513 })?;
9514 return Ok(values[0].sqrt());
9515 }
9516 #[cfg(not(feature = "tenferro-cuda"))]
9517 let _ = context;
9518 self.norm()
9519 }
9520}
9521
9522pub(crate) fn checked_product(dims: &[usize]) -> Result<usize> {
9523 dims.iter().try_fold(1usize, |acc, &dim| {
9524 acc.checked_mul(dim)
9525 .ok_or_else(|| anyhow::anyhow!("dimension product overflow"))
9526 })
9527}
9528
9529fn increment_col_major_coordinate(coords: &mut [usize], dims: &[usize]) {
9530 let mut carry = true;
9531 for (coordinate, &dim) in coords.iter_mut().zip(dims.iter()) {
9532 if !carry {
9533 break;
9534 }
9535 *coordinate += 1;
9536 if *coordinate == dim {
9537 *coordinate = 0;
9538 } else {
9539 carry = false;
9540 }
9541 }
9542}
9543
9544fn map_payload_support_coordinate(
9545 source_axis_classes: &[usize],
9546 target_axis_classes: &[usize],
9547 source_to_target_axes: &[usize],
9548 source_coords: &[usize],
9549 target_coords: &mut [usize],
9550 target_seen: &mut [bool],
9551) -> bool {
9552 if source_axis_classes.len() != source_to_target_axes.len()
9553 || target_coords.len() != target_seen.len()
9554 {
9555 return false;
9556 }
9557 target_seen.fill(false);
9558 for (source_axis, &target_axis) in source_to_target_axes.iter().enumerate() {
9559 let Some(&source_class) = source_axis_classes.get(source_axis) else {
9560 return false;
9561 };
9562 let Some(&target_class) = target_axis_classes.get(target_axis) else {
9563 return false;
9564 };
9565 let Some(&source_value) = source_coords.get(source_class) else {
9566 return false;
9567 };
9568 let Some(target_value) = target_coords.get_mut(target_class) else {
9569 return false;
9570 };
9571 if target_seen[target_class] {
9572 if *target_value != source_value {
9573 return false;
9574 }
9575 } else {
9576 *target_value = source_value;
9577 target_seen[target_class] = true;
9578 }
9579 }
9580 target_seen.iter().all(|&seen| seen)
9581}
9582
9583fn decode_col_major_linear_into(linear: usize, dims: &[usize], out: &mut Vec<usize>) {
9592 out.clear();
9593 let mut remaining = linear;
9594 for &dim in dims {
9595 out.push(remaining % dim);
9596 remaining /= dim;
9597 }
9598}
9599
9600fn encode_col_major_linear(indices: &[usize], dims: &[usize]) -> Result<usize> {
9601 if !(indices.len() == dims.len()) {
9602 return Err(anyhow::anyhow!(
9603 "index rank {} does not match dims {:?}",
9604 indices.len(),
9605 dims
9606 ));
9607 };
9608 let mut linear = 0usize;
9609 let mut stride = 1usize;
9610 for (&index, &dim) in indices.iter().zip(dims.iter()) {
9611 if !(index < dim) {
9612 return Err(anyhow::anyhow!(
9613 "index {} out of bounds for dimension {}",
9614 index,
9615 dim
9616 ));
9617 };
9618 let term = index
9619 .checked_mul(stride)
9620 .ok_or_else(|| anyhow::anyhow!("linear offset overflow"))?;
9621 linear = linear
9622 .checked_add(term)
9623 .ok_or_else(|| anyhow::anyhow!("linear offset overflow"))?;
9624 stride = stride
9625 .checked_mul(dim)
9626 .ok_or_else(|| anyhow::anyhow!("stride overflow"))?;
9627 }
9628 Ok(linear)
9629}
9630
9631pub(crate) enum SessionDenseOperand<'a> {
9637 Storage(Arc<Storage>, Vec<usize>),
9639 Read(Box<TensorRead<'a>>, usize),
9644}
9645
9646pub(crate) const TRACKED_SESSION_OPERAND_REFUSAL: &str =
9653 "a tracked value cannot enter the plain explicit-session route; detach it explicitly before the region";
9654
9655pub(crate) fn validate_permutation(rank: usize, perm: &[usize]) -> std::result::Result<(), String> {
9660 if perm.len() != rank {
9661 return Err("permutation length must match tensor rank".to_string());
9662 }
9663 let mut seen = HashSet::new();
9664 for &axis in perm {
9665 if axis >= rank {
9666 return Err(format!("permutation axis {axis} out of range"));
9667 }
9668 if !seen.insert(axis) {
9669 return Err(format!("duplicate axis {axis} in permutation"));
9670 }
9671 }
9672 Ok(())
9673}
9674
9675impl IdxTensor {
9676 pub(crate) fn session_dense_operand(
9683 &self,
9684 operation: &'static str,
9685 ) -> std::result::Result<SessionDenseOperand<'_>, IdxTensorError> {
9686 let refuse = |reason: String| IdxTensorError::operation(operation, anyhow::anyhow!(reason));
9687 if let IdxTensorStorage::Deferred { error, .. } = &self.storage {
9691 return Err(IdxTensorError::Storage {
9692 source: error.as_ref().clone(),
9693 });
9694 }
9695 if self.tracks_grad() {
9696 return Err(refuse(TRACKED_SESSION_OPERAND_REFUSAL.to_string()));
9697 }
9698 if self.is_cuda_resident() {
9699 return Err(refuse(
9700 "a device-resident operand needs the resident plan route, not the host session route"
9701 .to_string(),
9702 ));
9703 }
9704 let rank = self.indices.len();
9705 let dense = Self::dense_axis_classes(rank);
9706 let dims = self.dims();
9707 match &self.storage {
9708 IdxTensorStorage::Materialized(storage) => {
9709 if storage.axis_classes() != dense.as_slice() {
9710 return Err(refuse(
9711 "a non-dense (diagonal or structured) payload is not served by this slice"
9712 .to_string(),
9713 ));
9714 }
9715 Ok(SessionDenseOperand::Storage(storage.clone(), dims))
9716 }
9717 IdxTensorStorage::Eager {
9718 inner,
9719 axis_classes,
9720 } => {
9721 if axis_classes.as_slice() != dense.as_slice() {
9722 return Err(refuse(
9723 "a non-dense (diagonal or structured) payload is not served by this slice"
9724 .to_string(),
9725 ));
9726 }
9727 match inner.dtype() {
9728 DType::F64 | DType::C64 | DType::F32 | DType::C32 => Ok(
9729 SessionDenseOperand::Read(Box::new(inner.tensor_read()), rank),
9730 ),
9731 other => Err(refuse(format!(
9732 "this numerical route accepts floating-point operands, not {other:?}"
9733 ))),
9734 }
9735 }
9736 IdxTensorStorage::Compact(_) => Err(refuse(
9737 "a compact structured payload is not served by this slice".to_string(),
9738 )),
9739 IdxTensorStorage::Deferred { error, .. } => Err(IdxTensorError::Storage {
9740 source: error.as_ref().clone(),
9741 }),
9742 }
9743 }
9744
9745 pub(crate) fn aligned_identity_metadata(&self, indices: &[DynIndex]) -> Self {
9751 Self {
9752 indices: indices.to_vec(),
9753 storage: self.storage.clone(),
9754 eager_cache: Arc::clone(&self.eager_cache),
9755 }
9756 }
9757
9758 pub(crate) fn has_unique_indices(&self) -> bool {
9764 Self::indices_are_unique(&self.indices)
9765 }
9766
9767 fn indices_are_unique(indices: &[DynIndex]) -> bool {
9773 const QUADRATIC_SCAN_MAX: usize = 16;
9774 if indices.len() <= QUADRATIC_SCAN_MAX {
9775 indices
9776 .iter()
9777 .enumerate()
9778 .all(|(position, index)| !indices[..position].contains(index))
9779 } else {
9780 let mut seen = HashSet::with_capacity(indices.len());
9781 indices.iter().all(|index| seen.insert(index))
9782 }
9783 }
9784
9785 pub(crate) fn from_host_dense<T: tensor4all_tensorbackend::StorageScalar>(
9792 indices: Vec<DynIndex>,
9793 data: Vec<T>,
9794 ) -> std::result::Result<Self, IdxTensorError> {
9795 let dims = Self::expected_dims_from_indices(&indices);
9796 Self::validate_indices(&indices)?;
9797 Self::validate_dense_payload_len(data.len(), &dims)?;
9798 let storage = Storage::from_dense_col_major(data, &dims)
9799 .map_err(|error| IdxTensorError::operation("host publication", error))?;
9800 Self::from_storage(indices, Arc::new(storage))
9801 }
9802
9803 pub fn from_dense_compact<T: tensor4all_tensorbackend::TensorElement>(
9828 indices: Vec<DynIndex>,
9829 data: Vec<T>,
9830 ) -> std::result::Result<Self, IdxTensorError> {
9831 let dims = Self::expected_dims_from_indices(&indices);
9832 Self::validate_indices(&indices)?;
9833 Self::validate_dense_payload_len(data.len(), &dims)?;
9834 let native = T::dense_native_tensor_from_col_major_owned(data, &dims)
9835 .map_err(|error| IdxTensorError::operation("host publication", error))?;
9836 Self::publish_dense_session_result_owned(indices, native)
9837 }
9838
9839 pub(crate) fn gather_dense_host(
9844 &self,
9845 axis: usize,
9846 positions: &[usize],
9847 target_index: DynIndex,
9848 ) -> std::result::Result<Self, IdxTensorError> {
9849 let mut result_indices = self.indices.clone();
9850 let target_dim = target_index.dim();
9851 if let Some(slot) = result_indices.get_mut(axis) {
9852 *slot = target_index;
9853 }
9854 let dims = self.dims();
9855 let dtype = self.storage.dtype().ok_or_else(|| {
9856 IdxTensorError::operation(
9857 "index_select_on_session",
9858 anyhow::anyhow!("unable to determine the source payload dtype"),
9859 )
9860 })?;
9861 match dtype {
9862 DType::F64 => {
9863 let gathered = self.with_dense_host_slice::<f64, _>(|data| {
9864 Self::gather_host(data, &dims, axis, positions, target_dim)
9865 })??;
9866 Self::from_host_dense(result_indices, gathered)
9867 }
9868 DType::F32 => {
9869 let gathered = self.with_dense_host_slice::<f32, _>(|data| {
9870 Self::gather_host(data, &dims, axis, positions, target_dim)
9871 })??;
9872 Self::from_host_dense(result_indices, gathered)
9873 }
9874 DType::C64 => {
9875 let gathered = self.with_dense_host_slice::<Complex64, _>(|data| {
9876 Self::gather_host(data, &dims, axis, positions, target_dim)
9877 })??;
9878 Self::from_host_dense(result_indices, gathered)
9879 }
9880 DType::C32 => {
9881 let gathered = self.with_dense_host_slice::<Complex32, _>(|data| {
9882 Self::gather_host(data, &dims, axis, positions, target_dim)
9883 })??;
9884 Self::from_host_dense(result_indices, gathered)
9885 }
9886 other => Err(IdxTensorError::operation(
9887 "index_select_on_session",
9888 anyhow::anyhow!("unsupported dense storage scalar type {other:?}"),
9889 )),
9890 }
9891 }
9892
9893 fn gather_host<T: tensor4all_tensorbackend::StorageScalar>(
9894 source: &[T],
9895 dims: &[usize],
9896 axis: usize,
9897 positions: &[usize],
9898 target_dim: usize,
9899 ) -> Result<Vec<T>> {
9900 let mut result_dims = dims.to_vec();
9901 if let Some(slot) = result_dims.get_mut(axis) {
9902 *slot = target_dim;
9903 }
9904 let len = checked_product(&result_dims)?;
9905 let strides = Self::col_major_strides(dims)?;
9906 let mut output = Vec::with_capacity(len);
9907 for linear in 0..len {
9908 let mut coordinates = Self::col_major_coordinates(linear, &result_dims);
9909 let position = positions
9910 .get(coordinates[axis])
9911 .copied()
9912 .ok_or_else(|| anyhow::anyhow!("gather position is out of bounds"))?;
9913 coordinates[axis] = position;
9914 let offset = Self::col_major_offset(&coordinates, &strides)?;
9915 output.push(*source.get(offset).ok_or_else(|| {
9916 anyhow::anyhow!("column-major gather read offset {offset} is out of bounds")
9917 })?);
9918 }
9919 Ok(output)
9920 }
9921
9922 pub(crate) fn publish_dense_session_result(
9937 indices: Vec<DynIndex>,
9938 value: &NativeTensor,
9939 ) -> std::result::Result<Self, IdxTensorError> {
9940 let dims: Vec<usize> = value.shape().to_vec();
9941 let storage = match value.dtype() {
9942 DType::F32 => value
9943 .as_slice::<f32>()
9944 .map(|data| Storage::from_dense_col_major(data.to_vec(), &dims)),
9945 DType::F64 => value
9946 .as_slice::<f64>()
9947 .map(|data| Storage::from_dense_col_major(data.to_vec(), &dims)),
9948 DType::C32 => value
9949 .as_slice::<Complex32>()
9950 .map(|data| Storage::from_dense_col_major(data.to_vec(), &dims)),
9951 DType::C64 => value
9952 .as_slice::<Complex64>()
9953 .map(|data| Storage::from_dense_col_major(data.to_vec(), &dims)),
9954 other => return Err(Self::unsupported_session_dtype(other)),
9955 }
9956 .map_err(|error| IdxTensorError::operation("host read", anyhow::Error::new(error)))?
9957 .map_err(IdxTensorError::from)?;
9958 Self::from_storage(indices, Arc::new(storage))
9959 }
9960
9961 fn unsupported_session_dtype(dtype: DType) -> IdxTensorError {
9962 IdxTensorError::operation(
9963 "contract_pair_on_session",
9964 anyhow::anyhow!(format!(
9965 "the explicit-session route publishes floating-point results, not {dtype:?}"
9966 )),
9967 )
9968 }
9969
9970 pub(crate) fn publish_dense_session_result_owned(
9979 indices: Vec<DynIndex>,
9980 value: NativeTensor,
9981 ) -> std::result::Result<Self, IdxTensorError> {
9982 macro_rules! take_storage {
9983 ($scalar:ty) => {
9984 match value.into_vec_col_major::<$scalar>() {
9985 Ok((dims, data)) => {
9986 let storage = Storage::from_dense_col_major(data, &dims)
9987 .map_err(IdxTensorError::from)?;
9988 (dims, storage)
9989 }
9990 Err(failure) => {
9991 return Self::publish_dense_session_result(indices, &failure.into_owner());
9992 }
9993 }
9994 };
9995 }
9996 let (dims, storage) = match value.dtype() {
9997 DType::F32 => take_storage!(f32),
9998 DType::F64 => take_storage!(f64),
9999 DType::C32 => take_storage!(Complex32),
10000 DType::C64 => take_storage!(Complex64),
10001 other => return Err(Self::unsupported_session_dtype(other)),
10002 };
10003 let shape_matches = dims.len() == indices.len()
10004 && indices
10005 .iter()
10006 .zip(&dims)
10007 .all(|(index, &dim)| index.dim() == dim);
10008 if !shape_matches || !Self::indices_are_unique(&indices) {
10009 return Self::from_storage(indices, Arc::new(storage));
10011 }
10012 debug_assert!(Self::validate_indices(&indices).is_ok());
10013 debug_assert!(Self::validate_storage_matches_indices(&indices, &storage).is_ok());
10014 Ok(Self {
10015 indices,
10016 storage: IdxTensorStorage::from_storage(Arc::new(storage)),
10017 eager_cache: Self::empty_eager_cache(),
10018 })
10019 }
10020}
10021
10022#[cfg(test)]
10023mod tests;