1use tenferro_tensor::{
4 DType, DotGeneralAccumulation, Engine, Tensor, TensorRead, TensorScalar, TensorWrite,
5 TypedTensor, TypedTensorView, TypedTensorWrite,
6};
7use tinyvec::TinyVec;
8
9use crate::binary_dot::BinaryDotOperandOrder;
10use crate::eager::{
11 binary_dot_config_for_into, binary_dot_plan_for_shapes, eager_einsum_exec,
12 eager_einsum_exec_read, eager_einsum_exec_read_into, eager_einsum_exec_read_into_accum,
13 eager_einsum_read_subscripts_on_session, eager_einsum_subscripts_on_session,
14 execute_binary_dot_read_into, execute_binary_dot_read_into_accum, plan_subscripts,
15};
16use crate::ellipsis::resolve_einsum_notation;
17use crate::TensorDotAxes;
18use crate::{
19 parse_einsum_notation, ContractionTree, EinsumNotation, EinsumSubscripts, Error, Result,
20 Subscripts,
21};
22
23const TENSOR_EINSUM_INTO_OP: &str = "TensorEinsumIntoExt::einsum_into";
24const TENSOR_READ_EINSUM_INTO_OP: &str = "TensorReadEinsumIntoExt::einsum_read_into";
25const TYPED_TENSOR_EINSUM_OP: &str = "TypedTensorEinsumExt::einsum";
26const TYPED_TENSOR_EINSUM_INTO_OP: &str = "TypedTensorEinsumIntoExt::einsum_into";
27const TYPED_TENSOR_READ_EINSUM_OP: &str = "TypedTensorReadEinsumExt::einsum_read";
28const TYPED_TENSOR_READ_EINSUM_INTO_OP: &str = "TypedTensorReadEinsumIntoExt::einsum_read_into";
29const PLAN_EXECUTE_OP: &str = "ConcreteEinsumPlan::execute";
30const TYPED_TENSOR_TENSORDOT_OP: &str = "TypedTensorTensordotExt::tensordot";
31
32pub trait TensorTensordotExt {
34 fn tensordot(
43 &self,
44 rhs: &Tensor,
45 axes: TensorDotAxes<'_>,
46 session: &mut dyn Engine,
47 ) -> Result<Tensor>;
48}
49
50impl TensorTensordotExt for Tensor {
51 fn tensordot(
52 &self,
53 rhs: &Tensor,
54 axes: TensorDotAxes<'_>,
55 session: &mut dyn Engine,
56 ) -> Result<Tensor> {
57 let config =
58 crate::tensordot::dot_general_config(axes, self.shape().len(), rhs.shape().len())?;
59 crate::tensordot::validate_concrete_contract_dims(self.shape(), rhs.shape(), &config)?;
60 session
61 .dot_general_read(
62 TensorRead::from_tensor(self),
63 TensorRead::from_tensor(rhs),
64 &config,
65 )
66 .map_err(Error::from)
67 }
68}
69
70pub trait TypedTensorTensordotExt<T: TensorScalar> {
72 fn tensordot(
81 &self,
82 rhs: &TypedTensor<T>,
83 axes: TensorDotAxes<'_>,
84 session: &mut dyn Engine,
85 ) -> Result<TypedTensor<T>>;
86}
87
88impl<T: TensorScalar> TypedTensorTensordotExt<T> for TypedTensor<T> {
89 fn tensordot(
90 &self,
91 rhs: &TypedTensor<T>,
92 axes: TensorDotAxes<'_>,
93 session: &mut dyn Engine,
94 ) -> Result<TypedTensor<T>> {
95 let config =
96 crate::tensordot::dot_general_config(axes, self.shape().len(), rhs.shape().len())?;
97 crate::tensordot::validate_concrete_contract_dims(self.shape(), rhs.shape(), &config)?;
98 let result = session
99 .dot_general_read(T::tensor_read(self), T::tensor_read(rhs), &config)
100 .map_err(Error::from)?;
101 into_typed_result(result, TYPED_TENSOR_TENSORDOT_OP)
102 }
103}
104
105pub trait TensorEinsumExt {
129 fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor>;
137
138 fn einsum_notation(
152 &self,
153 notation: &EinsumNotation,
154 session: &mut dyn Engine,
155 ) -> Result<Tensor>;
156
157 fn einsum_subscripts(
164 &self,
165 subscripts: &EinsumSubscripts,
166 session: &mut dyn Engine,
167 ) -> Result<Tensor>;
168}
169
170impl TensorEinsumExt for [&Tensor] {
171 fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor> {
172 let notation = parse_einsum_notation(subscripts)?;
173 self.einsum_notation(¬ation, session)
174 }
175
176 fn einsum_notation(
177 &self,
178 notation: &EinsumNotation,
179 session: &mut dyn Engine,
180 ) -> Result<Tensor> {
181 let subscripts = resolve_tensor_notation(self, notation)?;
182 eager_einsum_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
183 }
184
185 fn einsum_subscripts(
186 &self,
187 subscripts: &EinsumSubscripts,
188 session: &mut dyn Engine,
189 ) -> Result<Tensor> {
190 let subscripts = Subscripts::from(subscripts);
191 eager_einsum_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
192 }
193}
194
195impl<const N: usize> TensorEinsumExt for [&Tensor; N] {
196 fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor> {
197 self.as_slice().einsum(subscripts, session)
198 }
199
200 fn einsum_notation(
201 &self,
202 notation: &EinsumNotation,
203 session: &mut dyn Engine,
204 ) -> Result<Tensor> {
205 self.as_slice().einsum_notation(notation, session)
206 }
207
208 fn einsum_subscripts(
209 &self,
210 subscripts: &EinsumSubscripts,
211 session: &mut dyn Engine,
212 ) -> Result<Tensor> {
213 self.as_slice().einsum_subscripts(subscripts, session)
214 }
215}
216
217pub trait TensorEinsumIntoExt {
219 fn einsum_into(
227 &self,
228 subscripts: &str,
229 session: &mut dyn Engine,
230 out: TensorWrite<'_>,
231 ) -> Result<()>;
232
233 fn einsum_into_notation(
247 &self,
248 notation: &EinsumNotation,
249 session: &mut dyn Engine,
250 out: TensorWrite<'_>,
251 ) -> Result<()>;
252
253 fn einsum_into_subscripts(
261 &self,
262 subscripts: &EinsumSubscripts,
263 session: &mut dyn Engine,
264 out: TensorWrite<'_>,
265 ) -> Result<()>;
266}
267
268impl TensorEinsumIntoExt for [&Tensor] {
269 fn einsum_into(
270 &self,
271 subscripts: &str,
272 session: &mut dyn Engine,
273 out: TensorWrite<'_>,
274 ) -> Result<()> {
275 if let ([lhs, rhs], Some((a, b, c))) = (self, parse_fast_ascii_binary_labels(subscripts)) {
276 let reads = [TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs)];
277 if let Some((order, config)) = read_binary_dot_config_for_labels(&reads, a, b, c, &out)
278 {
279 return execute_binary_dot_config_read_into(session, &reads, order, &config, out);
280 }
281 }
282 let notation = parse_einsum_notation(subscripts)?;
283 self.einsum_into_notation(¬ation, session, out)
284 }
285
286 fn einsum_into_notation(
287 &self,
288 notation: &EinsumNotation,
289 session: &mut dyn Engine,
290 out: TensorWrite<'_>,
291 ) -> Result<()> {
292 let subscripts = resolve_tensor_notation(self, notation)?;
293 tensor_einsum_into_subscripts(session, self, &subscripts, out, TENSOR_EINSUM_INTO_OP)
294 }
295
296 fn einsum_into_subscripts(
297 &self,
298 subscripts: &EinsumSubscripts,
299 session: &mut dyn Engine,
300 out: TensorWrite<'_>,
301 ) -> Result<()> {
302 let subscripts = Subscripts::from(subscripts);
303 tensor_einsum_into_subscripts(session, self, &subscripts, out, TENSOR_EINSUM_INTO_OP)
304 }
305}
306
307impl<const N: usize> TensorEinsumIntoExt for [&Tensor; N] {
308 fn einsum_into(
309 &self,
310 subscripts: &str,
311 session: &mut dyn Engine,
312 out: TensorWrite<'_>,
313 ) -> Result<()> {
314 self.as_slice().einsum_into(subscripts, session, out)
315 }
316
317 fn einsum_into_notation(
318 &self,
319 notation: &EinsumNotation,
320 session: &mut dyn Engine,
321 out: TensorWrite<'_>,
322 ) -> Result<()> {
323 self.as_slice().einsum_into_notation(notation, session, out)
324 }
325
326 fn einsum_into_subscripts(
327 &self,
328 subscripts: &EinsumSubscripts,
329 session: &mut dyn Engine,
330 out: TensorWrite<'_>,
331 ) -> Result<()> {
332 self.as_slice()
333 .einsum_into_subscripts(subscripts, session, out)
334 }
335}
336
337pub trait TypedTensorEinsumExt<T: TensorScalar> {
360 fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<TypedTensor<T>>;
368
369 fn einsum_notation(
383 &self,
384 notation: &EinsumNotation,
385 session: &mut dyn Engine,
386 ) -> Result<TypedTensor<T>>;
387
388 fn einsum_subscripts(
395 &self,
396 subscripts: &EinsumSubscripts,
397 session: &mut dyn Engine,
398 ) -> Result<TypedTensor<T>>;
399}
400
401impl<T: TensorScalar> TypedTensorEinsumExt<T> for [&TypedTensor<T>] {
402 fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<TypedTensor<T>> {
403 let notation = parse_einsum_notation(subscripts)?;
404 self.einsum_notation(¬ation, session)
405 }
406
407 fn einsum_notation(
408 &self,
409 notation: &EinsumNotation,
410 session: &mut dyn Engine,
411 ) -> Result<TypedTensor<T>> {
412 let subscripts = resolve_typed_notation(self, notation)?;
413 typed_einsum_subscripts(session, self, &subscripts, TYPED_TENSOR_EINSUM_OP)
414 }
415
416 fn einsum_subscripts(
417 &self,
418 subscripts: &EinsumSubscripts,
419 session: &mut dyn Engine,
420 ) -> Result<TypedTensor<T>> {
421 let subscripts = Subscripts::from(subscripts);
422 typed_einsum_subscripts(session, self, &subscripts, TYPED_TENSOR_EINSUM_OP)
423 }
424}
425
426impl<T: TensorScalar, const N: usize> TypedTensorEinsumExt<T> for [&TypedTensor<T>; N] {
427 fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<TypedTensor<T>> {
428 self.as_slice().einsum(subscripts, session)
429 }
430
431 fn einsum_notation(
432 &self,
433 notation: &EinsumNotation,
434 session: &mut dyn Engine,
435 ) -> Result<TypedTensor<T>> {
436 self.as_slice().einsum_notation(notation, session)
437 }
438
439 fn einsum_subscripts(
440 &self,
441 subscripts: &EinsumSubscripts,
442 session: &mut dyn Engine,
443 ) -> Result<TypedTensor<T>> {
444 self.as_slice().einsum_subscripts(subscripts, session)
445 }
446}
447
448pub trait TypedTensorReadEinsumExt<T: TensorScalar> {
471 fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<TypedTensor<T>>;
495
496 fn einsum_read_notation(
510 &self,
511 notation: &EinsumNotation,
512 session: &mut dyn Engine,
513 ) -> Result<TypedTensor<T>>;
514
515 fn einsum_read_subscripts(
540 &self,
541 subscripts: &EinsumSubscripts,
542 session: &mut dyn Engine,
543 ) -> Result<TypedTensor<T>>;
544}
545
546impl<'a, T: TensorScalar> TypedTensorReadEinsumExt<T> for [TypedTensorView<'a, T>] {
547 fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<TypedTensor<T>> {
548 let notation = parse_einsum_notation(subscripts)?;
549 self.einsum_read_notation(¬ation, session)
550 }
551
552 fn einsum_read_notation(
553 &self,
554 notation: &EinsumNotation,
555 session: &mut dyn Engine,
556 ) -> Result<TypedTensor<T>> {
557 let subscripts = resolve_view_notation(self, notation)?;
558 typed_view_einsum_subscripts(session, self, &subscripts, TYPED_TENSOR_READ_EINSUM_OP)
559 }
560
561 fn einsum_read_subscripts(
562 &self,
563 subscripts: &EinsumSubscripts,
564 session: &mut dyn Engine,
565 ) -> Result<TypedTensor<T>> {
566 let subscripts = Subscripts::from(subscripts);
567 typed_view_einsum_subscripts(session, self, &subscripts, TYPED_TENSOR_READ_EINSUM_OP)
568 }
569}
570
571impl<'a, T: TensorScalar, const N: usize> TypedTensorReadEinsumExt<T>
572 for [TypedTensorView<'a, T>; N]
573{
574 fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<TypedTensor<T>> {
575 self.as_slice().einsum_read(subscripts, session)
576 }
577
578 fn einsum_read_notation(
579 &self,
580 notation: &EinsumNotation,
581 session: &mut dyn Engine,
582 ) -> Result<TypedTensor<T>> {
583 self.as_slice().einsum_read_notation(notation, session)
584 }
585
586 fn einsum_read_subscripts(
587 &self,
588 subscripts: &EinsumSubscripts,
589 session: &mut dyn Engine,
590 ) -> Result<TypedTensor<T>> {
591 self.as_slice().einsum_read_subscripts(subscripts, session)
592 }
593}
594
595pub trait TypedTensorEinsumIntoExt<T: TensorScalar> {
597 fn einsum_into<'out, O>(
605 &self,
606 subscripts: &str,
607 session: &mut dyn Engine,
608 out: O,
609 ) -> Result<()>
610 where
611 O: Into<TypedTensorWrite<'out, T>>;
612
613 fn einsum_into_notation<'out, O>(
627 &self,
628 notation: &EinsumNotation,
629 session: &mut dyn Engine,
630 out: O,
631 ) -> Result<()>
632 where
633 O: Into<TypedTensorWrite<'out, T>>;
634
635 fn einsum_into_subscripts<'out, O>(
643 &self,
644 subscripts: &EinsumSubscripts,
645 session: &mut dyn Engine,
646 out: O,
647 ) -> Result<()>
648 where
649 O: Into<TypedTensorWrite<'out, T>>;
650}
651
652impl<T: TensorScalar> TypedTensorEinsumIntoExt<T> for [&TypedTensor<T>] {
653 fn einsum_into<'out, O>(&self, subscripts: &str, session: &mut dyn Engine, out: O) -> Result<()>
654 where
655 O: Into<TypedTensorWrite<'out, T>>,
656 {
657 let out = out.into().into_tensor_write();
658 if let ([lhs, rhs], Some((a, b, c))) = (self, parse_fast_ascii_binary_labels(subscripts)) {
659 let reads = [T::tensor_read(lhs), T::tensor_read(rhs)];
660 if let Some((order, config)) = read_binary_dot_config_for_labels(&reads, a, b, c, &out)
661 {
662 return execute_binary_dot_config_read_into(session, &reads, order, &config, out);
663 }
664 }
665 let notation = parse_einsum_notation(subscripts)?;
666 let subscripts = resolve_typed_notation(self, ¬ation)?;
667 typed_einsum_into_subscripts(session, self, &subscripts, out, TYPED_TENSOR_EINSUM_INTO_OP)
668 }
669
670 fn einsum_into_notation<'out, O>(
671 &self,
672 notation: &EinsumNotation,
673 session: &mut dyn Engine,
674 out: O,
675 ) -> Result<()>
676 where
677 O: Into<TypedTensorWrite<'out, T>>,
678 {
679 let subscripts = resolve_typed_notation(self, notation)?;
680 typed_einsum_into_subscripts(
681 session,
682 self,
683 &subscripts,
684 out.into().into_tensor_write(),
685 TYPED_TENSOR_EINSUM_INTO_OP,
686 )
687 }
688
689 fn einsum_into_subscripts<'out, O>(
690 &self,
691 subscripts: &EinsumSubscripts,
692 session: &mut dyn Engine,
693 out: O,
694 ) -> Result<()>
695 where
696 O: Into<TypedTensorWrite<'out, T>>,
697 {
698 let subscripts = Subscripts::from(subscripts);
699 typed_einsum_into_subscripts(
700 session,
701 self,
702 &subscripts,
703 out.into().into_tensor_write(),
704 TYPED_TENSOR_EINSUM_INTO_OP,
705 )
706 }
707}
708
709impl<T: TensorScalar, const N: usize> TypedTensorEinsumIntoExt<T> for [&TypedTensor<T>; N] {
710 fn einsum_into<'out, O>(&self, subscripts: &str, session: &mut dyn Engine, out: O) -> Result<()>
711 where
712 O: Into<TypedTensorWrite<'out, T>>,
713 {
714 self.as_slice().einsum_into(subscripts, session, out)
715 }
716
717 fn einsum_into_notation<'out, O>(
718 &self,
719 notation: &EinsumNotation,
720 session: &mut dyn Engine,
721 out: O,
722 ) -> Result<()>
723 where
724 O: Into<TypedTensorWrite<'out, T>>,
725 {
726 self.as_slice().einsum_into_notation(notation, session, out)
727 }
728
729 fn einsum_into_subscripts<'out, O>(
730 &self,
731 subscripts: &EinsumSubscripts,
732 session: &mut dyn Engine,
733 out: O,
734 ) -> Result<()>
735 where
736 O: Into<TypedTensorWrite<'out, T>>,
737 {
738 self.as_slice()
739 .einsum_into_subscripts(subscripts, session, out)
740 }
741}
742
743pub trait TypedTensorReadEinsumIntoExt<T: TensorScalar> {
763 fn einsum_read_into<'out, O>(
789 &self,
790 subscripts: &str,
791 session: &mut dyn Engine,
792 out: O,
793 ) -> Result<()>
794 where
795 O: Into<TypedTensorWrite<'out, T>>;
796
797 fn einsum_read_into_notation<'out, O>(
811 &self,
812 notation: &EinsumNotation,
813 session: &mut dyn Engine,
814 out: O,
815 ) -> Result<()>
816 where
817 O: Into<TypedTensorWrite<'out, T>>;
818
819 fn einsum_read_into_subscripts<'out, O>(
846 &self,
847 subscripts: &EinsumSubscripts,
848 session: &mut dyn Engine,
849 out: O,
850 ) -> Result<()>
851 where
852 O: Into<TypedTensorWrite<'out, T>>;
853}
854
855impl<'a, T: TensorScalar> TypedTensorReadEinsumIntoExt<T> for [TypedTensorView<'a, T>] {
856 fn einsum_read_into<'out, O>(
857 &self,
858 subscripts: &str,
859 session: &mut dyn Engine,
860 out: O,
861 ) -> Result<()>
862 where
863 O: Into<TypedTensorWrite<'out, T>>,
864 {
865 let out = out.into().into_tensor_write();
866 if let Some((lhs, rhs, output)) = parse_fast_ascii_binary_labels(subscripts) {
867 if let Some((order, config)) =
868 typed_view_binary_dot_config(self, lhs, rhs, output, &out)
869 {
870 return execute_typed_view_binary_dot_into(session, self, order, &config, out);
871 }
872 }
873 let notation = parse_einsum_notation(subscripts)?;
874 let subscripts = resolve_view_notation(self, ¬ation)?;
875 typed_view_einsum_into_subscripts(
876 session,
877 self,
878 &subscripts,
879 out,
880 TYPED_TENSOR_READ_EINSUM_INTO_OP,
881 )
882 }
883
884 fn einsum_read_into_notation<'out, O>(
885 &self,
886 notation: &EinsumNotation,
887 session: &mut dyn Engine,
888 out: O,
889 ) -> Result<()>
890 where
891 O: Into<TypedTensorWrite<'out, T>>,
892 {
893 let out = out.into().into_tensor_write();
894 if let Some([lhs, rhs, output]) = borrowed_notation_labels(notation) {
895 if let Some((order, config)) =
896 typed_view_binary_dot_config(self, &lhs, &rhs, &output, &out)
897 {
898 return execute_typed_view_binary_dot_into(session, self, order, &config, out);
899 }
900 }
901 let subscripts = resolve_view_notation(self, notation)?;
902 typed_view_einsum_into_subscripts(
903 session,
904 self,
905 &subscripts,
906 out,
907 TYPED_TENSOR_READ_EINSUM_INTO_OP,
908 )
909 }
910
911 fn einsum_read_into_subscripts<'out, O>(
912 &self,
913 subscripts: &EinsumSubscripts,
914 session: &mut dyn Engine,
915 out: O,
916 ) -> Result<()>
917 where
918 O: Into<TypedTensorWrite<'out, T>>,
919 {
920 let out = out.into().into_tensor_write();
921 if let [lhs, rhs] = subscripts.inputs.as_slice() {
922 if let Some((order, config)) =
923 typed_view_binary_dot_config(self, lhs, rhs, &subscripts.output, &out)
924 {
925 return execute_typed_view_binary_dot_into(session, self, order, &config, out);
926 }
927 }
928 let subscripts = Subscripts::from(subscripts);
929 typed_view_einsum_into_subscripts(
930 session,
931 self,
932 &subscripts,
933 out,
934 TYPED_TENSOR_READ_EINSUM_INTO_OP,
935 )
936 }
937}
938
939impl<'a, T: TensorScalar, const N: usize> TypedTensorReadEinsumIntoExt<T>
940 for [TypedTensorView<'a, T>; N]
941{
942 fn einsum_read_into<'out, O>(
943 &self,
944 subscripts: &str,
945 session: &mut dyn Engine,
946 out: O,
947 ) -> Result<()>
948 where
949 O: Into<TypedTensorWrite<'out, T>>,
950 {
951 self.as_slice().einsum_read_into(subscripts, session, out)
952 }
953
954 fn einsum_read_into_notation<'out, O>(
955 &self,
956 notation: &EinsumNotation,
957 session: &mut dyn Engine,
958 out: O,
959 ) -> Result<()>
960 where
961 O: Into<TypedTensorWrite<'out, T>>,
962 {
963 self.as_slice()
964 .einsum_read_into_notation(notation, session, out)
965 }
966
967 fn einsum_read_into_subscripts<'out, O>(
968 &self,
969 subscripts: &EinsumSubscripts,
970 session: &mut dyn Engine,
971 out: O,
972 ) -> Result<()>
973 where
974 O: Into<TypedTensorWrite<'out, T>>,
975 {
976 self.as_slice()
977 .einsum_read_into_subscripts(subscripts, session, out)
978 }
979}
980
981pub trait TensorReadEinsumExt {
1008 fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor>;
1016
1017 fn einsum_read_notation(
1031 &self,
1032 notation: &EinsumNotation,
1033 session: &mut dyn Engine,
1034 ) -> Result<Tensor>;
1035
1036 fn einsum_read_subscripts(
1044 &self,
1045 subscripts: &EinsumSubscripts,
1046 session: &mut dyn Engine,
1047 ) -> Result<Tensor>;
1048}
1049
1050impl<'a> TensorReadEinsumExt for [TensorRead<'a>] {
1051 fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor> {
1052 let notation = parse_einsum_notation(subscripts)?;
1053 self.einsum_read_notation(¬ation, session)
1054 }
1055
1056 fn einsum_read_notation(
1057 &self,
1058 notation: &EinsumNotation,
1059 session: &mut dyn Engine,
1060 ) -> Result<Tensor> {
1061 let subscripts = resolve_read_notation(self, notation)?;
1062 eager_einsum_read_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
1063 }
1064
1065 fn einsum_read_subscripts(
1066 &self,
1067 subscripts: &EinsumSubscripts,
1068 session: &mut dyn Engine,
1069 ) -> Result<Tensor> {
1070 let subscripts = Subscripts::from(subscripts);
1071 eager_einsum_read_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
1072 }
1073}
1074
1075impl<'a, const N: usize> TensorReadEinsumExt for [TensorRead<'a>; N] {
1076 fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor> {
1077 self.as_slice().einsum_read(subscripts, session)
1078 }
1079
1080 fn einsum_read_notation(
1081 &self,
1082 notation: &EinsumNotation,
1083 session: &mut dyn Engine,
1084 ) -> Result<Tensor> {
1085 self.as_slice().einsum_read_notation(notation, session)
1086 }
1087
1088 fn einsum_read_subscripts(
1089 &self,
1090 subscripts: &EinsumSubscripts,
1091 session: &mut dyn Engine,
1092 ) -> Result<Tensor> {
1093 self.as_slice().einsum_read_subscripts(subscripts, session)
1094 }
1095}
1096
1097pub trait TensorReadEinsumIntoExt {
1099 fn einsum_read_into(
1107 &self,
1108 subscripts: &str,
1109 session: &mut dyn Engine,
1110 out: TensorWrite<'_>,
1111 ) -> Result<()>;
1112
1113 fn einsum_read_into_notation(
1127 &self,
1128 notation: &EinsumNotation,
1129 session: &mut dyn Engine,
1130 out: TensorWrite<'_>,
1131 ) -> Result<()>;
1132
1133 fn einsum_read_into_subscripts(
1141 &self,
1142 subscripts: &EinsumSubscripts,
1143 session: &mut dyn Engine,
1144 out: TensorWrite<'_>,
1145 ) -> Result<()>;
1146}
1147
1148impl<'a> TensorReadEinsumIntoExt for [TensorRead<'a>] {
1149 fn einsum_read_into(
1150 &self,
1151 subscripts: &str,
1152 session: &mut dyn Engine,
1153 out: TensorWrite<'_>,
1154 ) -> Result<()> {
1155 if let Some((lhs, rhs, output)) = parse_fast_ascii_binary_labels(subscripts) {
1156 if let Some((order, config)) =
1157 read_binary_dot_config_for_labels(self, lhs, rhs, output, &out)
1158 {
1159 return execute_binary_dot_config_read_into(session, self, order, &config, out);
1160 }
1161 }
1162 let notation = parse_einsum_notation(subscripts)?;
1163 self.einsum_read_into_notation(¬ation, session, out)
1164 }
1165
1166 fn einsum_read_into_notation(
1167 &self,
1168 notation: &EinsumNotation,
1169 session: &mut dyn Engine,
1170 out: TensorWrite<'_>,
1171 ) -> Result<()> {
1172 if let Some([lhs, rhs, output]) = borrowed_notation_labels(notation) {
1173 if let Some((order, config)) =
1174 read_binary_dot_config_for_labels(self, &lhs, &rhs, &output, &out)
1175 {
1176 return execute_binary_dot_config_read_into(session, self, order, &config, out);
1177 }
1178 }
1179 let subscripts = resolve_read_notation(self, notation)?;
1180 tensor_read_einsum_into_subscripts(
1181 session,
1182 self,
1183 &subscripts,
1184 out,
1185 TENSOR_READ_EINSUM_INTO_OP,
1186 )
1187 }
1188
1189 fn einsum_read_into_subscripts(
1190 &self,
1191 subscripts: &EinsumSubscripts,
1192 session: &mut dyn Engine,
1193 out: TensorWrite<'_>,
1194 ) -> Result<()> {
1195 if let [lhs, rhs] = subscripts.inputs.as_slice() {
1196 if let Some((order, config)) =
1197 read_binary_dot_config_for_labels(self, lhs, rhs, &subscripts.output, &out)
1198 {
1199 return execute_binary_dot_config_read_into(session, self, order, &config, out);
1200 }
1201 }
1202 let subscripts = Subscripts::from(subscripts);
1203 tensor_read_einsum_into_subscripts(
1204 session,
1205 self,
1206 &subscripts,
1207 out,
1208 TENSOR_READ_EINSUM_INTO_OP,
1209 )
1210 }
1211}
1212
1213impl<'a, const N: usize> TensorReadEinsumIntoExt for [TensorRead<'a>; N] {
1214 fn einsum_read_into(
1215 &self,
1216 subscripts: &str,
1217 session: &mut dyn Engine,
1218 out: TensorWrite<'_>,
1219 ) -> Result<()> {
1220 self.as_slice().einsum_read_into(subscripts, session, out)
1221 }
1222
1223 fn einsum_read_into_notation(
1224 &self,
1225 notation: &EinsumNotation,
1226 session: &mut dyn Engine,
1227 out: TensorWrite<'_>,
1228 ) -> Result<()> {
1229 self.as_slice()
1230 .einsum_read_into_notation(notation, session, out)
1231 }
1232
1233 fn einsum_read_into_subscripts(
1234 &self,
1235 subscripts: &EinsumSubscripts,
1236 session: &mut dyn Engine,
1237 out: TensorWrite<'_>,
1238 ) -> Result<()> {
1239 self.as_slice()
1240 .einsum_read_into_subscripts(subscripts, session, out)
1241 }
1242}
1243
1244#[derive(Debug)]
1268pub struct ConcreteEinsumPlan {
1269 tree: ContractionTree,
1270 inputs: Vec<ConcreteEinsumInputSpec>,
1271 output_shape: Vec<usize>,
1272 binary_dot: Option<crate::binary_dot::BinaryDotPlan>,
1273 cpu_plan: crate::cpu_concrete::Cache,
1274}
1275
1276impl ConcreteEinsumPlan {
1277 pub fn clear_cpu_cache(&mut self) {
1292 self.cpu_plan.clear();
1293 }
1294
1295 pub fn cpu_cache_stats(&self) -> Result<tenferro_tensor::CacheStats> {
1314 self.cpu_plan.stats().map_err(Error::from)
1315 }
1316
1317 pub fn prepare<'a, I>(inputs: I, subscripts: &str) -> Result<Self>
1326 where
1327 I: AsRef<[&'a Tensor]>,
1328 {
1329 let notation = parse_einsum_notation(subscripts)?;
1330 Self::prepare_notation(inputs, ¬ation)
1331 }
1332
1333 pub fn prepare_subscripts<'a, I>(inputs: I, subscripts: &EinsumSubscripts) -> Result<Self>
1342 where
1343 I: AsRef<[&'a Tensor]>,
1344 {
1345 let subscripts = Subscripts::from(subscripts);
1346 Self::prepare_subscripts_internal(input_specs(inputs.as_ref()), &subscripts)
1347 }
1348
1349 pub fn prepare_notation<'a, I>(inputs: I, notation: &EinsumNotation) -> Result<Self>
1357 where
1358 I: AsRef<[&'a Tensor]>,
1359 {
1360 let inputs = inputs.as_ref();
1361 let subscripts = resolve_tensor_notation(inputs, notation)?;
1362 Self::prepare_subscripts_internal(input_specs(inputs), &subscripts)
1363 }
1364
1365 pub fn prepare_typed<'a, T, I>(inputs: I, subscripts: &str) -> Result<Self>
1373 where
1374 T: TensorScalar,
1375 I: AsRef<[&'a TypedTensor<T>]>,
1376 {
1377 let notation = parse_einsum_notation(subscripts)?;
1378 Self::prepare_typed_notation(inputs, ¬ation)
1379 }
1380
1381 pub fn prepare_typed_subscripts<'a, T, I>(
1389 inputs: I,
1390 subscripts: &EinsumSubscripts,
1391 ) -> Result<Self>
1392 where
1393 T: TensorScalar,
1394 I: AsRef<[&'a TypedTensor<T>]>,
1395 {
1396 let subscripts = Subscripts::from(subscripts);
1397 Self::prepare_subscripts_internal(typed_input_specs(inputs.as_ref()), &subscripts)
1398 }
1399
1400 pub fn prepare_typed_notation<'a, T, I>(inputs: I, notation: &EinsumNotation) -> Result<Self>
1408 where
1409 T: TensorScalar,
1410 I: AsRef<[&'a TypedTensor<T>]>,
1411 {
1412 let inputs = inputs.as_ref();
1413 let subscripts = resolve_typed_notation(inputs, notation)?;
1414 Self::prepare_subscripts_internal(typed_input_specs(inputs), &subscripts)
1415 }
1416
1417 pub fn prepare_read<'a, I>(inputs: I, subscripts: &str) -> Result<Self>
1425 where
1426 I: AsRef<[TensorRead<'a>]>,
1427 {
1428 let notation = parse_einsum_notation(subscripts)?;
1429 Self::prepare_read_notation(inputs, ¬ation)
1430 }
1431
1432 pub fn prepare_read_subscripts<'a, I>(inputs: I, subscripts: &EinsumSubscripts) -> Result<Self>
1441 where
1442 I: AsRef<[TensorRead<'a>]>,
1443 {
1444 let subscripts = Subscripts::from(subscripts);
1445 Self::prepare_subscripts_internal(read_input_specs(inputs.as_ref()), &subscripts)
1446 }
1447
1448 pub fn prepare_read_notation<'a, I>(inputs: I, notation: &EinsumNotation) -> Result<Self>
1456 where
1457 I: AsRef<[TensorRead<'a>]>,
1458 {
1459 let inputs = inputs.as_ref();
1460 let subscripts = resolve_read_notation(inputs, notation)?;
1461 Self::prepare_subscripts_internal(read_input_specs(inputs), &subscripts)
1462 }
1463
1464 pub(crate) fn step_count(&self) -> usize {
1466 self.tree.step_count()
1467 }
1468
1469 pub fn execute<'a, I>(&self, inputs: I, session: &mut dyn Engine) -> Result<Tensor>
1500 where
1501 I: AsRef<[&'a Tensor]>,
1502 {
1503 let inputs = inputs.as_ref();
1504 self.validate_tensor_inputs(inputs, PLAN_EXECUTE_OP)?;
1505 if inputs.len() >= 3 {
1506 let reads: Vec<_> = inputs
1507 .iter()
1508 .map(|tensor| TensorRead::from_tensor(tensor))
1509 .collect();
1510 if let Some(result) = crate::cpu_concrete::execute_cached(
1511 session,
1512 &reads,
1513 &self.tree,
1514 Some(&self.cpu_plan),
1515 ) {
1516 return result.map_err(Error::from);
1517 }
1518 }
1519 eager_einsum_exec(session, inputs, &self.tree).map_err(Error::from)
1520 }
1521
1522 pub fn execute_typed<'a, T, I>(
1553 &self,
1554 inputs: I,
1555 session: &mut dyn Engine,
1556 ) -> Result<TypedTensor<T>>
1557 where
1558 T: TensorScalar,
1559 I: AsRef<[&'a TypedTensor<T>]>,
1560 {
1561 let inputs = inputs.as_ref();
1562 self.validate_typed_inputs(inputs, PLAN_EXECUTE_OP)?;
1563 let reads: Vec<_> = inputs.iter().map(|tensor| T::tensor_read(tensor)).collect();
1564 let result = match crate::cpu_concrete::execute_cached(
1565 session,
1566 &reads,
1567 &self.tree,
1568 Some(&self.cpu_plan),
1569 ) {
1570 Some(result) => result?,
1571 None => eager_einsum_exec_read(session, &reads, &self.tree)?,
1572 };
1573 into_typed_result(result, PLAN_EXECUTE_OP)
1574 }
1575
1576 pub fn execute_read<'a, I>(&self, inputs: I, session: &mut dyn Engine) -> Result<Tensor>
1611 where
1612 I: AsRef<[TensorRead<'a>]>,
1613 {
1614 let inputs = inputs.as_ref();
1615 self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
1616 if let Some(result) =
1617 crate::cpu_concrete::execute_cached(session, inputs, &self.tree, Some(&self.cpu_plan))
1618 {
1619 return result.map_err(Error::from);
1620 }
1621 eager_einsum_exec_read(session, inputs, &self.tree).map_err(Error::from)
1622 }
1623
1624 pub fn execute_into<'a, I>(
1661 &self,
1662 inputs: I,
1663 session: &mut dyn Engine,
1664 out: TensorWrite<'_>,
1665 ) -> Result<()>
1666 where
1667 I: AsRef<[&'a Tensor]>,
1668 {
1669 let inputs = inputs.as_ref();
1670 self.validate_tensor_inputs(inputs, PLAN_EXECUTE_OP)?;
1671 self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
1672 if let Some(binary_dot) = &self.binary_dot {
1673 if let [lhs, rhs] = inputs {
1674 let reads = [TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs)];
1675 return execute_binary_dot_read_into(session, &reads, binary_dot, out)
1676 .map_err(Error::from);
1677 }
1678 }
1679 let reads: Vec<_> = inputs
1680 .iter()
1681 .map(|tensor| TensorRead::from_tensor(tensor))
1682 .collect();
1683 self.execute_cpu_or_fallback_into(session, &reads, out)
1684 .map_err(Error::from)
1685 }
1686
1687 pub fn execute_typed_into<'a, 'out, T, I, O>(
1721 &self,
1722 inputs: I,
1723 session: &mut dyn Engine,
1724 out: O,
1725 ) -> Result<()>
1726 where
1727 T: TensorScalar,
1728 I: AsRef<[&'a TypedTensor<T>]>,
1729 O: Into<TypedTensorWrite<'out, T>>,
1730 {
1731 let inputs = inputs.as_ref();
1732 self.validate_typed_inputs(inputs, PLAN_EXECUTE_OP)?;
1733 let out = out.into().into_tensor_write();
1734 self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
1735 if let Some(binary_dot) = &self.binary_dot {
1736 if let [lhs, rhs] = inputs {
1737 let reads = [T::tensor_read(lhs), T::tensor_read(rhs)];
1738 return execute_binary_dot_read_into(session, &reads, binary_dot, out)
1739 .map_err(Error::from);
1740 }
1741 }
1742 let reads: Vec<_> = inputs.iter().map(|tensor| T::tensor_read(tensor)).collect();
1743 self.execute_cpu_or_fallback_into(session, &reads, out)
1744 .map_err(Error::from)
1745 }
1746
1747 pub fn execute_read_into<'a, I>(
1784 &self,
1785 inputs: I,
1786 session: &mut dyn Engine,
1787 out: TensorWrite<'_>,
1788 ) -> Result<()>
1789 where
1790 I: AsRef<[TensorRead<'a>]>,
1791 {
1792 let inputs = inputs.as_ref();
1793 self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
1794 self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
1795 if let Some(binary_dot) = &self.binary_dot {
1796 return execute_binary_dot_read_into(session, inputs, binary_dot, out)
1797 .map_err(Error::from);
1798 }
1799 self.execute_cpu_or_fallback_into(session, inputs, out)
1800 .map_err(Error::from)
1801 }
1802
1803 pub fn execute_read_into_accum<'a, I>(
1843 &self,
1844 inputs: I,
1845 session: &mut dyn Engine,
1846 accumulation: DotGeneralAccumulation,
1847 out: TensorWrite<'_>,
1848 ) -> Result<()>
1849 where
1850 I: AsRef<[TensorRead<'a>]>,
1851 {
1852 let inputs = inputs.as_ref();
1853 self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
1854 self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
1855 if let Some(binary_dot) = &self.binary_dot {
1856 return execute_binary_dot_read_into_accum(
1857 session,
1858 inputs,
1859 binary_dot,
1860 accumulation,
1861 out,
1862 )
1863 .map_err(Error::from);
1864 }
1865 if crate::cpu_concrete::supports(session, inputs) {
1866 let result = self.execute_read(inputs, session)?;
1867 let accumulation = DotGeneralAccumulation {
1868 lhs_conj: false,
1869 rhs_conj: false,
1870 ..accumulation
1871 };
1872 let mut out = out;
1873 return tenferro_tensor::backend::accumulate_dot_result_into(
1874 &result,
1875 accumulation,
1876 &mut out,
1877 )
1878 .map_err(Error::from);
1879 }
1880 eager_einsum_exec_read_into_accum(session, inputs, &self.tree, accumulation, out)
1881 .map_err(Error::from)
1882 }
1883
1884 fn execute_cpu_or_fallback_into(
1885 &self,
1886 session: &mut dyn Engine,
1887 inputs: &[TensorRead<'_>],
1888 out: TensorWrite<'_>,
1889 ) -> tenferro_tensor::Result<()> {
1890 if crate::cpu_concrete::supports(session, inputs) {
1891 crate::cpu_concrete::execute_into(
1892 session,
1893 inputs,
1894 &self.tree,
1895 Some(&self.cpu_plan),
1896 out,
1897 )
1898 } else {
1899 eager_einsum_exec_read_into(session, inputs, &self.tree, out)
1900 }
1901 }
1902
1903 fn prepare_subscripts_internal(
1904 inputs: Vec<ConcreteEinsumInputSpec>,
1905 subscripts: &Subscripts,
1906 ) -> Result<Self> {
1907 let shapes: Vec<&[usize]> = inputs.iter().map(|input| input.shape.as_slice()).collect();
1908 let binary_dot = binary_dot_plan_for_shapes(&shapes, subscripts);
1909 let tree = plan_subscripts(subscripts, &shapes)?;
1910 let output_shape = tree.output_shape();
1911 Ok(Self {
1912 tree,
1913 inputs,
1914 output_shape,
1915 binary_dot,
1916 cpu_plan: crate::cpu_concrete::Cache::default(),
1917 })
1918 }
1919
1920 fn validate_tensor_inputs(&self, actual: &[&Tensor], op: &'static str) -> Result<()> {
1921 self.validate_input_metadata(
1922 actual.iter().map(|tensor| (tensor.dtype(), tensor.shape())),
1923 op,
1924 )
1925 }
1926
1927 fn validate_typed_inputs<T: TensorScalar>(
1928 &self,
1929 actual: &[&TypedTensor<T>],
1930 op: &'static str,
1931 ) -> Result<()> {
1932 self.validate_input_metadata(actual.iter().map(|tensor| (T::dtype(), tensor.shape())), op)
1933 }
1934
1935 fn validate_read_inputs(&self, actual: &[TensorRead<'_>], op: &'static str) -> Result<()> {
1936 self.validate_input_metadata(
1937 actual.iter().map(|tensor| (tensor.dtype(), tensor.shape())),
1938 op,
1939 )
1940 }
1941
1942 fn validate_input_metadata<'a>(
1943 &self,
1944 actual: impl ExactSizeIterator<Item = (DType, &'a [usize])>,
1945 op: &'static str,
1946 ) -> Result<()> {
1947 if actual.len() != self.inputs.len() {
1948 return Err(Error::invalid_argument(
1949 op,
1950 "inputs",
1951 format!(
1952 "prepared einsum expects {} inputs, got {}",
1953 self.inputs.len(),
1954 actual.len()
1955 ),
1956 ));
1957 }
1958 for (expected, (actual_dtype, actual_shape)) in self.inputs.iter().zip(actual) {
1959 if expected.dtype != actual_dtype {
1960 return Err(Error::dtype_mismatch(op, expected.dtype, actual_dtype));
1961 }
1962 if expected.shape != actual_shape {
1963 return Err(Error::shape_mismatch(
1964 op,
1965 expected.shape.clone(),
1966 actual_shape.to_vec(),
1967 ));
1968 }
1969 }
1970 Ok(())
1971 }
1972
1973 fn validate_cached_output(&self, out: &TensorWrite<'_>, op: &'static str) -> Result<()> {
1974 let dtype = self
1975 .inputs
1976 .first()
1977 .map(|input| input.dtype)
1978 .ok_or_else(|| {
1979 Error::invalid_argument(op, "inputs", "einsum requires at least one input tensor")
1980 })?;
1981 for input in &self.inputs[1..] {
1982 if input.dtype != dtype {
1983 return Err(Error::dtype_mismatch(op, dtype, input.dtype));
1984 }
1985 }
1986 if out.dtype() != dtype {
1987 return Err(Error::dtype_mismatch(op, dtype, out.dtype()));
1988 }
1989 if out.shape() != self.output_shape {
1990 return Err(Error::shape_mismatch(
1991 op,
1992 out.shape().to_vec(),
1993 self.output_shape.clone(),
1994 ));
1995 }
1996 Ok(())
1997 }
1998}
1999
2000#[derive(Clone, Debug)]
2001struct ConcreteEinsumInputSpec {
2002 dtype: DType,
2003 shape: Vec<usize>,
2004}
2005
2006fn resolve_shapes(notation: &EinsumNotation, shapes: Vec<&[usize]>) -> Result<Subscripts> {
2007 resolve_einsum_notation(notation, &shapes)
2008}
2009
2010fn resolve_tensor_notation(inputs: &[&Tensor], notation: &EinsumNotation) -> Result<Subscripts> {
2011 resolve_shapes(
2012 notation,
2013 inputs.iter().map(|tensor| tensor.shape()).collect(),
2014 )
2015}
2016
2017fn resolve_typed_notation<T: TensorScalar>(
2018 inputs: &[&TypedTensor<T>],
2019 notation: &EinsumNotation,
2020) -> Result<Subscripts> {
2021 resolve_shapes(
2022 notation,
2023 inputs.iter().map(|tensor| tensor.shape()).collect(),
2024 )
2025}
2026
2027fn resolve_view_notation<'a, T: TensorScalar>(
2028 inputs: &[TypedTensorView<'a, T>],
2029 notation: &EinsumNotation,
2030) -> Result<Subscripts> {
2031 resolve_shapes(notation, inputs.iter().map(|view| view.shape()).collect())
2032}
2033
2034fn resolve_read_notation<'a>(
2035 inputs: &[TensorRead<'a>],
2036 notation: &EinsumNotation,
2037) -> Result<Subscripts> {
2038 resolve_shapes(notation, inputs.iter().map(|input| input.shape()).collect())
2039}
2040
2041fn input_specs(inputs: &[&Tensor]) -> Vec<ConcreteEinsumInputSpec> {
2042 inputs
2043 .iter()
2044 .map(|tensor| ConcreteEinsumInputSpec {
2045 dtype: tensor.dtype(),
2046 shape: tensor.shape().to_vec(),
2047 })
2048 .collect()
2049}
2050
2051fn typed_input_specs<T: TensorScalar>(inputs: &[&TypedTensor<T>]) -> Vec<ConcreteEinsumInputSpec> {
2052 inputs
2053 .iter()
2054 .map(|tensor| ConcreteEinsumInputSpec {
2055 dtype: T::dtype(),
2056 shape: tensor.shape().to_vec(),
2057 })
2058 .collect()
2059}
2060
2061fn read_input_specs(inputs: &[TensorRead<'_>]) -> Vec<ConcreteEinsumInputSpec> {
2062 inputs
2063 .iter()
2064 .map(|tensor| ConcreteEinsumInputSpec {
2065 dtype: tensor.dtype(),
2066 shape: tensor.shape().to_vec(),
2067 })
2068 .collect()
2069}
2070
2071fn typed_view_einsum_subscripts<T: TensorScalar>(
2072 session: &mut dyn Engine,
2073 inputs: &[TypedTensorView<'_, T>],
2074 subscripts: &Subscripts,
2075 op: &'static str,
2076) -> Result<TypedTensor<T>> {
2077 let reads: Vec<_> = inputs
2078 .iter()
2079 .cloned()
2080 .map(|view| TensorRead::from_view(T::tensor_view(view)))
2081 .collect();
2082 let plan =
2083 ConcreteEinsumPlan::prepare_subscripts_internal(read_input_specs(&reads), subscripts)?;
2084 let result = plan.execute_read(&reads, session)?;
2085 into_typed_result(result, op)
2086}
2087
2088fn read_binary_dot_config_for_labels<L: Copy + PartialEq>(
2089 inputs: &[TensorRead<'_>],
2090 lhs_labels: &[L],
2091 rhs_labels: &[L],
2092 output_labels: &[L],
2093 out: &TensorWrite<'_>,
2094) -> Option<(BinaryDotOperandOrder, tenferro_tensor::DotGeneralConfig)> {
2095 if inputs.len() != 2
2096 || inputs[0].dtype() != inputs[1].dtype()
2097 || out.dtype() != inputs[0].dtype()
2098 {
2099 return None;
2100 }
2101 binary_dot_config_for_into(
2102 inputs[0].shape(),
2103 inputs[1].shape(),
2104 lhs_labels,
2105 rhs_labels,
2106 output_labels,
2107 out.shape(),
2108 )
2109}
2110
2111fn read_binary_dot_config(
2112 inputs: &[TensorRead<'_>],
2113 subscripts: &Subscripts,
2114 out: &TensorWrite<'_>,
2115) -> Option<(BinaryDotOperandOrder, tenferro_tensor::DotGeneralConfig)> {
2116 let [lhs, rhs] = subscripts.inputs.as_slice() else {
2117 return None;
2118 };
2119 read_binary_dot_config_for_labels(inputs, lhs, rhs, &subscripts.output, out)
2120}
2121
2122fn execute_binary_dot_config_read_into(
2123 session: &mut dyn Engine,
2124 inputs: &[TensorRead<'_>],
2125 order: BinaryDotOperandOrder,
2126 config: &tenferro_tensor::DotGeneralConfig,
2127 out: TensorWrite<'_>,
2128) -> Result<()> {
2129 let (lhs, rhs) = match order {
2130 BinaryDotOperandOrder::Original => (0, 1),
2131 BinaryDotOperandOrder::Swapped => (1, 0),
2132 };
2133 session
2134 .dot_general_read_into(inputs[lhs].clone(), inputs[rhs].clone(), config, out)
2135 .map_err(Error::from)
2136}
2137
2138fn tensor_einsum_into_subscripts(
2139 session: &mut dyn Engine,
2140 inputs: &[&Tensor],
2141 subscripts: &Subscripts,
2142 out: TensorWrite<'_>,
2143 op: &'static str,
2144) -> Result<()> {
2145 if inputs.len() == 2 {
2146 let reads = [
2147 TensorRead::from_tensor(inputs[0]),
2148 TensorRead::from_tensor(inputs[1]),
2149 ];
2150 if let Some((order, config)) = read_binary_dot_config(&reads, subscripts, &out) {
2151 return execute_binary_dot_config_read_into(session, &reads, order, &config, out);
2152 }
2153 }
2154 let plan = ConcreteEinsumPlan::prepare_subscripts_internal(input_specs(inputs), subscripts)?;
2155 validate_output(&plan.inputs, &plan.tree, &out, op)?;
2156 plan.execute_into(inputs, session, out)
2157}
2158
2159fn parse_fast_ascii_binary_labels(notation: &str) -> Option<(&[u8], &[u8], &[u8])> {
2160 let (terms, output) = notation.split_once("->")?;
2161 let (lhs, rhs) = terms.split_once(',')?;
2162 [lhs, rhs, output]
2165 .iter()
2166 .all(|term| term.bytes().all(|byte| byte.is_ascii_alphabetic()))
2167 .then_some((lhs.as_bytes(), rhs.as_bytes(), output.as_bytes()))
2168}
2169
2170fn borrowed_notation_labels(notation: &EinsumNotation) -> Option<[TinyVec<[u32; 8]>; 3]> {
2171 let [lhs, rhs] = notation.inputs.as_slice() else {
2172 return None;
2173 };
2174 let labels = |axes: &[crate::EinsumAxis]| {
2175 axes.iter()
2176 .map(|axis| match axis {
2177 crate::EinsumAxis::Label(label) => Some(*label),
2178 crate::EinsumAxis::Ellipsis => None,
2179 })
2180 .collect::<Option<TinyVec<[u32; 8]>>>()
2181 };
2182 Some([labels(lhs)?, labels(rhs)?, labels(¬ation.output)?])
2183}
2184
2185fn typed_view_binary_dot_config<T: TensorScalar, L: Copy + PartialEq>(
2186 inputs: &[TypedTensorView<'_, T>],
2187 lhs_labels: &[L],
2188 rhs_labels: &[L],
2189 output_labels: &[L],
2190 out: &TensorWrite<'_>,
2191) -> Option<(BinaryDotOperandOrder, tenferro_tensor::DotGeneralConfig)> {
2192 if inputs.len() != 2 || out.dtype() != T::dtype() {
2193 return None;
2194 }
2195 binary_dot_config_for_into(
2196 inputs[0].shape(),
2197 inputs[1].shape(),
2198 lhs_labels,
2199 rhs_labels,
2200 output_labels,
2201 out.shape(),
2202 )
2203}
2204
2205fn execute_typed_view_binary_dot_into<T: TensorScalar>(
2206 session: &mut dyn Engine,
2207 inputs: &[TypedTensorView<'_, T>],
2208 order: BinaryDotOperandOrder,
2209 config: &tenferro_tensor::DotGeneralConfig,
2210 out: TensorWrite<'_>,
2211) -> Result<()> {
2212 let (lhs, rhs) = match order {
2213 BinaryDotOperandOrder::Original => (0, 1),
2214 BinaryDotOperandOrder::Swapped => (1, 0),
2215 };
2216 let lhs = TensorRead::from_view(T::tensor_view(inputs[lhs].clone()));
2217 let rhs = TensorRead::from_view(T::tensor_view(inputs[rhs].clone()));
2218 session
2219 .dot_general_read_into(lhs, rhs, config, out)
2220 .map_err(Error::from)
2221}
2222
2223fn typed_view_einsum_into_subscripts<T: TensorScalar>(
2224 session: &mut dyn Engine,
2225 inputs: &[TypedTensorView<'_, T>],
2226 subscripts: &Subscripts,
2227 out: TensorWrite<'_>,
2228 op: &'static str,
2229) -> Result<()> {
2230 let reads: Vec<_> = inputs
2231 .iter()
2232 .cloned()
2233 .map(|view| TensorRead::from_view(T::tensor_view(view)))
2234 .collect();
2235 let plan =
2236 ConcreteEinsumPlan::prepare_subscripts_internal(read_input_specs(&reads), subscripts)?;
2237 validate_output(&plan.inputs, &plan.tree, &out, op)?;
2238 plan.execute_read_into(&reads, session, out)
2239}
2240
2241fn typed_einsum_into_subscripts<T: TensorScalar>(
2242 session: &mut dyn Engine,
2243 inputs: &[&TypedTensor<T>],
2244 subscripts: &Subscripts,
2245 out: TensorWrite<'_>,
2246 op: &'static str,
2247) -> Result<()> {
2248 if inputs.len() == 2 {
2249 if let [lhs, rhs] = subscripts.inputs.as_slice() {
2250 if let Some((order, config)) = binary_dot_config_for_into(
2251 inputs[0].shape(),
2252 inputs[1].shape(),
2253 lhs,
2254 rhs,
2255 &subscripts.output,
2256 out.shape(),
2257 ) {
2258 if out.dtype() == T::dtype() {
2259 let reads = [T::tensor_read(inputs[0]), T::tensor_read(inputs[1])];
2260 return execute_binary_dot_config_read_into(
2261 session, &reads, order, &config, out,
2262 );
2263 }
2264 }
2265 }
2266 }
2267 let reads: Vec<_> = inputs.iter().map(|tensor| T::tensor_read(tensor)).collect();
2268 let plan =
2269 ConcreteEinsumPlan::prepare_subscripts_internal(read_input_specs(&reads), subscripts)?;
2270 validate_output(&plan.inputs, &plan.tree, &out, op)?;
2271 plan.execute_read_into(&reads, session, out)
2272}
2273
2274fn tensor_read_einsum_into_subscripts(
2275 session: &mut dyn Engine,
2276 inputs: &[TensorRead<'_>],
2277 subscripts: &Subscripts,
2278 out: TensorWrite<'_>,
2279 op: &'static str,
2280) -> Result<()> {
2281 if let Some((order, config)) = read_binary_dot_config(inputs, subscripts, &out) {
2282 return execute_binary_dot_config_read_into(session, inputs, order, &config, out);
2283 }
2284 let plan =
2285 ConcreteEinsumPlan::prepare_subscripts_internal(read_input_specs(inputs), subscripts)?;
2286 validate_output(&plan.inputs, &plan.tree, &out, op)?;
2287 plan.execute_read_into(inputs, session, out)
2288}
2289
2290fn validate_output(
2291 inputs: &[ConcreteEinsumInputSpec],
2292 tree: &ContractionTree,
2293 out: &TensorWrite<'_>,
2294 op: &'static str,
2295) -> Result<()> {
2296 let expected = output_spec(inputs, tree, op)?;
2297 if out.dtype() != expected.dtype {
2298 return Err(Error::dtype_mismatch(op, expected.dtype, out.dtype()));
2299 }
2300 if out.shape() != expected.shape.as_slice() {
2301 return Err(Error::shape_mismatch(
2302 op,
2303 out.shape().to_vec(),
2304 expected.shape.clone(),
2305 ));
2306 }
2307 Ok(())
2308}
2309
2310fn output_spec(
2311 inputs: &[ConcreteEinsumInputSpec],
2312 tree: &ContractionTree,
2313 op: &'static str,
2314) -> Result<ConcreteEinsumInputSpec> {
2315 let dtype = inputs
2316 .first()
2317 .ok_or_else(|| {
2318 Error::invalid_argument(op, "inputs", "einsum requires at least one input tensor")
2319 })?
2320 .dtype;
2321 for input in inputs {
2322 if input.dtype != dtype {
2323 return Err(Error::dtype_mismatch(op, dtype, input.dtype));
2324 }
2325 }
2326
2327 for (input, labels) in inputs.iter().zip(tree.subscripts.inputs.iter()) {
2328 if labels.len() != input.shape.len() {
2329 return Err(Error::rank_mismatch(op, labels.len(), input.shape.len()));
2330 }
2331 }
2332 let output_shape = tree.output_shape();
2333 if output_shape.len() != tree.subscripts.output.len() {
2334 return Err(Error::invalid_argument(
2335 op,
2336 "output labels",
2337 "an output label is missing from all inputs",
2338 ));
2339 }
2340 Ok(ConcreteEinsumInputSpec {
2341 dtype,
2342 shape: output_shape,
2343 })
2344}
2345
2346fn typed_einsum_subscripts<T: TensorScalar>(
2347 session: &mut dyn Engine,
2348 inputs: &[&TypedTensor<T>],
2349 subscripts: &Subscripts,
2350 op: &'static str,
2351) -> Result<TypedTensor<T>> {
2352 let reads: Vec<_> = inputs.iter().map(|tensor| T::tensor_read(tensor)).collect();
2353 let plan =
2354 ConcreteEinsumPlan::prepare_subscripts_internal(read_input_specs(&reads), subscripts)?;
2355 let result = plan.execute_read(&reads, session)?;
2356 into_typed_result(result, op)
2357}
2358
2359pub(crate) fn into_typed_result<T: TensorScalar>(
2360 result: Tensor,
2361 op: &'static str,
2362) -> Result<TypedTensor<T>> {
2363 let actual = result.dtype();
2364 T::into_typed(result).map_err(|_| Error::dtype_mismatch(op, T::dtype(), actual))
2365}