1use tenferro_tensor::{DType, DotGeneralAccumulation, Engine, Tensor, TensorRead, TensorWrite};
8use tinyvec::TinyVec;
9
10use crate::binary_dot::BinaryDotOperandOrder;
11use crate::eager::{
12 binary_dot_config_for_into, binary_dot_plan_for_shapes, eager_einsum_exec,
13 eager_einsum_exec_read, eager_einsum_exec_read_into, eager_einsum_exec_read_into_accum,
14 eager_einsum_read_subscripts_on_session, eager_einsum_subscripts_on_session,
15 execute_binary_dot_read_into, execute_binary_dot_read_into_accum, execute_binary_dot_shortcut,
16 plan_subscripts, ConjugatedOperands,
17};
18use crate::ellipsis::resolve_einsum_notation;
19use crate::TensorDotAxes;
20use crate::{
21 parse_einsum_notation, ContractionTree, EinsumNotation, EinsumSubscripts, Error, Result,
22 Subscripts,
23};
24
25const TENSOR_EINSUM_INTO_OP: &str = "einsum_into";
26const TENSOR_READ_EINSUM_INTO_OP: &str = "einsum_into";
27const PLAN_EXECUTE_OP: &str = "ConcreteEinsumPlan::execute";
28
29pub(crate) fn tensordot_owned(
31 lhs: &Tensor,
32 rhs: &Tensor,
33 axes: TensorDotAxes<'_>,
34 session: &mut dyn Engine,
35) -> Result<Tensor> {
36 let config = crate::tensordot::dot_general_config(axes, lhs.shape().len(), rhs.shape().len())?;
37 crate::tensordot::validate_concrete_contract_dims(lhs.shape(), rhs.shape(), &config)?;
38 session
39 .dot_general_read(
40 TensorRead::from_tensor(lhs),
41 TensorRead::from_tensor(rhs),
42 &config,
43 )
44 .map_err(Error::from)
45}
46
47pub(crate) fn tensordot_reads(
49 lhs: TensorRead<'_>,
50 rhs: TensorRead<'_>,
51 axes: TensorDotAxes<'_>,
52 session: &mut dyn Engine,
53) -> Result<Tensor> {
54 let config = crate::tensordot::dot_general_config(axes, lhs.shape().len(), rhs.shape().len())?;
55 crate::tensordot::validate_concrete_contract_dims(lhs.shape(), rhs.shape(), &config)?;
56 session
57 .dot_general_read(lhs, rhs, &config)
58 .map_err(Error::from)
59}
60
61pub(crate) trait OwnedEinsum {
62 fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor>;
63
64 fn einsum_notation(
65 &self,
66 notation: &EinsumNotation,
67 session: &mut dyn Engine,
68 ) -> Result<Tensor>;
69
70 fn einsum_subscripts(
71 &self,
72 subscripts: &EinsumSubscripts,
73 session: &mut dyn Engine,
74 ) -> Result<Tensor>;
75}
76
77impl OwnedEinsum for [&Tensor] {
78 fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor> {
79 let notation = parse_einsum_notation(subscripts)?;
80 self.einsum_notation(¬ation, session)
81 }
82
83 fn einsum_notation(
84 &self,
85 notation: &EinsumNotation,
86 session: &mut dyn Engine,
87 ) -> Result<Tensor> {
88 let subscripts = resolve_tensor_notation(self, notation)?;
89 eager_einsum_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
90 }
91
92 fn einsum_subscripts(
93 &self,
94 subscripts: &EinsumSubscripts,
95 session: &mut dyn Engine,
96 ) -> Result<Tensor> {
97 let subscripts = Subscripts::from(subscripts);
98 eager_einsum_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
99 }
100}
101
102pub(crate) trait OwnedEinsumInto {
103 fn einsum_into(
104 &self,
105 subscripts: &str,
106 session: &mut dyn Engine,
107 out: TensorWrite<'_>,
108 ) -> Result<()>;
109
110 fn einsum_into_notation(
111 &self,
112 notation: &EinsumNotation,
113 session: &mut dyn Engine,
114 out: TensorWrite<'_>,
115 ) -> Result<()>;
116
117 fn einsum_into_subscripts(
118 &self,
119 subscripts: &EinsumSubscripts,
120 session: &mut dyn Engine,
121 out: TensorWrite<'_>,
122 ) -> Result<()>;
123}
124
125impl OwnedEinsumInto for [&Tensor] {
126 fn einsum_into(
127 &self,
128 subscripts: &str,
129 session: &mut dyn Engine,
130 out: TensorWrite<'_>,
131 ) -> Result<()> {
132 if let ([lhs, rhs], Some((a, b, c))) = (self, parse_fast_ascii_binary_labels(subscripts)) {
133 let reads = [TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs)];
134 if let Some((order, config)) = read_binary_dot_config_for_labels(&reads, a, b, c, &out)
135 {
136 return execute_binary_dot_config_read_into(session, &reads, order, &config, out);
137 }
138 }
139 let notation = parse_einsum_notation(subscripts)?;
140 self.einsum_into_notation(¬ation, session, out)
141 }
142
143 fn einsum_into_notation(
144 &self,
145 notation: &EinsumNotation,
146 session: &mut dyn Engine,
147 out: TensorWrite<'_>,
148 ) -> Result<()> {
149 let subscripts = resolve_tensor_notation(self, notation)?;
150 tensor_einsum_into_subscripts(session, self, &subscripts, out, TENSOR_EINSUM_INTO_OP)
151 }
152
153 fn einsum_into_subscripts(
154 &self,
155 subscripts: &EinsumSubscripts,
156 session: &mut dyn Engine,
157 out: TensorWrite<'_>,
158 ) -> Result<()> {
159 let subscripts = Subscripts::from(subscripts);
160 tensor_einsum_into_subscripts(session, self, &subscripts, out, TENSOR_EINSUM_INTO_OP)
161 }
162}
163
164pub(crate) trait ReadEinsum {
165 fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor>;
166
167 fn einsum_read_notation(
168 &self,
169 notation: &EinsumNotation,
170 session: &mut dyn Engine,
171 ) -> Result<Tensor>;
172
173 fn einsum_read_subscripts(
174 &self,
175 subscripts: &EinsumSubscripts,
176 session: &mut dyn Engine,
177 ) -> Result<Tensor>;
178}
179
180impl<'a> ReadEinsum for [TensorRead<'a>] {
181 fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor> {
182 let notation = parse_einsum_notation(subscripts)?;
183 self.einsum_read_notation(¬ation, session)
184 }
185
186 fn einsum_read_notation(
187 &self,
188 notation: &EinsumNotation,
189 session: &mut dyn Engine,
190 ) -> Result<Tensor> {
191 let subscripts = resolve_read_notation(self, notation)?;
192 eager_einsum_read_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
193 }
194
195 fn einsum_read_subscripts(
196 &self,
197 subscripts: &EinsumSubscripts,
198 session: &mut dyn Engine,
199 ) -> Result<Tensor> {
200 let subscripts = Subscripts::from(subscripts);
201 eager_einsum_read_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
202 }
203}
204
205pub(crate) trait ReadEinsumInto {
206 fn einsum_read_into(
207 &self,
208 subscripts: &str,
209 session: &mut dyn Engine,
210 out: TensorWrite<'_>,
211 ) -> Result<()>;
212
213 fn einsum_read_into_notation(
214 &self,
215 notation: &EinsumNotation,
216 session: &mut dyn Engine,
217 out: TensorWrite<'_>,
218 ) -> Result<()>;
219
220 fn einsum_read_into_subscripts(
221 &self,
222 subscripts: &EinsumSubscripts,
223 session: &mut dyn Engine,
224 out: TensorWrite<'_>,
225 ) -> Result<()>;
226}
227
228impl<'a> ReadEinsumInto for [TensorRead<'a>] {
229 fn einsum_read_into(
230 &self,
231 subscripts: &str,
232 session: &mut dyn Engine,
233 out: TensorWrite<'_>,
234 ) -> Result<()> {
235 if let Some((lhs, rhs, output)) = parse_fast_ascii_binary_labels(subscripts) {
236 if let Some((order, config)) =
237 read_binary_dot_config_for_labels(self, lhs, rhs, output, &out)
238 {
239 return execute_binary_dot_config_read_into(session, self, order, &config, out);
240 }
241 }
242 let notation = parse_einsum_notation(subscripts)?;
243 self.einsum_read_into_notation(¬ation, session, out)
244 }
245
246 fn einsum_read_into_notation(
247 &self,
248 notation: &EinsumNotation,
249 session: &mut dyn Engine,
250 out: TensorWrite<'_>,
251 ) -> Result<()> {
252 if let Some([lhs, rhs, output]) = borrowed_notation_labels(notation) {
253 if let Some((order, config)) =
254 read_binary_dot_config_for_labels(self, &lhs, &rhs, &output, &out)
255 {
256 return execute_binary_dot_config_read_into(session, self, order, &config, out);
257 }
258 }
259 let subscripts = resolve_read_notation(self, notation)?;
260 tensor_read_einsum_into_subscripts(
261 session,
262 self,
263 &subscripts,
264 out,
265 TENSOR_READ_EINSUM_INTO_OP,
266 )
267 }
268
269 fn einsum_read_into_subscripts(
270 &self,
271 subscripts: &EinsumSubscripts,
272 session: &mut dyn Engine,
273 out: TensorWrite<'_>,
274 ) -> Result<()> {
275 if let [lhs, rhs] = subscripts.inputs.as_slice() {
276 if let Some((order, config)) =
277 read_binary_dot_config_for_labels(self, lhs, rhs, &subscripts.output, &out)
278 {
279 return execute_binary_dot_config_read_into(session, self, order, &config, out);
280 }
281 }
282 let subscripts = Subscripts::from(subscripts);
283 tensor_read_einsum_into_subscripts(
284 session,
285 self,
286 &subscripts,
287 out,
288 TENSOR_READ_EINSUM_INTO_OP,
289 )
290 }
291}
292
293#[derive(Debug)]
317pub struct ConcreteEinsumPlan {
318 tree: ContractionTree,
319 inputs: Vec<ConcreteEinsumInputSpec>,
320 output_shape: Vec<usize>,
321 binary_dot: Option<crate::binary_dot::BinaryDotPlan>,
322 cpu_plan: crate::cpu_concrete::Cache,
323 conj: crate::api::ConjMask,
325}
326
327impl ConcreteEinsumPlan {
328 pub fn clear_cpu_cache(&mut self) {
344 self.cpu_plan.clear();
345 }
346
347 pub fn cpu_cache_stats(&self) -> Result<tenferro_tensor::CacheStats> {
367 self.cpu_plan.stats().map_err(Error::from)
368 }
369
370 pub(crate) fn prepare_owned_subscripts<'a, I>(
371 inputs: I,
372 subscripts: &EinsumSubscripts,
373 ) -> Result<Self>
374 where
375 I: AsRef<[&'a Tensor]>,
376 {
377 let subscripts = Subscripts::from(subscripts);
378 Self::prepare_subscripts_internal(input_specs(inputs.as_ref()), &subscripts)
379 }
380
381 pub(crate) fn prepare_read_subscripts<'a, I>(
382 inputs: I,
383 subscripts: &EinsumSubscripts,
384 ) -> Result<Self>
385 where
386 I: AsRef<[TensorRead<'a>]>,
387 {
388 let subscripts = Subscripts::from(subscripts);
389 Self::prepare_subscripts_internal(read_input_specs(inputs.as_ref()), &subscripts)
390 }
391
392 pub(crate) fn step_count(&self) -> usize {
394 self.tree.step_count()
395 }
396
397 pub(crate) fn run_owned<'a, I>(&self, inputs: I, session: &mut dyn Engine) -> Result<Tensor>
398 where
399 I: AsRef<[&'a Tensor]>,
400 {
401 let inputs = inputs.as_ref();
402 self.validate_tensor_inputs(inputs, PLAN_EXECUTE_OP)?;
403 if let ([lhs, rhs], Some(plan)) = (inputs, self.binary_dot_shortcut()) {
404 let reads = [TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs)];
405 return execute_binary_dot_shortcut(session, &reads, plan).map_err(Error::from);
406 }
407 if inputs.len() >= 3 {
408 let reads: Vec<_> = inputs
409 .iter()
410 .map(|tensor| TensorRead::from_tensor(tensor))
411 .collect();
412 if let Some(result) = crate::cpu_concrete::execute_cached(
413 session,
414 &reads,
415 &self.tree,
416 Some(&self.cpu_plan),
417 ) {
418 return result.map_err(Error::from);
419 }
420 }
421 eager_einsum_exec(session, inputs, &self.tree).map_err(Error::from)
422 }
423
424 pub(crate) fn run_reads<'a, I>(&self, inputs: I, session: &mut dyn Engine) -> Result<Tensor>
425 where
426 I: AsRef<[TensorRead<'a>]>,
427 {
428 let inputs = inputs.as_ref();
429 self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
430 if let Some(plan) = self.binary_dot_shortcut() {
431 return execute_binary_dot_shortcut(session, inputs, plan).map_err(Error::from);
432 }
433 if let Some(result) =
434 crate::cpu_concrete::execute_cached(session, inputs, &self.tree, Some(&self.cpu_plan))
435 {
436 return result.map_err(Error::from);
437 }
438 eager_einsum_exec_read(session, inputs, &self.tree).map_err(Error::from)
439 }
440
441 pub(crate) fn run_owned_into<'a, I>(
442 &self,
443 inputs: I,
444 session: &mut dyn Engine,
445 out: TensorWrite<'_>,
446 ) -> Result<()>
447 where
448 I: AsRef<[&'a Tensor]>,
449 {
450 let inputs = inputs.as_ref();
451 self.validate_tensor_inputs(inputs, PLAN_EXECUTE_OP)?;
452 self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
453 if let Some(binary_dot) = &self.binary_dot {
454 if let [lhs, rhs] = inputs {
455 let reads = [TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs)];
456 return execute_binary_dot_read_into(session, &reads, binary_dot, out)
457 .map_err(Error::from);
458 }
459 }
460 let reads: Vec<_> = inputs
461 .iter()
462 .map(|tensor| TensorRead::from_tensor(tensor))
463 .collect();
464 self.execute_cpu_or_fallback_into(session, &reads, out)
465 .map_err(Error::from)
466 }
467
468 pub(crate) fn run_reads_into<'a, I>(
469 &self,
470 inputs: I,
471 session: &mut dyn Engine,
472 out: TensorWrite<'_>,
473 ) -> Result<()>
474 where
475 I: AsRef<[TensorRead<'a>]>,
476 {
477 let inputs = inputs.as_ref();
478 self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
479 self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
480 if let Some(binary_dot) = &self.binary_dot {
481 return execute_binary_dot_read_into(session, inputs, binary_dot, out)
482 .map_err(Error::from);
483 }
484 self.execute_cpu_or_fallback_into(session, inputs, out)
485 .map_err(Error::from)
486 }
487
488 pub(crate) fn run_reads_into_accum<'a, I>(
489 &self,
490 inputs: I,
491 session: &mut dyn Engine,
492 accumulation: DotGeneralAccumulation,
493 out: TensorWrite<'_>,
494 ) -> Result<()>
495 where
496 I: AsRef<[TensorRead<'a>]>,
497 {
498 let inputs = inputs.as_ref();
499 self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
500 self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
501 if let Some(binary_dot) = &self.binary_dot {
502 return execute_binary_dot_read_into_accum(
503 session,
504 inputs,
505 binary_dot,
506 accumulation,
507 out,
508 )
509 .map_err(Error::from);
510 }
511 if crate::cpu_concrete::supports(session, inputs) {
512 let mut accumulation = accumulation;
513 let conjugated = ConjugatedOperands::apply(session, inputs, &mut accumulation)?;
514 let result = self.run_reads(conjugated.reads(inputs), session)?;
515 let mut out = out;
516 return tenferro_tensor::backend::accumulate_dot_result_into(
517 &result,
518 accumulation,
519 &mut out,
520 )
521 .map_err(Error::from);
522 }
523 eager_einsum_exec_read_into_accum(session, inputs, &self.tree, accumulation, out)
524 .map_err(Error::from)
525 }
526
527 fn execute_cpu_or_fallback_into(
528 &self,
529 session: &mut dyn Engine,
530 inputs: &[TensorRead<'_>],
531 out: TensorWrite<'_>,
532 ) -> tenferro_tensor::Result<()> {
533 if crate::cpu_concrete::supports(session, inputs) {
534 crate::cpu_concrete::execute_into(
535 session,
536 inputs,
537 &self.tree,
538 Some(&self.cpu_plan),
539 out,
540 )
541 } else {
542 eager_einsum_exec_read_into(session, inputs, &self.tree, out)
543 }
544 }
545
546 fn binary_dot_shortcut(&self) -> Option<&crate::binary_dot::BinaryDotPlan> {
551 let plan = self.binary_dot.as_ref()?;
552 let [lhs, rhs] = self.inputs.as_slice() else {
553 return None;
554 };
555 (plan.operand_order == BinaryDotOperandOrder::Original && lhs.dtype == rhs.dtype)
556 .then_some(plan)
557 }
558
559 fn prepare_subscripts_internal(
560 inputs: Vec<ConcreteEinsumInputSpec>,
561 subscripts: &Subscripts,
562 ) -> Result<Self> {
563 let shapes: Vec<&[usize]> = inputs.iter().map(|input| input.shape.as_slice()).collect();
564 let binary_dot = binary_dot_plan_for_shapes(&shapes, subscripts);
565 let tree = plan_subscripts(subscripts, &shapes)?;
566 let output_shape = tree.output_shape();
567 Ok(Self {
568 tree,
569 inputs,
570 output_shape,
571 binary_dot,
572 cpu_plan: crate::cpu_concrete::Cache::default(),
573 conj: crate::api::ConjMask::default(),
574 })
575 }
576
577 pub(crate) fn prepare_planned(
582 inputs: Vec<ConcreteEinsumInputSpec>,
583 subscripts: &Subscripts,
584 optimize: &crate::EinsumOptimize,
585 conj: crate::api::ConjMask,
586 ) -> Result<Self> {
587 if conj.len() > inputs.len() {
588 return Err(Error::invalid_argument(
589 "einsum",
590 "conj",
591 format!(
592 "the conjugation mask has {} entries for {} inputs",
593 conj.len(),
594 inputs.len()
595 ),
596 ));
597 }
598 let mut plan = match optimize {
599 crate::EinsumOptimize::Auto(options) if !options.anneals() => {
600 Self::prepare_subscripts_internal(inputs, subscripts)?
601 }
602 optimize => {
603 if inputs.is_empty() || subscripts.inputs.len() != inputs.len() {
604 return Self::prepare_subscripts_internal(inputs, subscripts);
606 }
607 let shapes: Vec<&[usize]> =
608 inputs.iter().map(|input| input.shape.as_slice()).collect();
609 let binary_dot = binary_dot_plan_for_shapes(&shapes, subscripts);
610 let (_, tree) = crate::optimize::resolve_einsum_strategy_with_spec(
611 optimize.clone(),
612 subscripts,
613 &shapes,
614 )?;
615 let output_shape = tree.output_shape();
616 Self {
617 tree,
618 inputs,
619 output_shape,
620 binary_dot,
621 cpu_plan: crate::cpu_concrete::Cache::default(),
622 conj: crate::api::ConjMask::default(),
623 }
624 }
625 };
626 plan.conj = conj;
627 Ok(plan)
628 }
629
630 pub(crate) fn conj_mask(&self) -> &crate::api::ConjMask {
632 &self.conj
633 }
634
635 #[cfg(test)]
637 pub(crate) fn tree_step_pair(&self, step: usize) -> Option<(usize, usize)> {
638 self.tree.step_pair(step)
639 }
640
641 fn validate_tensor_inputs(&self, actual: &[&Tensor], op: &'static str) -> Result<()> {
642 self.validate_input_metadata(
643 actual.iter().map(|tensor| (tensor.dtype(), tensor.shape())),
644 op,
645 )
646 }
647
648 fn validate_read_inputs(&self, actual: &[TensorRead<'_>], op: &'static str) -> Result<()> {
649 self.validate_input_metadata(
650 actual.iter().map(|tensor| (tensor.dtype(), tensor.shape())),
651 op,
652 )
653 }
654
655 fn validate_input_metadata<'a>(
656 &self,
657 actual: impl ExactSizeIterator<Item = (DType, &'a [usize])>,
658 op: &'static str,
659 ) -> Result<()> {
660 if actual.len() != self.inputs.len() {
661 return Err(Error::invalid_argument(
662 op,
663 "inputs",
664 format!(
665 "prepared einsum expects {} inputs, got {}",
666 self.inputs.len(),
667 actual.len()
668 ),
669 ));
670 }
671 for (expected, (actual_dtype, actual_shape)) in self.inputs.iter().zip(actual) {
672 if expected.dtype != actual_dtype {
673 return Err(Error::dtype_mismatch(op, expected.dtype, actual_dtype));
674 }
675 if expected.shape != actual_shape {
676 return Err(Error::shape_mismatch(
677 op,
678 expected.shape.clone(),
679 actual_shape.to_vec(),
680 ));
681 }
682 }
683 Ok(())
684 }
685
686 fn validate_cached_output(&self, out: &TensorWrite<'_>, op: &'static str) -> Result<()> {
687 let dtype = self
688 .inputs
689 .first()
690 .map(|input| input.dtype)
691 .ok_or_else(|| {
692 Error::invalid_argument(op, "inputs", "einsum requires at least one input tensor")
693 })?;
694 for input in &self.inputs[1..] {
695 if input.dtype != dtype {
696 return Err(Error::dtype_mismatch(op, dtype, input.dtype));
697 }
698 }
699 if out.dtype() != dtype {
700 return Err(Error::dtype_mismatch(op, dtype, out.dtype()));
701 }
702 if out.shape() != self.output_shape {
703 return Err(Error::shape_mismatch(
704 op,
705 out.shape().to_vec(),
706 self.output_shape.clone(),
707 ));
708 }
709 Ok(())
710 }
711}
712
713#[derive(Clone, Debug)]
714pub(crate) struct ConcreteEinsumInputSpec {
715 dtype: DType,
716 shape: Vec<usize>,
717}
718
719fn resolve_shapes(notation: &EinsumNotation, shapes: Vec<&[usize]>) -> Result<Subscripts> {
720 resolve_einsum_notation(notation, &shapes)
721}
722
723fn resolve_tensor_notation(inputs: &[&Tensor], notation: &EinsumNotation) -> Result<Subscripts> {
724 resolve_shapes(
725 notation,
726 inputs.iter().map(|tensor| tensor.shape()).collect(),
727 )
728}
729
730pub(crate) fn resolve_read_notation<'a>(
731 inputs: &[TensorRead<'a>],
732 notation: &EinsumNotation,
733) -> Result<Subscripts> {
734 resolve_shapes(notation, inputs.iter().map(|input| input.shape()).collect())
735}
736
737fn input_specs(inputs: &[&Tensor]) -> Vec<ConcreteEinsumInputSpec> {
738 inputs
739 .iter()
740 .map(|tensor| ConcreteEinsumInputSpec {
741 dtype: tensor.dtype(),
742 shape: tensor.shape().to_vec(),
743 })
744 .collect()
745}
746
747pub(crate) fn read_input_specs(inputs: &[TensorRead<'_>]) -> Vec<ConcreteEinsumInputSpec> {
748 inputs
749 .iter()
750 .map(|tensor| ConcreteEinsumInputSpec {
751 dtype: tensor.dtype(),
752 shape: tensor.shape().to_vec(),
753 })
754 .collect()
755}
756
757fn read_binary_dot_config_for_labels<L: Copy + PartialEq>(
758 inputs: &[TensorRead<'_>],
759 lhs_labels: &[L],
760 rhs_labels: &[L],
761 output_labels: &[L],
762 out: &TensorWrite<'_>,
763) -> Option<(BinaryDotOperandOrder, tenferro_tensor::DotGeneralConfig)> {
764 if inputs.len() != 2
765 || inputs[0].dtype() != inputs[1].dtype()
766 || out.dtype() != inputs[0].dtype()
767 {
768 return None;
769 }
770 binary_dot_config_for_into(
771 inputs[0].shape(),
772 inputs[1].shape(),
773 lhs_labels,
774 rhs_labels,
775 output_labels,
776 out.shape(),
777 )
778}
779
780fn read_binary_dot_config(
781 inputs: &[TensorRead<'_>],
782 subscripts: &Subscripts,
783 out: &TensorWrite<'_>,
784) -> Option<(BinaryDotOperandOrder, tenferro_tensor::DotGeneralConfig)> {
785 let [lhs, rhs] = subscripts.inputs.as_slice() else {
786 return None;
787 };
788 read_binary_dot_config_for_labels(inputs, lhs, rhs, &subscripts.output, out)
789}
790
791fn execute_binary_dot_config_read_into(
792 session: &mut dyn Engine,
793 inputs: &[TensorRead<'_>],
794 order: BinaryDotOperandOrder,
795 config: &tenferro_tensor::DotGeneralConfig,
796 out: TensorWrite<'_>,
797) -> Result<()> {
798 let (lhs, rhs) = match order {
799 BinaryDotOperandOrder::Original => (0, 1),
800 BinaryDotOperandOrder::Swapped => (1, 0),
801 };
802 session
803 .dot_general_read_into(inputs[lhs].clone(), inputs[rhs].clone(), config, out)
804 .map_err(Error::from)
805}
806
807fn tensor_einsum_into_subscripts(
808 session: &mut dyn Engine,
809 inputs: &[&Tensor],
810 subscripts: &Subscripts,
811 out: TensorWrite<'_>,
812 op: &'static str,
813) -> Result<()> {
814 if inputs.len() == 2 {
815 let reads = [
816 TensorRead::from_tensor(inputs[0]),
817 TensorRead::from_tensor(inputs[1]),
818 ];
819 if let Some((order, config)) = read_binary_dot_config(&reads, subscripts, &out) {
820 return execute_binary_dot_config_read_into(session, &reads, order, &config, out);
821 }
822 }
823 let plan = ConcreteEinsumPlan::prepare_subscripts_internal(input_specs(inputs), subscripts)?;
824 validate_output(&plan.inputs, &plan.tree, &out, op)?;
825 plan.run_owned_into(inputs, session, out)
826}
827
828fn parse_fast_ascii_binary_labels(notation: &str) -> Option<(&[u8], &[u8], &[u8])> {
829 let (terms, output) = notation.split_once("->")?;
830 let (lhs, rhs) = terms.split_once(',')?;
831 [lhs, rhs, output]
834 .iter()
835 .all(|term| term.bytes().all(|byte| byte.is_ascii_alphabetic()))
836 .then_some((lhs.as_bytes(), rhs.as_bytes(), output.as_bytes()))
837}
838
839fn borrowed_notation_labels(notation: &EinsumNotation) -> Option<[TinyVec<[u32; 8]>; 3]> {
840 let [lhs, rhs] = notation.inputs.as_slice() else {
841 return None;
842 };
843 let labels = |axes: &[crate::EinsumAxis]| {
844 axes.iter()
845 .map(|axis| match axis {
846 crate::EinsumAxis::Label(label) => Some(*label),
847 crate::EinsumAxis::Ellipsis => None,
848 })
849 .collect::<Option<TinyVec<[u32; 8]>>>()
850 };
851 Some([labels(lhs)?, labels(rhs)?, labels(¬ation.output)?])
852}
853
854fn tensor_read_einsum_into_subscripts(
855 session: &mut dyn Engine,
856 inputs: &[TensorRead<'_>],
857 subscripts: &Subscripts,
858 out: TensorWrite<'_>,
859 op: &'static str,
860) -> Result<()> {
861 if let Some((order, config)) = read_binary_dot_config(inputs, subscripts, &out) {
862 return execute_binary_dot_config_read_into(session, inputs, order, &config, out);
863 }
864 let plan =
865 ConcreteEinsumPlan::prepare_subscripts_internal(read_input_specs(inputs), subscripts)?;
866 validate_output(&plan.inputs, &plan.tree, &out, op)?;
867 plan.run_reads_into(inputs, session, out)
868}
869
870fn validate_output(
871 inputs: &[ConcreteEinsumInputSpec],
872 tree: &ContractionTree,
873 out: &TensorWrite<'_>,
874 op: &'static str,
875) -> Result<()> {
876 let expected = output_spec(inputs, tree, op)?;
877 if out.dtype() != expected.dtype {
878 return Err(Error::dtype_mismatch(op, expected.dtype, out.dtype()));
879 }
880 if out.shape() != expected.shape.as_slice() {
881 return Err(Error::shape_mismatch(
882 op,
883 out.shape().to_vec(),
884 expected.shape.clone(),
885 ));
886 }
887 Ok(())
888}
889
890fn output_spec(
891 inputs: &[ConcreteEinsumInputSpec],
892 tree: &ContractionTree,
893 op: &'static str,
894) -> Result<ConcreteEinsumInputSpec> {
895 let dtype = inputs
896 .first()
897 .ok_or_else(|| {
898 Error::invalid_argument(op, "inputs", "einsum requires at least one input tensor")
899 })?
900 .dtype;
901 for input in inputs {
902 if input.dtype != dtype {
903 return Err(Error::dtype_mismatch(op, dtype, input.dtype));
904 }
905 }
906
907 for (input, labels) in inputs.iter().zip(tree.subscripts.inputs.iter()) {
908 if labels.len() != input.shape.len() {
909 return Err(Error::rank_mismatch(op, labels.len(), input.shape.len()));
910 }
911 }
912 let output_shape = tree.output_shape();
913 if output_shape.len() != tree.subscripts.output.len() {
914 return Err(Error::invalid_argument(
915 op,
916 "output labels",
917 "an output label is missing from all inputs",
918 ));
919 }
920 Ok(ConcreteEinsumInputSpec {
921 dtype,
922 shape: output_shape,
923 })
924}