Skip to main content

tenferro_einsum/
concrete.rs

1//! Concrete einsum bodies and the prepared plan.
2//!
3//! The public grammar (`EinsumOps`, `TensordotOps`, the plan's
4//! `prepare`/`execute`/`execute_into`) lives in `crate::api` and forwards to
5//! the crate-internal entries here.
6
7use tenferro_tensor::{DType, DotGeneralAccumulation, Engine, Tensor, TensorRead, TensorWrite};
8use tinyvec::TinyVec;
9
10use crate::binary_dot::BinaryDotOperandOrder;
11use crate::eager::{
12    binary_dot_config_for_into, binary_dot_plan_for_shapes, eager_einsum_exec,
13    eager_einsum_exec_read, eager_einsum_exec_read_into, eager_einsum_exec_read_into_accum,
14    eager_einsum_read_subscripts_on_session, eager_einsum_subscripts_on_session,
15    execute_binary_dot_read_into, execute_binary_dot_read_into_accum, execute_binary_dot_shortcut,
16    plan_subscripts, ConjugatedOperands,
17};
18use crate::ellipsis::resolve_einsum_notation;
19use crate::TensorDotAxes;
20use crate::{
21    parse_einsum_notation, ContractionTree, EinsumNotation, EinsumSubscripts, Error, Result,
22    Subscripts,
23};
24
25const TENSOR_EINSUM_INTO_OP: &str = "einsum_into";
26const TENSOR_READ_EINSUM_INTO_OP: &str = "einsum_into";
27const PLAN_EXECUTE_OP: &str = "ConcreteEinsumPlan::execute";
28
29/// Tensordot of two owned erased tensors (the former `Tensor` entry).
30pub(crate) fn tensordot_owned(
31    lhs: &Tensor,
32    rhs: &Tensor,
33    axes: TensorDotAxes<'_>,
34    session: &mut dyn Engine,
35) -> Result<Tensor> {
36    let config = crate::tensordot::dot_general_config(axes, lhs.shape().len(), rhs.shape().len())?;
37    crate::tensordot::validate_concrete_contract_dims(lhs.shape(), rhs.shape(), &config)?;
38    session
39        .dot_general_read(
40            TensorRead::from_tensor(lhs),
41            TensorRead::from_tensor(rhs),
42            &config,
43        )
44        .map_err(Error::from)
45}
46
47/// Tensordot of two borrowed reads (the former typed entry, dtype-erased).
48pub(crate) fn tensordot_reads(
49    lhs: TensorRead<'_>,
50    rhs: TensorRead<'_>,
51    axes: TensorDotAxes<'_>,
52    session: &mut dyn Engine,
53) -> Result<Tensor> {
54    let config = crate::tensordot::dot_general_config(axes, lhs.shape().len(), rhs.shape().len())?;
55    crate::tensordot::validate_concrete_contract_dims(lhs.shape(), rhs.shape(), &config)?;
56    session
57        .dot_general_read(lhs, rhs, &config)
58        .map_err(Error::from)
59}
60
61pub(crate) trait OwnedEinsum {
62    fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor>;
63
64    fn einsum_notation(
65        &self,
66        notation: &EinsumNotation,
67        session: &mut dyn Engine,
68    ) -> Result<Tensor>;
69
70    fn einsum_subscripts(
71        &self,
72        subscripts: &EinsumSubscripts,
73        session: &mut dyn Engine,
74    ) -> Result<Tensor>;
75}
76
77impl OwnedEinsum for [&Tensor] {
78    fn einsum(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor> {
79        let notation = parse_einsum_notation(subscripts)?;
80        self.einsum_notation(&notation, session)
81    }
82
83    fn einsum_notation(
84        &self,
85        notation: &EinsumNotation,
86        session: &mut dyn Engine,
87    ) -> Result<Tensor> {
88        let subscripts = resolve_tensor_notation(self, notation)?;
89        eager_einsum_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
90    }
91
92    fn einsum_subscripts(
93        &self,
94        subscripts: &EinsumSubscripts,
95        session: &mut dyn Engine,
96    ) -> Result<Tensor> {
97        let subscripts = Subscripts::from(subscripts);
98        eager_einsum_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
99    }
100}
101
102pub(crate) trait OwnedEinsumInto {
103    fn einsum_into(
104        &self,
105        subscripts: &str,
106        session: &mut dyn Engine,
107        out: TensorWrite<'_>,
108    ) -> Result<()>;
109
110    fn einsum_into_notation(
111        &self,
112        notation: &EinsumNotation,
113        session: &mut dyn Engine,
114        out: TensorWrite<'_>,
115    ) -> Result<()>;
116
117    fn einsum_into_subscripts(
118        &self,
119        subscripts: &EinsumSubscripts,
120        session: &mut dyn Engine,
121        out: TensorWrite<'_>,
122    ) -> Result<()>;
123}
124
125impl OwnedEinsumInto for [&Tensor] {
126    fn einsum_into(
127        &self,
128        subscripts: &str,
129        session: &mut dyn Engine,
130        out: TensorWrite<'_>,
131    ) -> Result<()> {
132        if let ([lhs, rhs], Some((a, b, c))) = (self, parse_fast_ascii_binary_labels(subscripts)) {
133            let reads = [TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs)];
134            if let Some((order, config)) = read_binary_dot_config_for_labels(&reads, a, b, c, &out)
135            {
136                return execute_binary_dot_config_read_into(session, &reads, order, &config, out);
137            }
138        }
139        let notation = parse_einsum_notation(subscripts)?;
140        self.einsum_into_notation(&notation, session, out)
141    }
142
143    fn einsum_into_notation(
144        &self,
145        notation: &EinsumNotation,
146        session: &mut dyn Engine,
147        out: TensorWrite<'_>,
148    ) -> Result<()> {
149        let subscripts = resolve_tensor_notation(self, notation)?;
150        tensor_einsum_into_subscripts(session, self, &subscripts, out, TENSOR_EINSUM_INTO_OP)
151    }
152
153    fn einsum_into_subscripts(
154        &self,
155        subscripts: &EinsumSubscripts,
156        session: &mut dyn Engine,
157        out: TensorWrite<'_>,
158    ) -> Result<()> {
159        let subscripts = Subscripts::from(subscripts);
160        tensor_einsum_into_subscripts(session, self, &subscripts, out, TENSOR_EINSUM_INTO_OP)
161    }
162}
163
164pub(crate) trait ReadEinsum {
165    fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor>;
166
167    fn einsum_read_notation(
168        &self,
169        notation: &EinsumNotation,
170        session: &mut dyn Engine,
171    ) -> Result<Tensor>;
172
173    fn einsum_read_subscripts(
174        &self,
175        subscripts: &EinsumSubscripts,
176        session: &mut dyn Engine,
177    ) -> Result<Tensor>;
178}
179
180impl<'a> ReadEinsum for [TensorRead<'a>] {
181    fn einsum_read(&self, subscripts: &str, session: &mut dyn Engine) -> Result<Tensor> {
182        let notation = parse_einsum_notation(subscripts)?;
183        self.einsum_read_notation(&notation, session)
184    }
185
186    fn einsum_read_notation(
187        &self,
188        notation: &EinsumNotation,
189        session: &mut dyn Engine,
190    ) -> Result<Tensor> {
191        let subscripts = resolve_read_notation(self, notation)?;
192        eager_einsum_read_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
193    }
194
195    fn einsum_read_subscripts(
196        &self,
197        subscripts: &EinsumSubscripts,
198        session: &mut dyn Engine,
199    ) -> Result<Tensor> {
200        let subscripts = Subscripts::from(subscripts);
201        eager_einsum_read_subscripts_on_session(session, self, &subscripts).map_err(Error::from)
202    }
203}
204
205pub(crate) trait ReadEinsumInto {
206    fn einsum_read_into(
207        &self,
208        subscripts: &str,
209        session: &mut dyn Engine,
210        out: TensorWrite<'_>,
211    ) -> Result<()>;
212
213    fn einsum_read_into_notation(
214        &self,
215        notation: &EinsumNotation,
216        session: &mut dyn Engine,
217        out: TensorWrite<'_>,
218    ) -> Result<()>;
219
220    fn einsum_read_into_subscripts(
221        &self,
222        subscripts: &EinsumSubscripts,
223        session: &mut dyn Engine,
224        out: TensorWrite<'_>,
225    ) -> Result<()>;
226}
227
228impl<'a> ReadEinsumInto for [TensorRead<'a>] {
229    fn einsum_read_into(
230        &self,
231        subscripts: &str,
232        session: &mut dyn Engine,
233        out: TensorWrite<'_>,
234    ) -> Result<()> {
235        if let Some((lhs, rhs, output)) = parse_fast_ascii_binary_labels(subscripts) {
236            if let Some((order, config)) =
237                read_binary_dot_config_for_labels(self, lhs, rhs, output, &out)
238            {
239                return execute_binary_dot_config_read_into(session, self, order, &config, out);
240            }
241        }
242        let notation = parse_einsum_notation(subscripts)?;
243        self.einsum_read_into_notation(&notation, session, out)
244    }
245
246    fn einsum_read_into_notation(
247        &self,
248        notation: &EinsumNotation,
249        session: &mut dyn Engine,
250        out: TensorWrite<'_>,
251    ) -> Result<()> {
252        if let Some([lhs, rhs, output]) = borrowed_notation_labels(notation) {
253            if let Some((order, config)) =
254                read_binary_dot_config_for_labels(self, &lhs, &rhs, &output, &out)
255            {
256                return execute_binary_dot_config_read_into(session, self, order, &config, out);
257            }
258        }
259        let subscripts = resolve_read_notation(self, notation)?;
260        tensor_read_einsum_into_subscripts(
261            session,
262            self,
263            &subscripts,
264            out,
265            TENSOR_READ_EINSUM_INTO_OP,
266        )
267    }
268
269    fn einsum_read_into_subscripts(
270        &self,
271        subscripts: &EinsumSubscripts,
272        session: &mut dyn Engine,
273        out: TensorWrite<'_>,
274    ) -> Result<()> {
275        if let [lhs, rhs] = subscripts.inputs.as_slice() {
276            if let Some((order, config)) =
277                read_binary_dot_config_for_labels(self, lhs, rhs, &subscripts.output, &out)
278            {
279                return execute_binary_dot_config_read_into(session, self, order, &config, out);
280            }
281        }
282        let subscripts = Subscripts::from(subscripts);
283        tensor_read_einsum_into_subscripts(
284            session,
285            self,
286            &subscripts,
287            out,
288            TENSOR_READ_EINSUM_INTO_OP,
289        )
290    }
291}
292
293/// Prepared concrete einsum plan for repeated executions with fixed input
294/// dtype and shape metadata.
295///
296/// Preparing a plan parses and optimizes the contraction tree once. Execution
297/// validates the later inputs against the prepared dtype and shape contract,
298/// then runs the stored tree without re-planning.
299///
300/// # Examples
301///
302/// ```
303/// use tenferro_cpu::CpuEngine;
304/// use tenferro_einsum::ConcreteEinsumPlan;
305/// use tenferro_tensor::Tensor;
306///
307/// let lhs = Tensor::from_vec_col_major(vec![2, 3], vec![1.0_f64; 6]).unwrap();
308/// let rhs = Tensor::from_vec_col_major(vec![3, 4], vec![1.0_f64; 12]).unwrap();
309/// let plan = ConcreteEinsumPlan::prepare(&[&lhs, &rhs], "ij,jk->ik", &Default::default())?;
310///
311/// let mut backend = CpuEngine::serial();
312/// let out = plan.execute(&[&lhs, &rhs], &mut backend)?;
313/// assert_eq!(out.shape(), &[2, 4]);
314/// # Ok::<(), tenferro_tensor::Error>(())
315/// ```
316#[derive(Debug)]
317pub struct ConcreteEinsumPlan {
318    tree: ContractionTree,
319    inputs: Vec<ConcreteEinsumInputSpec>,
320    output_shape: Vec<usize>,
321    binary_dot: Option<crate::binary_dot::BinaryDotPlan>,
322    cpu_plan: crate::cpu_concrete::Cache,
323    /// Per-operand conjugation chosen at preparation (#2552 `EinsumOptions::conj`).
324    conj: crate::api::ConjMask,
325}
326
327impl ConcreteEinsumPlan {
328    /// Release this plan's retained CPU numerical descriptors.
329    ///
330    /// The semantic contraction tree and the owning backend's workspaces are unchanged.
331    ///
332    /// # Examples
333    /// ```
334    /// use tenferro_einsum::ConcreteEinsumPlan;
335    /// use tenferro_tensor::Tensor;
336    /// let a = Tensor::from_vec_col_major(vec![2, 2], vec![1.0_f64; 4])?;
337    /// let mut plan =
338    ///     ConcreteEinsumPlan::prepare(&[&a, &a, &a], "ij,jk,kl->il", &Default::default())?;
339    /// plan.clear_cpu_cache();
340    /// assert_eq!(plan.cpu_cache_stats()?.entries, 0);
341    /// # Ok::<(), Box<dyn std::error::Error>>(())
342    /// ```
343    pub fn clear_cpu_cache(&mut self) {
344        self.cpu_plan.clear();
345    }
346
347    /// Inspect this plan's bounded CPU numerical descriptor cache.
348    ///
349    /// Counts exclude backend-owned scratch and allocator RSS. Retained bytes
350    /// currently describe host layout descriptors and opaque lower plan headers.
351    ///
352    /// # Examples
353    /// ```
354    /// use tenferro_einsum::ConcreteEinsumPlan;
355    /// use tenferro_tensor::Tensor;
356    /// let a = Tensor::from_vec_col_major(vec![2, 2], vec![1.0_f64; 4])?;
357    /// let plan =
358    ///     ConcreteEinsumPlan::prepare(&[&a, &a, &a], "ij,jk,kl->il", &Default::default())?;
359    /// assert_eq!(plan.cpu_cache_stats()?.entries, 0);
360    /// # Ok::<(), Box<dyn std::error::Error>>(())
361    /// ```
362    ///
363    /// # Errors
364    ///
365    /// Returns [`Error`] when no CPU plan cache exists for this plan.
366    pub fn cpu_cache_stats(&self) -> Result<tenferro_tensor::CacheStats> {
367        self.cpu_plan.stats().map_err(Error::from)
368    }
369
370    pub(crate) fn prepare_owned_subscripts<'a, I>(
371        inputs: I,
372        subscripts: &EinsumSubscripts,
373    ) -> Result<Self>
374    where
375        I: AsRef<[&'a Tensor]>,
376    {
377        let subscripts = Subscripts::from(subscripts);
378        Self::prepare_subscripts_internal(input_specs(inputs.as_ref()), &subscripts)
379    }
380
381    pub(crate) fn prepare_read_subscripts<'a, I>(
382        inputs: I,
383        subscripts: &EinsumSubscripts,
384    ) -> Result<Self>
385    where
386        I: AsRef<[TensorRead<'a>]>,
387    {
388        let subscripts = Subscripts::from(subscripts);
389        Self::prepare_subscripts_internal(read_input_specs(inputs.as_ref()), &subscripts)
390    }
391
392    /// Number of binary contraction steps in the prepared tree (diagnostics).
393    pub(crate) fn step_count(&self) -> usize {
394        self.tree.step_count()
395    }
396
397    pub(crate) fn run_owned<'a, I>(&self, inputs: I, session: &mut dyn Engine) -> Result<Tensor>
398    where
399        I: AsRef<[&'a Tensor]>,
400    {
401        let inputs = inputs.as_ref();
402        self.validate_tensor_inputs(inputs, PLAN_EXECUTE_OP)?;
403        if let ([lhs, rhs], Some(plan)) = (inputs, self.binary_dot_shortcut()) {
404            let reads = [TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs)];
405            return execute_binary_dot_shortcut(session, &reads, plan).map_err(Error::from);
406        }
407        if inputs.len() >= 3 {
408            let reads: Vec<_> = inputs
409                .iter()
410                .map(|tensor| TensorRead::from_tensor(tensor))
411                .collect();
412            if let Some(result) = crate::cpu_concrete::execute_cached(
413                session,
414                &reads,
415                &self.tree,
416                Some(&self.cpu_plan),
417            ) {
418                return result.map_err(Error::from);
419            }
420        }
421        eager_einsum_exec(session, inputs, &self.tree).map_err(Error::from)
422    }
423
424    pub(crate) fn run_reads<'a, I>(&self, inputs: I, session: &mut dyn Engine) -> Result<Tensor>
425    where
426        I: AsRef<[TensorRead<'a>]>,
427    {
428        let inputs = inputs.as_ref();
429        self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
430        if let Some(plan) = self.binary_dot_shortcut() {
431            return execute_binary_dot_shortcut(session, inputs, plan).map_err(Error::from);
432        }
433        if let Some(result) =
434            crate::cpu_concrete::execute_cached(session, inputs, &self.tree, Some(&self.cpu_plan))
435        {
436            return result.map_err(Error::from);
437        }
438        eager_einsum_exec_read(session, inputs, &self.tree).map_err(Error::from)
439    }
440
441    pub(crate) fn run_owned_into<'a, I>(
442        &self,
443        inputs: I,
444        session: &mut dyn Engine,
445        out: TensorWrite<'_>,
446    ) -> Result<()>
447    where
448        I: AsRef<[&'a Tensor]>,
449    {
450        let inputs = inputs.as_ref();
451        self.validate_tensor_inputs(inputs, PLAN_EXECUTE_OP)?;
452        self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
453        if let Some(binary_dot) = &self.binary_dot {
454            if let [lhs, rhs] = inputs {
455                let reads = [TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs)];
456                return execute_binary_dot_read_into(session, &reads, binary_dot, out)
457                    .map_err(Error::from);
458            }
459        }
460        let reads: Vec<_> = inputs
461            .iter()
462            .map(|tensor| TensorRead::from_tensor(tensor))
463            .collect();
464        self.execute_cpu_or_fallback_into(session, &reads, out)
465            .map_err(Error::from)
466    }
467
468    pub(crate) fn run_reads_into<'a, I>(
469        &self,
470        inputs: I,
471        session: &mut dyn Engine,
472        out: TensorWrite<'_>,
473    ) -> Result<()>
474    where
475        I: AsRef<[TensorRead<'a>]>,
476    {
477        let inputs = inputs.as_ref();
478        self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
479        self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
480        if let Some(binary_dot) = &self.binary_dot {
481            return execute_binary_dot_read_into(session, inputs, binary_dot, out)
482                .map_err(Error::from);
483        }
484        self.execute_cpu_or_fallback_into(session, inputs, out)
485            .map_err(Error::from)
486    }
487
488    pub(crate) fn run_reads_into_accum<'a, I>(
489        &self,
490        inputs: I,
491        session: &mut dyn Engine,
492        accumulation: DotGeneralAccumulation,
493        out: TensorWrite<'_>,
494    ) -> Result<()>
495    where
496        I: AsRef<[TensorRead<'a>]>,
497    {
498        let inputs = inputs.as_ref();
499        self.validate_read_inputs(inputs, PLAN_EXECUTE_OP)?;
500        self.validate_cached_output(&out, PLAN_EXECUTE_OP)?;
501        if let Some(binary_dot) = &self.binary_dot {
502            return execute_binary_dot_read_into_accum(
503                session,
504                inputs,
505                binary_dot,
506                accumulation,
507                out,
508            )
509            .map_err(Error::from);
510        }
511        if crate::cpu_concrete::supports(session, inputs) {
512            let mut accumulation = accumulation;
513            let conjugated = ConjugatedOperands::apply(session, inputs, &mut accumulation)?;
514            let result = self.run_reads(conjugated.reads(inputs), session)?;
515            let mut out = out;
516            return tenferro_tensor::backend::accumulate_dot_result_into(
517                &result,
518                accumulation,
519                &mut out,
520            )
521            .map_err(Error::from);
522        }
523        eager_einsum_exec_read_into_accum(session, inputs, &self.tree, accumulation, out)
524            .map_err(Error::from)
525    }
526
527    fn execute_cpu_or_fallback_into(
528        &self,
529        session: &mut dyn Engine,
530        inputs: &[TensorRead<'_>],
531        out: TensorWrite<'_>,
532    ) -> tenferro_tensor::Result<()> {
533        if crate::cpu_concrete::supports(session, inputs) {
534            crate::cpu_concrete::execute_into(
535                session,
536                inputs,
537                &self.tree,
538                Some(&self.cpu_plan),
539                out,
540            )
541        } else {
542            eager_einsum_exec_read_into(session, inputs, &self.tree, out)
543        }
544    }
545
546    /// The prepared binary-dot plan when `execute*` may skip the contraction
547    /// tree: operands in written order, same dtype (see
548    /// `binary_dot_shortcut_plan`). The input count and metadata are validated
549    /// by the caller before this is used.
550    fn binary_dot_shortcut(&self) -> Option<&crate::binary_dot::BinaryDotPlan> {
551        let plan = self.binary_dot.as_ref()?;
552        let [lhs, rhs] = self.inputs.as_slice() else {
553            return None;
554        };
555        (plan.operand_order == BinaryDotOperandOrder::Original && lhs.dtype == rhs.dtype)
556            .then_some(plan)
557    }
558
559    fn prepare_subscripts_internal(
560        inputs: Vec<ConcreteEinsumInputSpec>,
561        subscripts: &Subscripts,
562    ) -> Result<Self> {
563        let shapes: Vec<&[usize]> = inputs.iter().map(|input| input.shape.as_slice()).collect();
564        let binary_dot = binary_dot_plan_for_shapes(&shapes, subscripts);
565        let tree = plan_subscripts(subscripts, &shapes)?;
566        let output_shape = tree.output_shape();
567        Ok(Self {
568            tree,
569            inputs,
570            output_shape,
571            binary_dot,
572            cpu_plan: crate::cpu_concrete::Cache::default(),
573            conj: crate::api::ConjMask::default(),
574        })
575    }
576
577    /// Prepare from input metadata with an explicit contraction order and
578    /// conjugation (#2552). The automatic order without annealing plans
579    /// exactly as [`Self::prepare_subscripts_internal`]; every other order is
580    /// honoured as given.
581    pub(crate) fn prepare_planned(
582        inputs: Vec<ConcreteEinsumInputSpec>,
583        subscripts: &Subscripts,
584        optimize: &crate::EinsumOptimize,
585        conj: crate::api::ConjMask,
586    ) -> Result<Self> {
587        if conj.len() > inputs.len() {
588            return Err(Error::invalid_argument(
589                "einsum",
590                "conj",
591                format!(
592                    "the conjugation mask has {} entries for {} inputs",
593                    conj.len(),
594                    inputs.len()
595                ),
596            ));
597        }
598        let mut plan = match optimize {
599            crate::EinsumOptimize::Auto(options) if !options.anneals() => {
600                Self::prepare_subscripts_internal(inputs, subscripts)?
601            }
602            optimize => {
603                if inputs.is_empty() || subscripts.inputs.len() != inputs.len() {
604                    // Same validation and message as the automatic route.
605                    return Self::prepare_subscripts_internal(inputs, subscripts);
606                }
607                let shapes: Vec<&[usize]> =
608                    inputs.iter().map(|input| input.shape.as_slice()).collect();
609                let binary_dot = binary_dot_plan_for_shapes(&shapes, subscripts);
610                let (_, tree) = crate::optimize::resolve_einsum_strategy_with_spec(
611                    optimize.clone(),
612                    subscripts,
613                    &shapes,
614                )?;
615                let output_shape = tree.output_shape();
616                Self {
617                    tree,
618                    inputs,
619                    output_shape,
620                    binary_dot,
621                    cpu_plan: crate::cpu_concrete::Cache::default(),
622                    conj: crate::api::ConjMask::default(),
623                }
624            }
625        };
626        plan.conj = conj;
627        Ok(plan)
628    }
629
630    /// The conjugation chosen at preparation.
631    pub(crate) fn conj_mask(&self) -> &crate::api::ConjMask {
632        &self.conj
633    }
634
635    /// The operand pair contracted at `step` of the prepared tree.
636    #[cfg(test)]
637    pub(crate) fn tree_step_pair(&self, step: usize) -> Option<(usize, usize)> {
638        self.tree.step_pair(step)
639    }
640
641    fn validate_tensor_inputs(&self, actual: &[&Tensor], op: &'static str) -> Result<()> {
642        self.validate_input_metadata(
643            actual.iter().map(|tensor| (tensor.dtype(), tensor.shape())),
644            op,
645        )
646    }
647
648    fn validate_read_inputs(&self, actual: &[TensorRead<'_>], op: &'static str) -> Result<()> {
649        self.validate_input_metadata(
650            actual.iter().map(|tensor| (tensor.dtype(), tensor.shape())),
651            op,
652        )
653    }
654
655    fn validate_input_metadata<'a>(
656        &self,
657        actual: impl ExactSizeIterator<Item = (DType, &'a [usize])>,
658        op: &'static str,
659    ) -> Result<()> {
660        if actual.len() != self.inputs.len() {
661            return Err(Error::invalid_argument(
662                op,
663                "inputs",
664                format!(
665                    "prepared einsum expects {} inputs, got {}",
666                    self.inputs.len(),
667                    actual.len()
668                ),
669            ));
670        }
671        for (expected, (actual_dtype, actual_shape)) in self.inputs.iter().zip(actual) {
672            if expected.dtype != actual_dtype {
673                return Err(Error::dtype_mismatch(op, expected.dtype, actual_dtype));
674            }
675            if expected.shape != actual_shape {
676                return Err(Error::shape_mismatch(
677                    op,
678                    expected.shape.clone(),
679                    actual_shape.to_vec(),
680                ));
681            }
682        }
683        Ok(())
684    }
685
686    fn validate_cached_output(&self, out: &TensorWrite<'_>, op: &'static str) -> Result<()> {
687        let dtype = self
688            .inputs
689            .first()
690            .map(|input| input.dtype)
691            .ok_or_else(|| {
692                Error::invalid_argument(op, "inputs", "einsum requires at least one input tensor")
693            })?;
694        for input in &self.inputs[1..] {
695            if input.dtype != dtype {
696                return Err(Error::dtype_mismatch(op, dtype, input.dtype));
697            }
698        }
699        if out.dtype() != dtype {
700            return Err(Error::dtype_mismatch(op, dtype, out.dtype()));
701        }
702        if out.shape() != self.output_shape {
703            return Err(Error::shape_mismatch(
704                op,
705                out.shape().to_vec(),
706                self.output_shape.clone(),
707            ));
708        }
709        Ok(())
710    }
711}
712
713#[derive(Clone, Debug)]
714pub(crate) struct ConcreteEinsumInputSpec {
715    dtype: DType,
716    shape: Vec<usize>,
717}
718
719fn resolve_shapes(notation: &EinsumNotation, shapes: Vec<&[usize]>) -> Result<Subscripts> {
720    resolve_einsum_notation(notation, &shapes)
721}
722
723fn resolve_tensor_notation(inputs: &[&Tensor], notation: &EinsumNotation) -> Result<Subscripts> {
724    resolve_shapes(
725        notation,
726        inputs.iter().map(|tensor| tensor.shape()).collect(),
727    )
728}
729
730pub(crate) fn resolve_read_notation<'a>(
731    inputs: &[TensorRead<'a>],
732    notation: &EinsumNotation,
733) -> Result<Subscripts> {
734    resolve_shapes(notation, inputs.iter().map(|input| input.shape()).collect())
735}
736
737fn input_specs(inputs: &[&Tensor]) -> Vec<ConcreteEinsumInputSpec> {
738    inputs
739        .iter()
740        .map(|tensor| ConcreteEinsumInputSpec {
741            dtype: tensor.dtype(),
742            shape: tensor.shape().to_vec(),
743        })
744        .collect()
745}
746
747pub(crate) fn read_input_specs(inputs: &[TensorRead<'_>]) -> Vec<ConcreteEinsumInputSpec> {
748    inputs
749        .iter()
750        .map(|tensor| ConcreteEinsumInputSpec {
751            dtype: tensor.dtype(),
752            shape: tensor.shape().to_vec(),
753        })
754        .collect()
755}
756
757fn read_binary_dot_config_for_labels<L: Copy + PartialEq>(
758    inputs: &[TensorRead<'_>],
759    lhs_labels: &[L],
760    rhs_labels: &[L],
761    output_labels: &[L],
762    out: &TensorWrite<'_>,
763) -> Option<(BinaryDotOperandOrder, tenferro_tensor::DotGeneralConfig)> {
764    if inputs.len() != 2
765        || inputs[0].dtype() != inputs[1].dtype()
766        || out.dtype() != inputs[0].dtype()
767    {
768        return None;
769    }
770    binary_dot_config_for_into(
771        inputs[0].shape(),
772        inputs[1].shape(),
773        lhs_labels,
774        rhs_labels,
775        output_labels,
776        out.shape(),
777    )
778}
779
780fn read_binary_dot_config(
781    inputs: &[TensorRead<'_>],
782    subscripts: &Subscripts,
783    out: &TensorWrite<'_>,
784) -> Option<(BinaryDotOperandOrder, tenferro_tensor::DotGeneralConfig)> {
785    let [lhs, rhs] = subscripts.inputs.as_slice() else {
786        return None;
787    };
788    read_binary_dot_config_for_labels(inputs, lhs, rhs, &subscripts.output, out)
789}
790
791fn execute_binary_dot_config_read_into(
792    session: &mut dyn Engine,
793    inputs: &[TensorRead<'_>],
794    order: BinaryDotOperandOrder,
795    config: &tenferro_tensor::DotGeneralConfig,
796    out: TensorWrite<'_>,
797) -> Result<()> {
798    let (lhs, rhs) = match order {
799        BinaryDotOperandOrder::Original => (0, 1),
800        BinaryDotOperandOrder::Swapped => (1, 0),
801    };
802    session
803        .dot_general_read_into(inputs[lhs].clone(), inputs[rhs].clone(), config, out)
804        .map_err(Error::from)
805}
806
807fn tensor_einsum_into_subscripts(
808    session: &mut dyn Engine,
809    inputs: &[&Tensor],
810    subscripts: &Subscripts,
811    out: TensorWrite<'_>,
812    op: &'static str,
813) -> Result<()> {
814    if inputs.len() == 2 {
815        let reads = [
816            TensorRead::from_tensor(inputs[0]),
817            TensorRead::from_tensor(inputs[1]),
818        ];
819        if let Some((order, config)) = read_binary_dot_config(&reads, subscripts, &out) {
820            return execute_binary_dot_config_read_into(session, &reads, order, &config, out);
821        }
822    }
823    let plan = ConcreteEinsumPlan::prepare_subscripts_internal(input_specs(inputs), subscripts)?;
824    validate_output(&plan.inputs, &plan.tree, &out, op)?;
825    plan.run_owned_into(inputs, session, out)
826}
827
828fn parse_fast_ascii_binary_labels(notation: &str) -> Option<(&[u8], &[u8], &[u8])> {
829    let (terms, output) = notation.split_once("->")?;
830    let (lhs, rhs) = terms.split_once(',')?;
831    // Borrow labels directly. Separators or unsupported labels in any term send
832    // the entire expression through the canonical parser, including its errors.
833    [lhs, rhs, output]
834        .iter()
835        .all(|term| term.bytes().all(|byte| byte.is_ascii_alphabetic()))
836        .then_some((lhs.as_bytes(), rhs.as_bytes(), output.as_bytes()))
837}
838
839fn borrowed_notation_labels(notation: &EinsumNotation) -> Option<[TinyVec<[u32; 8]>; 3]> {
840    let [lhs, rhs] = notation.inputs.as_slice() else {
841        return None;
842    };
843    let labels = |axes: &[crate::EinsumAxis]| {
844        axes.iter()
845            .map(|axis| match axis {
846                crate::EinsumAxis::Label(label) => Some(*label),
847                crate::EinsumAxis::Ellipsis => None,
848            })
849            .collect::<Option<TinyVec<[u32; 8]>>>()
850    };
851    Some([labels(lhs)?, labels(rhs)?, labels(&notation.output)?])
852}
853
854fn tensor_read_einsum_into_subscripts(
855    session: &mut dyn Engine,
856    inputs: &[TensorRead<'_>],
857    subscripts: &Subscripts,
858    out: TensorWrite<'_>,
859    op: &'static str,
860) -> Result<()> {
861    if let Some((order, config)) = read_binary_dot_config(inputs, subscripts, &out) {
862        return execute_binary_dot_config_read_into(session, inputs, order, &config, out);
863    }
864    let plan =
865        ConcreteEinsumPlan::prepare_subscripts_internal(read_input_specs(inputs), subscripts)?;
866    validate_output(&plan.inputs, &plan.tree, &out, op)?;
867    plan.run_reads_into(inputs, session, out)
868}
869
870fn validate_output(
871    inputs: &[ConcreteEinsumInputSpec],
872    tree: &ContractionTree,
873    out: &TensorWrite<'_>,
874    op: &'static str,
875) -> Result<()> {
876    let expected = output_spec(inputs, tree, op)?;
877    if out.dtype() != expected.dtype {
878        return Err(Error::dtype_mismatch(op, expected.dtype, out.dtype()));
879    }
880    if out.shape() != expected.shape.as_slice() {
881        return Err(Error::shape_mismatch(
882            op,
883            out.shape().to_vec(),
884            expected.shape.clone(),
885        ));
886    }
887    Ok(())
888}
889
890fn output_spec(
891    inputs: &[ConcreteEinsumInputSpec],
892    tree: &ContractionTree,
893    op: &'static str,
894) -> Result<ConcreteEinsumInputSpec> {
895    let dtype = inputs
896        .first()
897        .ok_or_else(|| {
898            Error::invalid_argument(op, "inputs", "einsum requires at least one input tensor")
899        })?
900        .dtype;
901    for input in inputs {
902        if input.dtype != dtype {
903            return Err(Error::dtype_mismatch(op, dtype, input.dtype));
904        }
905    }
906
907    for (input, labels) in inputs.iter().zip(tree.subscripts.inputs.iter()) {
908        if labels.len() != input.shape.len() {
909            return Err(Error::rank_mismatch(op, labels.len(), input.shape.len()));
910        }
911    }
912    let output_shape = tree.output_shape();
913    if output_shape.len() != tree.subscripts.output.len() {
914        return Err(Error::invalid_argument(
915            op,
916            "output labels",
917            "an output label is missing from all inputs",
918        ));
919    }
920    Ok(ConcreteEinsumInputSpec {
921        dtype,
922        shape: output_shape,
923    })
924}