Skip to main content

tenferro_linalg/
tensor_ext.rs

1//! Crate-internal concrete linear algebra bodies behind `LinalgOps` (#2552).
2//!
3//! Owned tensors use [`OwnedLinalg`] and borrowed reads use [`ReadLinalg`];
4//! the public grammar forwards to one of them. All methods dispatch internally to the built-in
5//! CPU/CUDA execution sessions through an erased `&mut dyn Engine`
6//! (issue #1680 Phase 3); third-party [`LinalgBackend`] implementations
7//! remain supported through the SPI trait, but the concrete op path is
8//! built-in-session only.
9
10use num_complex::{Complex32, Complex64};
11use tenferro_cpu::linalg_interop::PoolScalar;
12use tenferro_cpu::with_cpu_engine;
13#[cfg(feature = "cuda")]
14use tenferro_gpu::cuda::raw::with_cuda_engine;
15use tenferro_tensor::{
16    CompareDir, DType, DotGeneralConfig, Engine, Tensor, TensorRead, TensorScalar, TensorWrite,
17};
18
19use crate::extension::{EighOptions, QrOptions, SvdOptions};
20use crate::lane;
21use crate::traced::real_values_dtype;
22use crate::validation::validate_square_matrix;
23use crate::{LinalgBackend, RankRevealingQrOptions, RankRevealingQrResult};
24
25/// Scalar types supported by statically typed linear algebra methods: `f32`, `f64`,
26/// `Complex32` and `Complex64`.
27///
28/// The trait is sealed: its supertrait lives in a private module, so no other type can implement
29/// it. It names the two output scalars a factorization of `Self` can return.
30///
31/// | `Self` | [`RealScalar`](LinalgScalar::RealScalar) | [`Complex`](LinalgScalar::Complex) |
32/// | --- | --- | --- |
33/// | `f32` | `f32` | `Complex32` |
34/// | `f64` | `f64` | `Complex64` |
35/// | `Complex32` | `f32` | `Complex32` |
36/// | `Complex64` | `f64` | `Complex64` |
37///
38/// # Examples
39///
40/// ```rust
41/// use num_complex::Complex64;
42/// use tenferro_linalg::LinalgScalar;
43///
44/// fn accepts_linalg_scalar<T: LinalgScalar>() {}
45/// accepts_linalg_scalar::<f64>();
46///
47/// fn spectrum_scalar<T: LinalgScalar>() -> &'static str {
48///     std::any::type_name::<T::RealScalar>()
49/// }
50/// assert_eq!(spectrum_scalar::<Complex64>(), "f64");
51/// ```
52pub trait LinalgScalar: TensorScalar + private::Sealed {
53    /// Complex counterpart used by general eigendecomposition.
54    type Complex: TensorScalar + PoolScalar + Default;
55    /// Real scalar of singular values and Hermitian eigenvalues.
56    type RealScalar: TensorScalar + PoolScalar + Default;
57}
58
59pub(crate) mod private {
60    use tcontract_exec::Exec;
61    use tenferro_cpu::linalg_interop::BufferPool;
62    use tenferro_tensor::{TypedTensor, TypedTensorView};
63
64    /// `(U, S, Vᴴ)` of the borrowed SVD entry.
65    pub type Svd<T> = (
66        TypedTensor<T>,
67        TypedTensor<<T as super::LinalgScalar>::RealScalar>,
68        TypedTensor<T>,
69    );
70
71    /// The sealing supertrait of [`LinalgScalar`](super::LinalgScalar).
72    ///
73    /// `pub` inside a private module: nameable as a supertrait, unreachable outside the crate.
74    /// Its methods route the public borrowed-`Exec` entries to the compiled provider route's typed
75    /// bodies per concrete type, which is what lets the public bound stay free of the provider's
76    /// own scalar traits (whose `Real`/`Complex` names would otherwise collide with
77    /// `LinalgScalar::Complex`).
78    pub trait Sealed {
79        /// The body of the public `svd_on_exec`.
80        ///
81        /// # Errors
82        ///
83        /// As the public `svd_on_exec`: `ValidationError::RankMismatch` below rank 2,
84        /// `Error::Unsupported` for a negative stride, `Error::RuntimeState` for non-host storage,
85        /// `ValidationError::InvalidArgument` for an overflowing count or out-of-range layout, and
86        /// a provider failure mapped by `map_error`.
87        fn svd_on_exec(
88            exec: &mut Exec<'_>,
89            buffers: &mut BufferPool,
90            view: &TypedTensorView<'_, Self>,
91            full: bool,
92        ) -> tenferro_tensor::Result<Svd<Self>>
93        where
94            Self: super::LinalgScalar + Sized;
95        /// The body of the public `svd_values_on_exec`.
96        ///
97        /// # Errors
98        ///
99        /// As the public `svd_values_on_exec`: `ValidationError::RankMismatch` below rank 2,
100        /// `Error::Unsupported` for a negative stride, `Error::RuntimeState` for non-host storage,
101        /// `ValidationError::InvalidArgument` for an overflowing count or out-of-range layout, and
102        /// a provider failure mapped by `map_error`.
103        fn svd_values_on_exec(
104            exec: &mut Exec<'_>,
105            buffers: &mut BufferPool,
106            view: &TypedTensorView<'_, Self>,
107        ) -> tenferro_tensor::Result<TypedTensor<<Self as super::LinalgScalar>::RealScalar>>
108        where
109            Self: super::LinalgScalar + Sized;
110        /// The body of the public `qr_on_exec`.
111        ///
112        /// # Errors
113        ///
114        /// As the public `qr_on_exec`: `ValidationError::RankMismatch` below rank 2,
115        /// `Error::Unsupported` for a negative stride, `Error::RuntimeState` for non-host storage,
116        /// `ValidationError::InvalidArgument` for an overflowing count or out-of-range layout, and
117        /// a provider failure mapped by `map_error`.
118        fn qr_on_exec(
119            exec: &mut Exec<'_>,
120            buffers: &mut BufferPool,
121            view: &TypedTensorView<'_, Self>,
122        ) -> tenferro_tensor::Result<(TypedTensor<Self>, TypedTensor<Self>)>
123        where
124            Self: super::LinalgScalar + Sized;
125        /// The body of the public `eigh_on_exec`.
126        ///
127        /// # Errors
128        ///
129        /// As the public `eigh_on_exec`: `ValidationError::RankMismatch` below rank 2, `ValidationError::ShapeMismatch` for a non-square matrix,
130        /// `Error::Unsupported` for a negative stride, `Error::RuntimeState` for non-host storage,
131        /// `ValidationError::InvalidArgument` for an overflowing count or out-of-range layout, and
132        /// a provider failure mapped by `map_error`.
133        #[allow(clippy::type_complexity)]
134        fn eigh_on_exec(
135            exec: &mut Exec<'_>,
136            buffers: &mut BufferPool,
137            view: &TypedTensorView<'_, Self>,
138        ) -> tenferro_tensor::Result<(
139            TypedTensor<<Self as super::LinalgScalar>::RealScalar>,
140            TypedTensor<Self>,
141        )>
142        where
143            Self: super::LinalgScalar + Sized;
144        /// The body of the public `eigh_values_on_exec`.
145        ///
146        /// # Errors
147        ///
148        /// As the public `eigh_values_on_exec`: `ValidationError::RankMismatch` below rank 2, `ValidationError::ShapeMismatch` for a non-square matrix,
149        /// `Error::Unsupported` for a negative stride, `Error::RuntimeState` for non-host storage,
150        /// `ValidationError::InvalidArgument` for an overflowing count or out-of-range layout, and
151        /// a provider failure mapped by `map_error`.
152        fn eigh_values_on_exec(
153            exec: &mut Exec<'_>,
154            buffers: &mut BufferPool,
155            view: &TypedTensorView<'_, Self>,
156        ) -> tenferro_tensor::Result<TypedTensor<<Self as super::LinalgScalar>::RealScalar>>
157        where
158            Self: super::LinalgScalar + Sized;
159        /// The body of the public `eig_values_on_exec`.
160        ///
161        /// # Errors
162        ///
163        /// As the public `eig_values_on_exec`: `ValidationError::RankMismatch` below rank 2, `ValidationError::ShapeMismatch` for a non-square matrix,
164        /// `Error::Unsupported` for a negative stride, `Error::RuntimeState` for non-host storage,
165        /// `ValidationError::InvalidArgument` for an overflowing count or out-of-range layout, and
166        /// a provider failure mapped by `map_error`.
167        fn eig_values_on_exec(
168            exec: &mut Exec<'_>,
169            buffers: &mut BufferPool,
170            view: &TypedTensorView<'_, Self>,
171        ) -> tenferro_tensor::Result<TypedTensor<<Self as super::LinalgScalar>::Complex>>
172        where
173            Self: super::LinalgScalar + Sized;
174    }
175
176    macro_rules! impl_sealed {
177        ($($scalar:ty),*) => {$(
178            impl Sealed for $scalar {
179                fn svd_on_exec(
180                    exec: &mut Exec<'_>,
181                    buffers: &mut BufferPool,
182                    view: &TypedTensorView<'_, Self>,
183                    full: bool,
184                ) -> tenferro_tensor::Result<Svd<Self>> {
185                    crate::cpu::linalg::faer_linalg::svd_typed(exec, buffers, view, full)
186                }
187                fn svd_values_on_exec(
188                    exec: &mut Exec<'_>,
189                    buffers: &mut BufferPool,
190                    view: &TypedTensorView<'_, Self>,
191                ) -> tenferro_tensor::Result<TypedTensor<<Self as super::LinalgScalar>::RealScalar>> {
192                    crate::cpu::linalg::faer_linalg::svd_values_typed(exec, buffers, view)
193                }
194                fn qr_on_exec(
195                    exec: &mut Exec<'_>,
196                    buffers: &mut BufferPool,
197                    view: &TypedTensorView<'_, Self>,
198                ) -> tenferro_tensor::Result<(TypedTensor<Self>, TypedTensor<Self>)> {
199                    crate::cpu::linalg::faer_linalg::qr_typed(exec, buffers, view)
200                }
201                fn eigh_on_exec(
202                    exec: &mut Exec<'_>,
203                    buffers: &mut BufferPool,
204                    view: &TypedTensorView<'_, Self>,
205                ) -> tenferro_tensor::Result<(
206                    TypedTensor<<Self as super::LinalgScalar>::RealScalar>,
207                    TypedTensor<Self>,
208                )> {
209                    crate::cpu::linalg::faer_linalg::eigh_typed(exec, buffers, view)
210                }
211                fn eigh_values_on_exec(
212                    exec: &mut Exec<'_>,
213                    buffers: &mut BufferPool,
214                    view: &TypedTensorView<'_, Self>,
215                ) -> tenferro_tensor::Result<TypedTensor<<Self as super::LinalgScalar>::RealScalar>> {
216                    crate::cpu::linalg::faer_linalg::eigh_values_typed(exec, buffers, view)
217                }
218                fn eig_values_on_exec(
219                    exec: &mut Exec<'_>,
220                    buffers: &mut BufferPool,
221                    view: &TypedTensorView<'_, Self>,
222                ) -> tenferro_tensor::Result<TypedTensor<<Self as super::LinalgScalar>::Complex>> {
223                    crate::cpu::linalg::faer_linalg::eig_values_typed(exec, buffers, view)
224                }
225            }
226        )*};
227    }
228    impl_sealed!(f32, f64, num_complex::Complex32, num_complex::Complex64);
229}
230
231impl LinalgScalar for f32 {
232    type Complex = Complex32;
233    type RealScalar = f32;
234}
235impl LinalgScalar for f64 {
236    type Complex = Complex64;
237    type RealScalar = f64;
238}
239impl LinalgScalar for Complex32 {
240    type Complex = Complex32;
241    type RealScalar = f32;
242}
243impl LinalgScalar for Complex64 {
244    type Complex = Complex64;
245    type RealScalar = f64;
246}
247
248/// Crate-internal bodies of the owned-tensor linalg entries; the public
249/// grammar is `LinalgOps` (#2552), which forwards here.
250pub(crate) trait OwnedLinalg {
251    fn svd(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)>;
252    fn svdvals(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
253    fn svd_with_options(
254        &self,
255        options: SvdOptions,
256        session: &mut dyn Engine,
257    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)>;
258    fn svd_full(
259        &self,
260        session: &mut dyn Engine,
261    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)>;
262    fn qr(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)>;
263
264    fn householder_qr(
265        &self,
266        session: &mut dyn Engine,
267    ) -> tenferro_tensor::Result<crate::HouseholderQr<Tensor>>;
268
269    fn qr_with_options(
270        &self,
271        options: QrOptions,
272        session: &mut dyn Engine,
273    ) -> tenferro_tensor::Result<(Tensor, Tensor)>;
274    fn rank_revealing_qr(
275        &self,
276        options: RankRevealingQrOptions,
277        session: &mut dyn Engine,
278    ) -> tenferro_tensor::Result<RankRevealingQrResult<Tensor>>;
279    fn lu(
280        &self,
281        session: &mut dyn Engine,
282    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor)>;
283    fn full_piv_lu(
284        &self,
285        session: &mut dyn Engine,
286    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor, Tensor)>;
287    fn full_piv_lu_solve(
288        &self,
289        b: &Tensor,
290        session: &mut dyn Engine,
291    ) -> tenferro_tensor::Result<Tensor>;
292    fn solve(&self, b: &Tensor, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
293    fn cholesky(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
294    fn eigh(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)>;
295    fn eigh_with_options(
296        &self,
297        options: EighOptions,
298        session: &mut dyn Engine,
299    ) -> tenferro_tensor::Result<(Tensor, Tensor)>;
300    fn eig(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)>;
301    #[allow(clippy::too_many_arguments)]
302    fn triangular_solve(
303        &self,
304        b: &Tensor,
305        left_side: bool,
306        lower: bool,
307        transpose_a: bool,
308        unit_diagonal: bool,
309        session: &mut dyn Engine,
310    ) -> tenferro_tensor::Result<Tensor>;
311    fn slogdet(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)>;
312    fn det(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
313    fn inv(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
314    fn eigvalsh(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
315    fn eigvals(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
316    fn pinv(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
317    fn pinv_with_rtol(
318        &self,
319        rtol: f64,
320        session: &mut dyn Engine,
321    ) -> tenferro_tensor::Result<Tensor>;
322    fn norm(
323        &self,
324        ord: Option<f64>,
325        dim: Option<&[usize]>,
326        keepdim: bool,
327        session: &mut dyn Engine,
328    ) -> tenferro_tensor::Result<Tensor>;
329}
330
331/// Crate-internal bodies of the borrowed-read linalg entries; the public
332/// grammar is `LinalgOps` (#2552), which forwards here.
333pub(crate) trait ReadLinalg {
334    fn svd_read(
335        self,
336        session: &mut dyn Engine,
337    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)>;
338    fn svdvals_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
339    fn svd_with_options_read(
340        self,
341        options: SvdOptions,
342        session: &mut dyn Engine,
343    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)>;
344    fn svd_full_read(
345        self,
346        session: &mut dyn Engine,
347    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)>;
348    fn qr_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)>;
349    fn qr_with_options_read(
350        self,
351        options: QrOptions,
352        session: &mut dyn Engine,
353    ) -> tenferro_tensor::Result<(Tensor, Tensor)>;
354    fn rank_revealing_qr_read(
355        self,
356        options: RankRevealingQrOptions,
357        session: &mut dyn Engine,
358    ) -> tenferro_tensor::Result<RankRevealingQrResult<Tensor>>;
359    fn lu_read(
360        self,
361        session: &mut dyn Engine,
362    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor)>;
363    fn full_piv_lu_read(
364        self,
365        session: &mut dyn Engine,
366    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor, Tensor)>;
367    fn full_piv_lu_solve_read(
368        self,
369        b: TensorRead<'_>,
370        session: &mut dyn Engine,
371    ) -> tenferro_tensor::Result<Tensor>;
372    fn solve_read(
373        self,
374        b: TensorRead<'_>,
375        session: &mut dyn Engine,
376    ) -> tenferro_tensor::Result<Tensor>;
377    fn solve_read_into(
378        self,
379        b: TensorRead<'_>,
380        out: TensorWrite<'_>,
381        session: &mut dyn Engine,
382    ) -> tenferro_tensor::Result<()>
383    where
384        Self: Sized,
385    {
386        let _ = (self, b, out, session);
387        Err(tenferro_tensor::Error::unsupported(
388            "solve_read_into",
389            "this tensor-read extension implementation does not accept borrowed solve targets",
390        ))
391    }
392    fn cholesky_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
393    fn eigh_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)>;
394    fn eigh_with_options_read(
395        self,
396        options: EighOptions,
397        session: &mut dyn Engine,
398    ) -> tenferro_tensor::Result<(Tensor, Tensor)>;
399    fn eig_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)>;
400    #[allow(clippy::too_many_arguments)]
401    fn triangular_solve_read(
402        self,
403        b: TensorRead<'_>,
404        left_side: bool,
405        lower: bool,
406        transpose_a: bool,
407        unit_diagonal: bool,
408        session: &mut dyn Engine,
409    ) -> tenferro_tensor::Result<Tensor>;
410    fn slogdet_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)>;
411    fn det_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
412    fn inv_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
413    fn eigvalsh_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
414    fn eigvals_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
415    fn pinv_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor>;
416    fn pinv_with_rtol_read(
417        self,
418        rtol: f64,
419        session: &mut dyn Engine,
420    ) -> tenferro_tensor::Result<Tensor>;
421    fn norm_read(
422        self,
423        ord: Option<f64>,
424        dim: Option<&[usize]>,
425        keepdim: bool,
426        session: &mut dyn Engine,
427    ) -> tenferro_tensor::Result<Tensor>;
428}
429
430impl OwnedLinalg for Tensor {
431    fn svd(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)> {
432        if let Some(result) =
433            lane::try_lane_svd(session, TensorRead::from_tensor(self), false, None)
434        {
435            return result;
436        }
437        with_linalg_backend(session, "svd", |backend| three(backend.svd(self)?, "svd"))
438    }
439    fn svdvals(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
440        if let Some(result) = lane::try_lane_svdvals(session, TensorRead::from_tensor(self)) {
441            return result;
442        }
443        with_linalg_backend(session, "svdvals", |backend| backend.svd_values(self))
444    }
445
446    fn svd_with_options(
447        &self,
448        options: SvdOptions,
449        session: &mut dyn Engine,
450    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)> {
451        if let Some(result) = lane::try_lane_svd(
452            session,
453            TensorRead::from_tensor(self),
454            false,
455            Some(("svd_with_options", options)),
456        ) {
457            return result;
458        }
459        with_linalg_backend(session, "svd_with_options", |backend| {
460            three(backend.svd_with_options(self, options)?, "svd_with_options")
461        })
462    }
463    fn svd_full(
464        &self,
465        session: &mut dyn Engine,
466    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)> {
467        if let Some(result) = lane::try_lane_svd(session, TensorRead::from_tensor(self), true, None)
468        {
469            return result;
470        }
471        with_linalg_backend(session, "svd_full", |backend| {
472            three(backend.svd_full(self)?, "svd_full")
473        })
474    }
475    fn qr(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)> {
476        if let Some(result) = lane::try_lane_qr(session, TensorRead::from_tensor(self)) {
477            return result;
478        }
479        with_linalg_backend(session, "qr", |backend| two(backend.qr(self)?, "qr"))
480    }
481    fn householder_qr(
482        &self,
483        session: &mut dyn Engine,
484    ) -> tenferro_tensor::Result<crate::HouseholderQr<Tensor>> {
485        with_linalg_backend(session, "householder_qr", |backend| {
486            backend
487                .householder_qr(self)
488                .map(crate::HouseholderQr::from_backend)
489        })
490    }
491    fn qr_with_options(
492        &self,
493        options: QrOptions,
494        session: &mut dyn Engine,
495    ) -> tenferro_tensor::Result<(Tensor, Tensor)> {
496        with_linalg_backend(session, "qr_with_options", |backend| {
497            two(backend.qr_with_options(self, options)?, "qr_with_options")
498        })
499    }
500    fn rank_revealing_qr(
501        &self,
502        options: RankRevealingQrOptions,
503        session: &mut dyn Engine,
504    ) -> tenferro_tensor::Result<RankRevealingQrResult<Tensor>> {
505        with_linalg_backend(session, "rank_revealing_qr", |backend| {
506            rank_revealing_qr_result(backend.rank_revealing_qr(self, options)?)
507        })
508    }
509    fn lu(
510        &self,
511        session: &mut dyn Engine,
512    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor)> {
513        with_linalg_backend(session, "lu", |backend| four(backend.lu(self)?, "lu"))
514    }
515    fn full_piv_lu(
516        &self,
517        session: &mut dyn Engine,
518    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor, Tensor)> {
519        with_linalg_backend(session, "full_piv_lu", |backend| {
520            five(backend.full_piv_lu(self)?, "full_piv_lu")
521        })
522    }
523    fn full_piv_lu_solve(
524        &self,
525        b: &Tensor,
526        session: &mut dyn Engine,
527    ) -> tenferro_tensor::Result<Tensor> {
528        with_linalg_backend(session, "full_piv_lu_solve", |backend| {
529            backend.full_piv_lu_solve(self, b, false)
530        })
531    }
532    fn solve(&self, b: &Tensor, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
533        with_linalg_backend(session, "solve", |backend| backend.solve(self, b))
534    }
535    fn cholesky(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
536        with_linalg_backend(session, "cholesky", |backend| backend.cholesky(self))
537    }
538    fn eigh(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)> {
539        if let Some(result) = lane::try_lane_eigh(session, TensorRead::from_tensor(self), None) {
540            return result;
541        }
542        with_linalg_backend(session, "eigh", |backend| two(backend.eigh(self)?, "eigh"))
543    }
544    fn eigh_with_options(
545        &self,
546        options: EighOptions,
547        session: &mut dyn Engine,
548    ) -> tenferro_tensor::Result<(Tensor, Tensor)> {
549        if let Some(result) = lane::try_lane_eigh(
550            session,
551            TensorRead::from_tensor(self),
552            Some(("eigh_with_options", options)),
553        ) {
554            return result;
555        }
556        with_linalg_backend(session, "eigh_with_options", |backend| {
557            two(
558                backend.eigh_with_options(self, options)?,
559                "eigh_with_options",
560            )
561        })
562    }
563    fn eig(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)> {
564        with_linalg_backend(session, "eig", |backend| two(backend.eig(self)?, "eig"))
565    }
566    fn triangular_solve(
567        &self,
568        b: &Tensor,
569        left_side: bool,
570        lower: bool,
571        transpose_a: bool,
572        unit_diagonal: bool,
573        session: &mut dyn Engine,
574    ) -> tenferro_tensor::Result<Tensor> {
575        with_linalg_backend(session, "triangular_solve", |backend| {
576            backend.triangular_solve(self, b, left_side, lower, transpose_a, unit_diagonal)
577        })
578    }
579    fn slogdet(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)> {
580        validate_square_matrix("slogdet", self.shape())?;
581        if self.shape()[0] == 0 && self.shape()[1] == 0 {
582            return empty_square_slogdet_on(session, "slogdet", TensorRead::from_tensor(self));
583        }
584        with_linalg_backend(session, "slogdet", |backend| {
585            slogdet_from_lu(four(backend.lu(self)?, "slogdet")?, backend)
586        })
587    }
588    fn det(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
589        validate_square_matrix("det", self.shape())?;
590        with_linalg_backend(session, "det", |backend| {
591            det_impl(self.slogdet(backend)?, backend)
592        })
593    }
594    fn inv(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
595        with_linalg_backend(session, "inv", |backend| inv_owned(self, backend))
596    }
597    fn eigvalsh(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
598        if let Some(result) = lane::try_lane_eigvalsh(session, TensorRead::from_tensor(self)) {
599            return result;
600        }
601        with_linalg_backend(session, "eigvalsh", |backend| backend.eigh_values(self))
602    }
603    fn eigvals(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
604        if let Some(result) = lane::try_lane_eigvals(session, TensorRead::from_tensor(self)) {
605            return result;
606        }
607        with_linalg_backend(session, "eigvals", |backend| backend.eig_values(self))
608    }
609    fn pinv(&self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
610        with_linalg_backend(session, "pinv", |backend| pinv_owned(self, None, backend))
611    }
612    fn pinv_with_rtol(
613        &self,
614        rtol: f64,
615        session: &mut dyn Engine,
616    ) -> tenferro_tensor::Result<Tensor> {
617        with_linalg_backend(session, "pinv_with_rtol", |backend| {
618            pinv_owned(self, Some(rtol), backend)
619        })
620    }
621    fn norm(
622        &self,
623        ord: Option<f64>,
624        dim: Option<&[usize]>,
625        keepdim: bool,
626        session: &mut dyn Engine,
627    ) -> tenferro_tensor::Result<Tensor> {
628        with_linalg_backend(session, "norm", |backend| {
629            norm_from_read(TensorRead::from_tensor(self), ord, dim, keepdim, backend)
630        })
631    }
632}
633
634impl ReadLinalg for TensorRead<'_> {
635    fn svd_read(
636        self,
637        session: &mut dyn Engine,
638    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)> {
639        if let Some(result) = lane::try_lane_svd(session, self.clone(), false, None) {
640            return result;
641        }
642        with_linalg_backend(session, "svd_read", |backend| {
643            three(backend.svd_read(self)?, "svd_read")
644        })
645    }
646    fn svdvals_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
647        if let Some(result) = lane::try_lane_svdvals(session, self.clone()) {
648            return result;
649        }
650        with_linalg_backend(session, "svdvals_read", |backend| {
651            backend.svd_values_read(self)
652        })
653    }
654
655    fn svd_with_options_read(
656        self,
657        options: SvdOptions,
658        session: &mut dyn Engine,
659    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)> {
660        if let Some(result) = lane::try_lane_svd(
661            session,
662            self.clone(),
663            false,
664            Some(("svd_with_options_read", options)),
665        ) {
666            return result;
667        }
668        with_linalg_backend(session, "svd_with_options_read", |backend| {
669            three(
670                backend.svd_with_options_read(self, options)?,
671                "svd_with_options_read",
672            )
673        })
674    }
675    fn svd_full_read(
676        self,
677        session: &mut dyn Engine,
678    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)> {
679        if let Some(result) = lane::try_lane_svd(session, self.clone(), true, None) {
680            return result;
681        }
682        with_linalg_backend(session, "svd_full_read", |backend| {
683            three(backend.svd_full_read(self)?, "svd_full_read")
684        })
685    }
686    fn qr_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)> {
687        if let Some(result) = lane::try_lane_qr(session, self.clone()) {
688            return result;
689        }
690        with_linalg_backend(session, "qr_read", |backend| {
691            two(backend.qr_read(self)?, "qr_read")
692        })
693    }
694    fn qr_with_options_read(
695        self,
696        options: QrOptions,
697        session: &mut dyn Engine,
698    ) -> tenferro_tensor::Result<(Tensor, Tensor)> {
699        with_linalg_backend(session, "qr_with_options_read", |backend| {
700            two(
701                backend.qr_with_options_read(self, options)?,
702                "qr_with_options_read",
703            )
704        })
705    }
706    fn rank_revealing_qr_read(
707        self,
708        options: RankRevealingQrOptions,
709        session: &mut dyn Engine,
710    ) -> tenferro_tensor::Result<RankRevealingQrResult<Tensor>> {
711        with_linalg_backend(session, "rank_revealing_qr_read", |backend| {
712            rank_revealing_qr_result(backend.rank_revealing_qr_read(self, options)?)
713        })
714    }
715    fn lu_read(
716        self,
717        session: &mut dyn Engine,
718    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor)> {
719        with_linalg_backend(session, "lu_read", |backend| {
720            four(backend.lu_read(self)?, "lu_read")
721        })
722    }
723    fn full_piv_lu_read(
724        self,
725        session: &mut dyn Engine,
726    ) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor, Tensor)> {
727        with_linalg_backend(session, "full_piv_lu_read", |backend| {
728            five(backend.full_piv_lu_read(self)?, "full_piv_lu_read")
729        })
730    }
731    fn full_piv_lu_solve_read(
732        self,
733        b: TensorRead<'_>,
734        session: &mut dyn Engine,
735    ) -> tenferro_tensor::Result<Tensor> {
736        with_linalg_backend(session, "full_piv_lu_solve_read", |backend| {
737            // Every surface goes through the backend solve, which rejects a rank-deficient
738            // system (a pivot at or below `eps * max|pivot|`). A borrowed view is made compact
739            // first.
740            let a_owned;
741            let a = match self.as_tensor() {
742                Some(a) => a,
743                None => {
744                    let shape = self.shape().to_vec();
745                    a_owned = backend.reshape_read(self, &shape)?;
746                    &a_owned
747                }
748            };
749            let b_owned;
750            let b = match b.as_tensor() {
751                Some(b) => b,
752                None => {
753                    let shape = b.shape().to_vec();
754                    b_owned = backend.reshape_read(b, &shape)?;
755                    &b_owned
756                }
757            };
758            backend.full_piv_lu_solve(a, b, false)
759        })
760    }
761    fn solve_read(
762        self,
763        b: TensorRead<'_>,
764        session: &mut dyn Engine,
765    ) -> tenferro_tensor::Result<Tensor> {
766        with_linalg_backend(session, "solve_read", |backend| backend.solve_read(self, b))
767    }
768    fn solve_read_into(
769        self,
770        b: TensorRead<'_>,
771        out: TensorWrite<'_>,
772        session: &mut dyn Engine,
773    ) -> tenferro_tensor::Result<()> {
774        with_linalg_backend(session, "solve_read_into", |backend| {
775            backend.solve_read_into(self, b, out)
776        })
777    }
778    fn cholesky_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
779        with_linalg_backend(session, "cholesky_read", |backend| {
780            backend.cholesky_read(self)
781        })
782    }
783    fn eigh_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)> {
784        if let Some(result) = lane::try_lane_eigh(session, self.clone(), None) {
785            return result;
786        }
787        with_linalg_backend(session, "eigh_read", |backend| {
788            two(backend.eigh_read(self)?, "eigh_read")
789        })
790    }
791    fn eigh_with_options_read(
792        self,
793        options: EighOptions,
794        session: &mut dyn Engine,
795    ) -> tenferro_tensor::Result<(Tensor, Tensor)> {
796        if let Some(result) = lane::try_lane_eigh(
797            session,
798            self.clone(),
799            Some(("eigh_with_options_read", options)),
800        ) {
801            return result;
802        }
803        with_linalg_backend(session, "eigh_with_options_read", |backend| {
804            two(
805                backend.eigh_with_options_read(self, options)?,
806                "eigh_with_options_read",
807            )
808        })
809    }
810    fn eig_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)> {
811        with_linalg_backend(session, "eig_read", |backend| {
812            two(backend.eig_read(self)?, "eig_read")
813        })
814    }
815    fn triangular_solve_read(
816        self,
817        b: TensorRead<'_>,
818        left_side: bool,
819        lower: bool,
820        transpose_a: bool,
821        unit_diagonal: bool,
822        session: &mut dyn Engine,
823    ) -> tenferro_tensor::Result<Tensor> {
824        with_linalg_backend(session, "triangular_solve_read", |backend| {
825            backend.triangular_solve_read(self, b, left_side, lower, transpose_a, unit_diagonal)
826        })
827    }
828    fn slogdet_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<(Tensor, Tensor)> {
829        validate_square_matrix("slogdet_read", self.shape())?;
830        if self.shape()[0] == 0 && self.shape()[1] == 0 {
831            return empty_square_slogdet_on(session, "slogdet_read", self);
832        }
833        with_linalg_backend(session, "slogdet_read", |backend| {
834            slogdet_from_lu(four(backend.lu_read(self)?, "slogdet_read")?, backend)
835        })
836    }
837    fn det_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
838        validate_square_matrix("det_read", self.shape())?;
839        with_linalg_backend(session, "det_read", |backend| {
840            det_impl(self.slogdet_read(backend)?, backend)
841        })
842    }
843    fn inv_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
844        with_linalg_backend(session, "inv_read", |backend| inv_read(self, backend))
845    }
846    fn eigvalsh_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
847        if let Some(result) = lane::try_lane_eigvalsh(session, self.clone()) {
848            return result;
849        }
850        with_linalg_backend(session, "eigvalsh_read", |backend| {
851            backend.eigh_values_read(self)
852        })
853    }
854    fn eigvals_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
855        if let Some(result) = lane::try_lane_eigvals(session, self.clone()) {
856            return result;
857        }
858        with_linalg_backend(session, "eigvals_read", |backend| {
859            backend.eig_values_read(self)
860        })
861    }
862    fn pinv_read(self, session: &mut dyn Engine) -> tenferro_tensor::Result<Tensor> {
863        with_linalg_backend(session, "pinv_read", |backend| {
864            pinv_read(self, None, backend)
865        })
866    }
867    fn pinv_with_rtol_read(
868        self,
869        rtol: f64,
870        session: &mut dyn Engine,
871    ) -> tenferro_tensor::Result<Tensor> {
872        with_linalg_backend(session, "pinv_with_rtol_read", |backend| {
873            pinv_read(self, Some(rtol), backend)
874        })
875    }
876    fn norm_read(
877        self,
878        ord: Option<f64>,
879        dim: Option<&[usize]>,
880        keepdim: bool,
881        session: &mut dyn Engine,
882    ) -> tenferro_tensor::Result<Tensor> {
883        with_linalg_backend(session, "norm_read", |backend| {
884            norm_from_read(self, ord, dim, keepdim, backend)
885        })
886    }
887}
888
889fn rank_revealing_qr_result(
890    outputs: Vec<Tensor>,
891) -> tenferro_tensor::Result<RankRevealingQrResult<Tensor>> {
892    let (q, r, column_permutation, rank) = four(outputs, "rank_revealing_qr")?;
893    Ok(RankRevealingQrResult {
894        q,
895        r,
896        column_permutation,
897        rank,
898    })
899}
900
901/// Run a concrete linalg body against the built-in linalg execution sessions
902/// (CPU/CUDA) carried by `session`, returning a typed capability error when
903/// the session does not expose a linalg execution capability.
904///
905/// This is the built-in dispatch shared by the concrete linalg surface;
906/// callers never downcast themselves (issue #1680 Phase 3). Third-party
907/// [`LinalgBackend`] implementations remain supported through the SPI trait,
908/// but the concrete op path is built-in-session only. The composite bodies
909/// run on the borrowed `&mut dyn LinalgBackend` exactly as before.
910pub(crate) fn with_linalg_backend<X>(
911    session: &mut dyn Engine,
912    op: &'static str,
913    f: impl FnOnce(&mut dyn LinalgBackend) -> tenferro_tensor::Result<X>,
914) -> tenferro_tensor::Result<X> {
915    // The capability branches are mutually exclusive, so `f` runs exactly
916    // once. Probe the marker first, then re-extract the same exec session and
917    // run the composite body on it (FnOnce cannot be captured by several
918    // branch closures).
919    if with_cpu_engine(session, |_| ()).is_some() {
920        return with_cpu_engine(session, |exec| f(exec as &mut dyn LinalgBackend))
921            .expect("marker probe matched a CPU execution session");
922    }
923    #[cfg(feature = "cuda")]
924    if with_cuda_engine(session, |_| ()).is_some() {
925        return with_cuda_engine(session, |exec| f(exec as &mut dyn LinalgBackend))
926            .expect("marker probe matched a CUDA execution session");
927    }
928    Err(tenferro_tensor::Error::unsupported(
929        op,
930        "selected backend session does not expose a linalg execution capability",
931    ))
932}
933
934/// The empty-determinant shortcut, taken after the backend's own input-residency boundary.
935///
936/// An empty square input needs no factorization, but it still crosses the same boundary the LU
937/// route would: a host session narrows it to host once and reads the output shape and dtype from
938/// that host view, and a device session applies the device-residency check. Neither copies or
939/// factorizes the input.
940fn empty_square_slogdet_on(
941    session: &mut dyn Engine,
942    op: &'static str,
943    input: TensorRead<'_>,
944) -> tenferro_tensor::Result<(Tensor, Tensor)> {
945    if with_cpu_engine(session, |_| ()).is_some() {
946        return with_cpu_engine(session, |exec| {
947            let input = input.try_host(op)?;
948            empty_square_slogdet(input.shape(), input.dtype(), exec as &mut dyn LinalgBackend)
949        })
950        .expect("marker probe matched a CPU execution session");
951    }
952    #[cfg(feature = "cuda")]
953    if with_cuda_engine(session, |_| ()).is_some() {
954        return with_cuda_engine(session, |backend| {
955            crate::gpu::ensure_device_resident_read(backend, op, &input)?;
956            empty_square_slogdet(
957                input.shape(),
958                input.dtype(),
959                backend as &mut dyn LinalgBackend,
960            )
961        })
962        .expect("marker probe matched a CUDA execution session");
963    }
964    Err(tenferro_tensor::Error::unsupported(
965        op,
966        "selected backend session does not expose a linalg execution capability",
967    ))
968}
969
970fn arity(name: &'static str, expected: usize, actual: usize) -> tenferro_tensor::Error {
971    tenferro_tensor::Error::Internal(format!(
972        "{name} backend contract expected {expected} outputs, got {actual}"
973    ))
974}
975fn two(mut out: Vec<Tensor>, name: &'static str) -> tenferro_tensor::Result<(Tensor, Tensor)> {
976    if out.len() != 2 {
977        return Err(arity(name, 2, out.len()));
978    }
979    let b = out.pop().unwrap();
980    let a = out.pop().unwrap();
981    Ok((a, b))
982}
983fn three(
984    mut out: Vec<Tensor>,
985    name: &'static str,
986) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor)> {
987    if out.len() != 3 {
988        return Err(arity(name, 3, out.len()));
989    }
990    let c = out.pop().unwrap();
991    let b = out.pop().unwrap();
992    let a = out.pop().unwrap();
993    Ok((a, b, c))
994}
995fn four(
996    mut out: Vec<Tensor>,
997    name: &'static str,
998) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor)> {
999    if out.len() != 4 {
1000        return Err(arity(name, 4, out.len()));
1001    }
1002    let d = out.pop().unwrap();
1003    let c = out.pop().unwrap();
1004    let b = out.pop().unwrap();
1005    let a = out.pop().unwrap();
1006    Ok((a, b, c, d))
1007}
1008fn five(
1009    mut out: Vec<Tensor>,
1010    name: &'static str,
1011) -> tenferro_tensor::Result<(Tensor, Tensor, Tensor, Tensor, Tensor)> {
1012    if out.len() != 5 {
1013        return Err(arity(name, 5, out.len()));
1014    }
1015    let e = out.pop().unwrap();
1016    let d = out.pop().unwrap();
1017    let c = out.pop().unwrap();
1018    let b = out.pop().unwrap();
1019    let a = out.pop().unwrap();
1020    Ok((a, b, c, d, e))
1021}
1022
1023// Composite implementations are kept below the surface adapters so every
1024// backend-visible operation remains explicit and testable.
1025fn slogdet_from_lu<B: LinalgBackend + ?Sized>(
1026    (_p, _l, u, parity): (Tensor, Tensor, Tensor, Tensor),
1027    backend: &mut B,
1028) -> tenferro_tensor::Result<(Tensor, Tensor)> {
1029    let diag = backend.extract_diagonal(&u, 0, 1)?;
1030    let sign = backend.sign_read(TensorRead::from_tensor(&diag))?;
1031    let sign_u = backend.reduce_prod_read(TensorRead::from_tensor(&sign), &[0])?;
1032    let sign = backend.mul_read(
1033        TensorRead::from_tensor(&parity),
1034        TensorRead::from_tensor(&sign_u),
1035    )?;
1036    let abs = backend.abs_read(TensorRead::from_tensor(&diag))?;
1037    let log = backend.log_read(TensorRead::from_tensor(&abs))?;
1038    let logabsdet = backend.reduce_sum_read(TensorRead::from_tensor(&log), &[0])?;
1039    Ok((sign, logabsdet))
1040}
1041fn det_impl<B: LinalgBackend + ?Sized>(
1042    (sign, logabsdet): (Tensor, Tensor),
1043    backend: &mut B,
1044) -> tenferro_tensor::Result<Tensor> {
1045    let magnitude = backend.exp_read(TensorRead::from_tensor(&logabsdet))?;
1046    backend.mul_read(
1047        TensorRead::from_tensor(&sign),
1048        TensorRead::from_tensor(&magnitude),
1049    )
1050}
1051fn inv_owned<B: LinalgBackend + ?Sized>(
1052    a: &Tensor,
1053    backend: &mut B,
1054) -> tenferro_tensor::Result<Tensor> {
1055    let eye = eye_like(a.dtype(), a.shape(), backend)?;
1056    backend.solve(a, &eye)
1057}
1058fn inv_read<B: LinalgBackend + ?Sized>(
1059    a: TensorRead<'_>,
1060    backend: &mut B,
1061) -> tenferro_tensor::Result<Tensor> {
1062    let eye = eye_like(a.dtype(), a.shape(), backend)?;
1063    backend.solve_read(a, TensorRead::from_tensor(&eye))
1064}
1065fn pinv_owned<B: LinalgBackend + ?Sized>(
1066    a: &Tensor,
1067    rtol: Option<f64>,
1068    backend: &mut B,
1069) -> tenferro_tensor::Result<Tensor> {
1070    ensure_float_or_complex("pinv", a.dtype())?;
1071    let outputs = three(backend.svd(a)?, "pinv")?;
1072    pinv_from_svd(
1073        outputs,
1074        rtol.unwrap_or_else(|| default_pinv_rtol(a.dtype(), a.shape())),
1075        backend,
1076    )
1077}
1078fn pinv_read<B: LinalgBackend + ?Sized>(
1079    a: TensorRead<'_>,
1080    rtol: Option<f64>,
1081    backend: &mut B,
1082) -> tenferro_tensor::Result<Tensor> {
1083    ensure_float_or_complex("pinv", a.dtype())?;
1084    let default_rtol = default_pinv_rtol(a.dtype(), a.shape());
1085    let outputs = three(backend.svd_read(a)?, "pinv_read")?;
1086    pinv_from_svd(outputs, rtol.unwrap_or(default_rtol), backend)
1087}
1088fn norm_from_read<B: LinalgBackend + ?Sized>(
1089    input: TensorRead<'_>,
1090    ord: Option<f64>,
1091    dim: Option<&[usize]>,
1092    keepdim: bool,
1093    backend: &mut B,
1094) -> tenferro_tensor::Result<Tensor> {
1095    ensure_float_or_complex("norm", input.dtype())?;
1096    let original_shape = input.shape().to_vec();
1097    let axes = dim.map_or_else(
1098        || (0..original_shape.len()).collect::<Vec<_>>(),
1099        <[usize]>::to_vec,
1100    );
1101    validate_axes("norm", original_shape.len(), &axes)?;
1102    if axes.is_empty() {
1103        return backend.to_contiguous_read(input);
1104    }
1105    let reduced = if ord.is_none() && axes.len() == original_shape.len() {
1106        // Full Frobenius reduction: one fused `|x|^2` pass with no `abs`
1107        // temporary and no leading-axis layout copy. Every backend that owns
1108        // the norm surface implements the fused primitive.
1109        let squared = backend.norm_squared_read(input.clone())?;
1110        backend.sqrt_read(TensorRead::from_tensor(&squared))?
1111    } else if can_square_without_abs(input.dtype(), axes.len(), ord) {
1112        frobenius_norm_read(input.clone(), &axes, backend)?
1113    } else if axes.len() == 2 {
1114        matrix_norm(input, &axes, ord, backend)?
1115    } else {
1116        let abs = backend.abs_read(input)?;
1117        norm_over_axes(&abs, &axes, ord, backend)?
1118    };
1119    if !keepdim {
1120        return Ok(reduced);
1121    }
1122    let mut shape = original_shape;
1123    for &axis in &axes {
1124        shape[axis] = 1;
1125    }
1126    reshape(&reduced, &shape, backend)
1127}
1128
1129/// Sign and log-absolute determinant of the empty `0 x 0` batch, whose
1130/// determinant is the empty product `1` and whose log-absolute determinant is
1131/// therefore `0`. The batch axes become the output shape.
1132fn empty_square_slogdet<B: LinalgBackend + ?Sized>(
1133    shape: &[usize],
1134    dtype: DType,
1135    backend: &mut B,
1136) -> tenferro_tensor::Result<(Tensor, Tensor)> {
1137    let batch_shape = &shape[2..];
1138    let sign_scalar = resident(scalar_real(dtype, 1.0)?, backend)?;
1139    let sign = broadcast(&sign_scalar, batch_shape, &[], backend)?;
1140    let log_scalar = resident(scalar_real(real_values_dtype(dtype), 0.0)?, backend)?;
1141    let logabsdet = broadcast(&log_scalar, batch_shape, &[], backend)?;
1142    Ok((sign, logabsdet))
1143}
1144
1145fn scalar_real(dtype: DType, value: f64) -> tenferro_tensor::Result<Tensor> {
1146    match dtype {
1147        DType::F32 => Tensor::from_vec_col_major(vec![], vec![value as f32]),
1148        DType::F64 => Tensor::from_vec_col_major(vec![], vec![value]),
1149        DType::C32 => Tensor::from_vec_col_major(vec![], vec![Complex32::new(value as f32, 0.0)]),
1150        DType::C64 => Tensor::from_vec_col_major(vec![], vec![Complex64::new(value, 0.0)]),
1151        _ => Err(crate::error::unsupported_dtype("linalg_scalar", dtype)),
1152    }
1153}
1154
1155/// Make a freshly built host tensor resident on `backend`.
1156///
1157/// The composite helpers below build their internal constants on the host - a
1158/// ones scalar, a pseudoinverse threshold, a norm order - and a device backend
1159/// rejects a host tensor at the first operation that reads it; the caller has no
1160/// way to supply a replacement (#2147). This is the concrete-session counterpart
1161/// of the eager composites' `constant_from_host`: on a CPU backend it is a host
1162/// copy, and every constant here is rank-0 or a broadcast of one, so the copy is
1163/// a scalar.
1164fn resident<B: LinalgBackend + ?Sized>(
1165    tensor: Tensor,
1166    backend: &mut B,
1167) -> tenferro_tensor::Result<Tensor> {
1168    backend.upload_host_tensor(TensorRead::from_tensor(&tensor))
1169}
1170
1171fn broadcast<B: LinalgBackend + ?Sized>(
1172    input: &Tensor,
1173    shape: &[usize],
1174    dims: &[usize],
1175    backend: &mut B,
1176) -> tenferro_tensor::Result<Tensor> {
1177    if input.shape() == shape {
1178        return input.duplicate();
1179    }
1180    backend.broadcast_in_dim_read(TensorRead::from_tensor(input), shape, dims)
1181}
1182
1183fn eye_like<B: LinalgBackend + ?Sized>(
1184    dtype: DType,
1185    shape: &[usize],
1186    backend: &mut B,
1187) -> tenferro_tensor::Result<Tensor> {
1188    if shape.len() < 2 {
1189        return Err(tenferro_tensor::Error::rank_mismatch("inv", 2, shape.len()));
1190    }
1191    let mut diagonal_shape = vec![shape[0]];
1192    diagonal_shape.extend_from_slice(&shape[2..]);
1193    let scalar = resident(scalar_real(dtype, 1.0)?, backend)?;
1194    let diagonal = broadcast(&scalar, &diagonal_shape, &[], backend)?;
1195    backend.embed_diagonal(&diagonal, 0, 1)
1196}
1197
1198fn ensure_float_or_complex(op: &'static str, dtype: DType) -> tenferro_tensor::Result<()> {
1199    match dtype {
1200        DType::F32 | DType::F64 | DType::C32 | DType::C64 => Ok(()),
1201        _ => Err(crate::error::unsupported_dtype(op, dtype)),
1202    }
1203}
1204
1205fn can_square_without_abs(dtype: DType, axes_len: usize, ord: Option<f64>) -> bool {
1206    matches!(dtype, DType::F32 | DType::F64)
1207        && (ord.is_none() || (ord == Some(2.0) && axes_len != 2))
1208}
1209
1210fn validate_axes(op: &'static str, rank: usize, axes: &[usize]) -> tenferro_tensor::Result<()> {
1211    tenferro_tensor::validate::validate_unique_axes(op, "dim", rank, axes)
1212}
1213
1214fn default_pinv_rtol(dtype: DType, shape: &[usize]) -> f64 {
1215    let max_dim = shape
1216        .first()
1217        .copied()
1218        .unwrap_or(0)
1219        .max(shape.get(1).copied().unwrap_or(0));
1220    let eps = match dtype {
1221        DType::F32 | DType::C32 => f32::EPSILON as f64,
1222        DType::F64 | DType::C64 => f64::EPSILON,
1223        _ => 0.0,
1224    };
1225    eps * max_dim as f64
1226}
1227
1228fn pinv_from_svd<B: LinalgBackend + ?Sized>(
1229    (u, s, vt): (Tensor, Tensor, Tensor),
1230    rtol: f64,
1231    backend: &mut B,
1232) -> tenferro_tensor::Result<Tensor> {
1233    let abs_s = backend.abs_read(TensorRead::from_tensor(&s))?;
1234    let s_max = reduce_max(&abs_s, &[0], backend)?;
1235    let threshold_scalar = resident(scalar_real(abs_s.dtype(), rtol.max(0.0))?, backend)?;
1236    let threshold = binary_mul(&s_max, &threshold_scalar, backend)?;
1237    let threshold = broadcast_batch_scalar_to_leading_axis(&threshold, s.shape(), backend)?;
1238    let mask = {
1239        let mask = backend.compare_read(
1240            TensorRead::from_tensor(&abs_s),
1241            TensorRead::from_tensor(&threshold),
1242            &CompareDir::Gt,
1243        )?;
1244        backend.convert(&mask, s.dtype())?
1245    };
1246    let ones = ones_like(&s, backend)?;
1247    let neg_mask = backend.neg_read(TensorRead::from_tensor(&mask))?;
1248    let offset = binary_add(&ones, &neg_mask, backend)?;
1249    let denom = binary_add(&s, &offset, backend)?;
1250    let s_inv = backend.div_read(
1251        TensorRead::from_tensor(&mask),
1252        TensorRead::from_tensor(&denom),
1253    )?;
1254    let v = conjugate_transpose(&vt, backend)?;
1255    let uh = conjugate_transpose(&u, backend)?;
1256    // A complex input has real singular values, so `s_inv` is real while `v` is
1257    // complex, and `scale_matrix_columns` multiplies them through `binary_mul`,
1258    // which requires one dtype. Casting the real factor into the complex type is
1259    // exact (#2049).
1260    let s_inv = if s_inv.dtype() == v.dtype() {
1261        s_inv
1262    } else {
1263        backend.convert(&s_inv, v.dtype())?
1264    };
1265    let vs = scale_matrix_columns(&v, &s_inv, backend)?;
1266    matmul_preserve_trailing_batch(&vs, &uh, backend)
1267}
1268
1269fn ones_like<B: LinalgBackend + ?Sized>(
1270    input: &Tensor,
1271    backend: &mut B,
1272) -> tenferro_tensor::Result<Tensor> {
1273    let one = resident(scalar_real(input.dtype(), 1.0)?, backend)?;
1274    broadcast(&one, input.shape(), &[], backend)
1275}
1276
1277fn binary_add<B: LinalgBackend + ?Sized>(
1278    lhs: &Tensor,
1279    rhs: &Tensor,
1280    backend: &mut B,
1281) -> tenferro_tensor::Result<Tensor> {
1282    backend.add_read(TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs))
1283}
1284
1285fn binary_mul<B: LinalgBackend + ?Sized>(
1286    lhs: &Tensor,
1287    rhs: &Tensor,
1288    backend: &mut B,
1289) -> tenferro_tensor::Result<Tensor> {
1290    backend.mul_read(TensorRead::from_tensor(lhs), TensorRead::from_tensor(rhs))
1291}
1292
1293fn reduce_max<B: LinalgBackend + ?Sized>(
1294    input: &Tensor,
1295    axes: &[usize],
1296    backend: &mut B,
1297) -> tenferro_tensor::Result<Tensor> {
1298    backend.reduce_max_read(TensorRead::from_tensor(input), axes)
1299}
1300
1301fn reduce_sum<B: LinalgBackend + ?Sized>(
1302    input: &Tensor,
1303    axes: &[usize],
1304    backend: &mut B,
1305) -> tenferro_tensor::Result<Tensor> {
1306    backend.reduce_sum_read(TensorRead::from_tensor(input), axes)
1307}
1308
1309fn reduce_min<B: LinalgBackend + ?Sized>(
1310    input: &Tensor,
1311    axes: &[usize],
1312    backend: &mut B,
1313) -> tenferro_tensor::Result<Tensor> {
1314    backend.reduce_min_read(TensorRead::from_tensor(input), axes)
1315}
1316
1317fn reshape<B: LinalgBackend + ?Sized>(
1318    input: &Tensor,
1319    shape: &[usize],
1320    backend: &mut B,
1321) -> tenferro_tensor::Result<Tensor> {
1322    backend.reshape_read(TensorRead::from_tensor(input), shape)
1323}
1324
1325fn transpose<B: LinalgBackend + ?Sized>(
1326    input: &Tensor,
1327    perm: &[usize],
1328    backend: &mut B,
1329) -> tenferro_tensor::Result<Tensor> {
1330    backend.transpose_read(TensorRead::from_tensor(input), perm)
1331}
1332
1333fn broadcast_batch_scalar_to_leading_axis<B: LinalgBackend + ?Sized>(
1334    input: &Tensor,
1335    shape: &[usize],
1336    backend: &mut B,
1337) -> tenferro_tensor::Result<Tensor> {
1338    let dims: Vec<usize> = (1..shape.len()).collect();
1339    broadcast(input, shape, &dims, backend)
1340}
1341
1342fn conjugate_transpose<B: LinalgBackend + ?Sized>(
1343    input: &Tensor,
1344    backend: &mut B,
1345) -> tenferro_tensor::Result<Tensor> {
1346    let conj = backend.conj_read(TensorRead::from_tensor(input))?;
1347    let mut perm: Vec<usize> = (0..input.shape().len()).collect();
1348    perm.swap(0, 1);
1349    transpose(&conj, &perm, backend)
1350}
1351
1352fn scale_matrix_columns<B: LinalgBackend + ?Sized>(
1353    matrix: &Tensor,
1354    scale: &Tensor,
1355    backend: &mut B,
1356) -> tenferro_tensor::Result<Tensor> {
1357    let mut scale_shape = vec![1, scale.shape()[0]];
1358    scale_shape.extend_from_slice(&matrix.shape()[2..]);
1359    let reshaped = reshape(scale, &scale_shape, backend)?;
1360    let dims: Vec<usize> = (0..matrix.shape().len()).collect();
1361    let expanded = broadcast(&reshaped, matrix.shape(), &dims, backend)?;
1362    binary_mul(matrix, &expanded, backend)
1363}
1364
1365fn matmul_preserve_trailing_batch<B: Engine + ?Sized>(
1366    lhs: &Tensor,
1367    rhs: &Tensor,
1368    session: &mut B,
1369) -> tenferro_tensor::Result<Tensor> {
1370    matmul_preserve_trailing_batch_read(lhs, TensorRead::from_tensor(rhs), session)
1371}
1372
1373fn matmul_preserve_trailing_batch_read<B: Engine + ?Sized>(
1374    lhs: &Tensor,
1375    rhs: TensorRead<'_>,
1376    session: &mut B,
1377) -> tenferro_tensor::Result<Tensor> {
1378    let batch: Vec<usize> = (2..lhs.shape().len()).collect();
1379    let config = DotGeneralConfig {
1380        lhs_contracting_dims: [1].as_slice().into(),
1381        rhs_contracting_dims: [0].as_slice().into(),
1382        lhs_batch_dims: batch.clone().iter().copied().collect(),
1383        rhs_batch_dims: batch.iter().copied().collect(),
1384    };
1385    session.dot_general_read(TensorRead::from_tensor(lhs), rhs, &config)
1386}
1387
1388/// Compact Householder QR of a borrowed operand.
1389///
1390/// The compact factorization takes an owned tensor, so a borrowed operand is
1391/// first made compact (same placement and dtype) by the backend.
1392pub(crate) fn householder_qr_read(
1393    a: TensorRead<'_>,
1394    session: &mut dyn Engine,
1395) -> tenferro_tensor::Result<crate::HouseholderQr<Tensor>> {
1396    with_linalg_backend(session, "householder_qr", |backend| {
1397        let compact = backend.to_contiguous_read(a)?;
1398        backend
1399            .householder_qr(&compact)
1400            .map(crate::HouseholderQr::from_backend)
1401    })
1402}
1403
1404/// Least squares `argmin_x |a x - b|` for a tall or square full-column-rank `a`.
1405///
1406/// The same composition the traced and eager `lstsq` build: thin QR `a = q r`,
1407/// then the triangular solve `r x = qᴴ b`. `a` is read like `qr_read` reads it;
1408/// `b` is read in place by the contraction.
1409pub(crate) fn lstsq_read(
1410    a: TensorRead<'_>,
1411    b: TensorRead<'_>,
1412    session: &mut dyn Engine,
1413) -> tenferro_tensor::Result<Tensor> {
1414    const OP: &str = "lstsq";
1415    ensure_float_or_complex(OP, a.dtype())?;
1416    for rank in [a.shape().len(), b.shape().len()] {
1417        if rank < 2 {
1418            return Err(tenferro_tensor::Error::rank_mismatch(OP, 2, rank));
1419        }
1420    }
1421    let (m, n) = (a.shape()[0], a.shape()[1]);
1422    if m < n {
1423        return Err(tenferro_tensor::Error::invalid_argument(
1424            OP,
1425            "shape",
1426            format!(
1427                "lstsq requires a tall or square matrix (rows {m} >= cols {n}); \
1428                 underdetermined (wide) systems are not supported"
1429            ),
1430        ));
1431    }
1432    with_linalg_backend(session, OP, |backend| {
1433        let (q, r) = two(backend.qr_read(a)?, OP)?;
1434        let qh = conjugate_transpose(&q, backend)?;
1435        let qh_b = matmul_preserve_trailing_batch_read(&qh, b, backend)?;
1436        backend.triangular_solve(&r, &qh_b, true, false, false, false)
1437    })
1438}
1439
1440fn frobenius_norm<B: LinalgBackend + ?Sized>(
1441    abs: &Tensor,
1442    axes: &[usize],
1443    backend: &mut B,
1444) -> tenferro_tensor::Result<Tensor> {
1445    let sum = backend.reduce_sum_squares_read(TensorRead::from_tensor(abs), axes)?;
1446    backend.sqrt_read(TensorRead::from_tensor(&sum))
1447}
1448
1449fn frobenius_norm_read<B: LinalgBackend + ?Sized>(
1450    input: TensorRead<'_>,
1451    axes: &[usize],
1452    backend: &mut B,
1453) -> tenferro_tensor::Result<Tensor> {
1454    let sum = backend.reduce_sum_squares_read(input, axes)?;
1455    backend.sqrt_read(TensorRead::from_tensor(&sum))
1456}
1457
1458fn p_norm<B: LinalgBackend + ?Sized>(
1459    abs: &Tensor,
1460    axes: &[usize],
1461    p: f64,
1462    backend: &mut B,
1463) -> tenferro_tensor::Result<Tensor> {
1464    if !p.is_finite() || p == 0.0 {
1465        return Err(tenferro_tensor::Error::invalid_argument(
1466            "norm",
1467            "p",
1468            format!("p-norm order must be finite and nonzero, got {p}"),
1469        ));
1470    }
1471    if p == 2.0 {
1472        return frobenius_norm(abs, axes, backend);
1473    }
1474    let power = resident(scalar_real(abs.dtype(), p)?, backend)?;
1475    let powered = backend.pow_read(
1476        TensorRead::from_tensor(abs),
1477        TensorRead::from_tensor(&power),
1478    )?;
1479    let sum = reduce_sum(&powered, axes, backend)?;
1480    let inverse = resident(scalar_real(abs.dtype(), 1.0 / p)?, backend)?;
1481    backend.pow_read(
1482        TensorRead::from_tensor(&sum),
1483        TensorRead::from_tensor(&inverse),
1484    )
1485}
1486
1487fn count_nonzero<B: LinalgBackend + ?Sized>(
1488    abs: &Tensor,
1489    axes: &[usize],
1490    backend: &mut B,
1491) -> tenferro_tensor::Result<Tensor> {
1492    let zero = resident(scalar_real(abs.dtype(), 0.0)?, backend)?;
1493    let zero = broadcast(&zero, abs.shape(), &[], backend)?;
1494    // `ord = 0` counts `x != 0` (#2359). Equality with zero, then negated,
1495    // keeps NaN entries in the count: IEEE 754 makes `NaN > 0` false while
1496    // `NaN != 0` is true, so the former `Gt` mask dropped them.
1497    let is_zero = backend.compare_read(
1498        TensorRead::from_tensor(abs),
1499        TensorRead::from_tensor(&zero),
1500        &CompareDir::Eq,
1501    )?;
1502    let is_zero = backend.convert(&is_zero, abs.dtype())?;
1503    let ones = ones_like(abs, backend)?;
1504    let mask = backend.sub_read(
1505        TensorRead::from_tensor(&ones),
1506        TensorRead::from_tensor(&is_zero),
1507    )?;
1508    reduce_sum(&mask, axes, backend)
1509}
1510
1511fn norm_over_axes<B: LinalgBackend + ?Sized>(
1512    abs: &Tensor,
1513    axes: &[usize],
1514    ord: Option<f64>,
1515    backend: &mut B,
1516) -> tenferro_tensor::Result<Tensor> {
1517    match ord {
1518        None => frobenius_norm(abs, axes, backend),
1519        Some(p) if p == f64::INFINITY => reduce_max(abs, axes, backend),
1520        Some(p) if p == f64::NEG_INFINITY => reduce_min(abs, axes, backend),
1521        Some(0.0) => count_nonzero(abs, axes, backend),
1522        Some(p) => p_norm(abs, axes, p, backend),
1523    }
1524}
1525
1526fn matrix_norm<B: LinalgBackend + ?Sized>(
1527    input: TensorRead<'_>,
1528    axes: &[usize],
1529    ord: Option<f64>,
1530    backend: &mut B,
1531) -> tenferro_tensor::Result<Tensor> {
1532    let matrix = move_axes_to_front(input, axes, backend)?;
1533    if matches!(ord, Some(2.0) | Some(-2.0)) {
1534        let (_, singular_values, _) = three(backend.svd(&matrix)?, "norm")?;
1535        return if ord == Some(2.0) {
1536            reduce_max(&singular_values, &[0], backend)
1537        } else {
1538            reduce_min(&singular_values, &[0], backend)
1539        };
1540    }
1541
1542    let abs = backend.abs_read(TensorRead::from_tensor(&matrix))?;
1543    match ord {
1544        None => frobenius_norm(&abs, &[0, 1], backend),
1545        Some(p) if p == f64::INFINITY => row_sum_norm(&abs, true, backend),
1546        Some(p) if p == f64::NEG_INFINITY => row_sum_norm(&abs, false, backend),
1547        Some(1.0) => col_sum_norm(&abs, true, backend),
1548        Some(-1.0) => col_sum_norm(&abs, false, backend),
1549        Some(0.0) => count_nonzero(&abs, &[0, 1], backend),
1550        Some(p) => p_norm(&abs, &[0, 1], p, backend),
1551    }
1552}
1553
1554fn row_sum_norm<B: LinalgBackend + ?Sized>(
1555    input: &Tensor,
1556    take_max: bool,
1557    backend: &mut B,
1558) -> tenferro_tensor::Result<Tensor> {
1559    let sums = reduce_sum(input, &[1], backend)?;
1560    if take_max {
1561        reduce_max(&sums, &[0], backend)
1562    } else {
1563        reduce_min(&sums, &[0], backend)
1564    }
1565}
1566
1567fn col_sum_norm<B: LinalgBackend + ?Sized>(
1568    input: &Tensor,
1569    take_max: bool,
1570    backend: &mut B,
1571) -> tenferro_tensor::Result<Tensor> {
1572    let sums = reduce_sum(input, &[0], backend)?;
1573    if take_max {
1574        reduce_max(&sums, &[0], backend)
1575    } else {
1576        reduce_min(&sums, &[0], backend)
1577    }
1578}
1579
1580fn move_axes_to_front<B: LinalgBackend + ?Sized>(
1581    input: TensorRead<'_>,
1582    axes: &[usize],
1583    backend: &mut B,
1584) -> tenferro_tensor::Result<Tensor> {
1585    if axes.iter().enumerate().all(|(index, &axis)| index == axis) {
1586        return backend.to_contiguous_read(input);
1587    }
1588    let mut selected = vec![false; input.shape().len()];
1589    for &axis in axes {
1590        selected[axis] = true;
1591    }
1592    let mut perm = axes.to_vec();
1593    perm.extend(
1594        selected
1595            .iter()
1596            .enumerate()
1597            .filter_map(|(axis, selected)| (!selected).then_some(axis)),
1598    );
1599    backend.transpose_read(input, &perm)
1600}