1use crate::backend::{unsupported_dtype, CompactQrResult, LinalgBackend};
2#[derive(Clone, Copy, Debug)]
3struct TriangularSolveOptions {
4 left_side: bool,
5 lower: bool,
6 transpose_a: bool,
7 unit_diagonal: bool,
8}
9use crate::extension::apply_qr_gauge;
10use crate::rank_revealing_qr::validate_rank_revealing_qr_options;
11use crate::validation::validate_square_matrix;
12
13use super::linalg;
14use tenferro_cpu::same_variant_pair;
15
16use super::linalg::HostView;
17use num_complex::{Complex32, Complex64};
18use tcontract_exec::Exec;
19use tenferro_cpu::linalg_interop::BufferPool;
20use tenferro_cpu::CpuEngine;
21use tenferro_tensor::{
22 validate::validate_nonsingular_u, AllocationDomainId, DType, Error, Host, HostAccessError,
23 MemoryKind, SharedTensorAllocationDomain, Tensor, TensorRead, TensorScalar, TensorStructural,
24 TensorView, TensorViewMut, TensorWrite, TypedTensor,
25};
26
27type HostOperand<'a> = TensorView<'a, Host>;
33
34fn host_operand<'a>(
36 op: &'static str,
37 input: &'a Tensor,
38) -> tenferro_tensor::Result<HostOperand<'a>> {
39 TensorRead::from_tensor(input).try_host(op)
40}
41
42fn host_engine_output<'a>(
47 op: &'static str,
48 produced: &'a Tensor,
49) -> tenferro_tensor::Result<HostOperand<'a>> {
50 TensorRead::from_tensor(produced).try_host(op)
51}
52
53trait HostArm: Sized {
55 fn arm<'v, 'a>(view: &'v HostOperand<'a>) -> Option<&'v HostView<'a, Self>>;
56}
57
58macro_rules! impl_host_arm {
59 ($($scalar:ty => $variant:ident),* $(,)?) => {$(
60 impl HostArm for $scalar {
61 fn arm<'v, 'a>(view: &'v HostOperand<'a>) -> Option<&'v HostView<'a, Self>> {
62 match view {
63 TensorView::$variant(typed) => Some(typed),
64 _ => None,
65 }
66 }
67 }
68 )*};
69}
70
71impl_host_arm!(
72 f32 => F32,
73 f64 => F64,
74 i32 => I32,
75 i64 => I64,
76 bool => Bool,
77 Complex32 => C32,
78 Complex64 => C64,
79);
80
81trait HostOperandExt<'a> {
83 fn as_typed<T: HostArm>(&self) -> Option<&HostView<'a, T>>;
84}
85
86impl<'a> HostOperandExt<'a> for HostOperand<'a> {
87 fn as_typed<T: HostArm>(&self) -> Option<&HostView<'a, T>> {
88 T::arm(self)
89 }
90}
91
92fn with_owned_layout<R>(
97 exec: &mut Exec<'_>,
98 buffers: &mut BufferPool,
99 op: &'static str,
100 input: HostOperand<'_>,
101 operation: impl FnOnce(
102 &HostOperand<'_>,
103 &mut BufferPool,
104 &mut Exec<'_>,
105 ) -> tenferro_tensor::Result<R>,
106) -> tenferro_tensor::Result<R> {
107 if input.is_owned_layout() {
108 return operation(&input, buffers, exec);
109 }
110 tenferro_cpu::with_materialized_tensor_read(
111 exec,
112 buffers,
113 op,
114 TensorRead::from_view(input.into_dynamic()),
115 |materialized, buffers, exec| {
116 operation(&host_engine_output(op, materialized)?, buffers, exec)
117 },
118 )
119}
120
121fn reshape_host<'a>(
126 input: &HostOperand<'a>,
127 shape: &[usize],
128) -> tenferro_tensor::Result<HostOperand<'a>> {
129 Ok(match input {
130 TensorView::F32(view) => TensorView::F32(view.reshape_view(shape)?),
131 TensorView::F64(view) => TensorView::F64(view.reshape_view(shape)?),
132 TensorView::I32(view) => TensorView::I32(view.reshape_view(shape)?),
133 TensorView::I64(view) => TensorView::I64(view.reshape_view(shape)?),
134 TensorView::Bool(view) => TensorView::Bool(view.reshape_view(shape)?),
135 TensorView::C32(view) => TensorView::C32(view.reshape_view(shape)?),
136 TensorView::C64(view) => TensorView::C64(view.reshape_view(shape)?),
137 })
138}
139
140macro_rules! preset_scalar {
142 (F32) => {
143 f32
144 };
145 (F64) => {
146 f64
147 };
148 (I32) => {
149 i32
150 };
151 (I64) => {
152 i64
153 };
154 (Bool) => {
155 bool
156 };
157 (C32) => {
158 num_complex::Complex32
159 };
160 (C64) => {
161 num_complex::Complex64
162 };
163}
164impl LinalgBackend for CpuEngine {
165 fn cholesky(&mut self, input: &Tensor) -> tenferro_tensor::Result<Tensor> {
166 let domain = self.shared_allocation_domain().cloned();
167 self.with_linalg_pool(move |exec, buffers| {
168 if tensor_uses_backend_storage(input)
169 && let Some(domain) = domain.as_deref()
170 {
171 return managed_cholesky(exec, buffers, TensorRead::from_tensor(input), domain);
172 }
173 let input = &host_operand("cholesky", input)?;
174 cholesky_entered(exec, buffers, input)
175 })
176 }
177
178 fn triangular_solve(
179 &mut self,
180 a: &Tensor,
181 b: &Tensor,
182 left_side: bool,
183 lower: bool,
184 transpose_a: bool,
185 unit_diagonal: bool,
186 ) -> tenferro_tensor::Result<Tensor> {
187 let a = &host_operand("triangular_solve", a)?;
188 let b = &host_operand("triangular_solve", b)?;
189 let options = TriangularSolveOptions {
190 left_side,
191 lower,
192 transpose_a,
193 unit_diagonal,
194 };
195 self.with_linalg_pool(|exec, buffers| {
196 triangular_solve_entered(exec, buffers, a, b, options)
197 })
198 }
199
200 fn triangular_solve_read(
201 &mut self,
202 a: TensorRead<'_>,
203 b: TensorRead<'_>,
204 left_side: bool,
205 lower: bool,
206 transpose_a: bool,
207 unit_diagonal: bool,
208 ) -> tenferro_tensor::Result<Tensor> {
209 let a = a.try_host("triangular_solve")?;
210 let b = b.try_host("triangular_solve")?;
211 ensure_supported_linalg_dtypes("triangular_solve", a.dtype(), b.dtype())?;
212 let options = TriangularSolveOptions {
213 left_side,
214 lower,
215 transpose_a,
216 unit_diagonal,
217 };
218 self.with_linalg_pool(move |exec, buffers| {
219 if faer_strided_host_ok(&a) && faer_rhs_host_ok(&b) {
224 return triangular_solve_faer_view_entered(exec, buffers, a, b, options);
225 }
226 with_owned_layout(exec, buffers, "triangular_solve", a, |a, buffers, exec| {
227 with_owned_layout(exec, buffers, "triangular_solve", b, |b, buffers, exec| {
228 triangular_solve_entered(exec, buffers, a, b, options)
229 })
230 })
231 })
232 }
233
234 fn lu(&mut self, input: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
235 let input = &host_operand("lu", input)?;
236 self.with_linalg_pool(|exec, buffers| lu_entered(exec, buffers, input))
237 }
238
239 fn lu_factor(&mut self, input: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
240 let input = &host_operand("lu_factor", input)?;
241 {
242 {
243 {
244 self.with_linalg_pool(|exec, buffers| match input.dtype() {
245 DType::F32 => {
246 let t = input
247 .as_typed::<f32>()
248 .ok_or_else(|| unsupported_dtype("lu_factor", input.dtype()))?;
249 linalg::faer_linalg::lu_factor(exec, buffers, t).map(
250 |(lu, pivots, parity)| {
251 vec![
252 Tensor::from_typed::<f32>(lu),
253 Tensor::from_typed::<i32>(pivots),
254 Tensor::from_typed::<f32>(parity),
255 ]
256 },
257 )
258 }
259 DType::F64 => {
260 let t = input
261 .as_typed::<f64>()
262 .ok_or_else(|| unsupported_dtype("lu_factor", input.dtype()))?;
263 linalg::faer_linalg::lu_factor(exec, buffers, t).map(
264 |(lu, pivots, parity)| {
265 vec![
266 Tensor::from_typed::<f64>(lu),
267 Tensor::from_typed::<i32>(pivots),
268 Tensor::from_typed::<f64>(parity),
269 ]
270 },
271 )
272 }
273 DType::C32 => {
274 let t = input
275 .as_typed::<Complex32>()
276 .ok_or_else(|| unsupported_dtype("lu_factor", input.dtype()))?;
277 linalg::faer_linalg::lu_factor(exec, buffers, t).map(
278 |(lu, pivots, parity)| {
279 vec![
280 Tensor::from_typed::<Complex32>(lu),
281 Tensor::from_typed::<i32>(pivots),
282 Tensor::from_typed::<Complex32>(parity),
283 ]
284 },
285 )
286 }
287 DType::C64 => {
288 let t = input
289 .as_typed::<Complex64>()
290 .ok_or_else(|| unsupported_dtype("lu_factor", input.dtype()))?;
291 linalg::faer_linalg::lu_factor(exec, buffers, t).map(
292 |(lu, pivots, parity)| {
293 vec![
294 Tensor::from_typed::<Complex64>(lu),
295 Tensor::from_typed::<i32>(pivots),
296 Tensor::from_typed::<Complex64>(parity),
297 ]
298 },
299 )
300 }
301 _ => Err(unsupported_dtype("lu_factor", input.dtype())),
302 })
303 }
304 }
305 }
306 }
307
308 fn full_piv_lu(&mut self, input: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
309 let input = &host_operand("full_piv_lu", input)?;
310 self.with_linalg_pool(|exec, buffers| full_piv_lu_entered(exec, buffers, input))
311 }
312
313 fn full_piv_lu_solve(
314 &mut self,
315 a: &Tensor,
316 b: &Tensor,
317 transpose_a: bool,
318 ) -> tenferro_tensor::Result<Tensor> {
319 let a = &host_operand("full_piv_lu_solve", a)?;
320 let b = &host_operand("full_piv_lu_solve", b)?;
321 ensure_supported_linalg_pair("full_piv_lu_solve", a, b)?;
322 validate_square_matrix("full_piv_lu_solve", a.shape())?;
323 if has_zero_dim(a.shape()) || has_zero_dim(b.shape()) {
324 return self.with_linalg_pool(|_, _| zeros_like_tensor(b.dtype(), b.shape()));
325 }
326
327 let (rhs, restore_shape) = match batched_vector_rhs_shape(a, b) {
330 Some(matrix_rhs_shape) => (
331 reshape_host(b, &matrix_rhs_shape)?,
332 Some(b.shape().to_vec()),
333 ),
334 None => (b.clone(), None),
335 };
336
337 let result = self.with_linalg_pool(|exec, buffers| {
339 same_variant_pair!(
340 a,
341 &rhs,
342 |a, b| { linalg::faer_linalg::full_piv_lu_solve(exec, buffers, a, b, transpose_a) },
343 unsupported_pair("full_piv_lu_solve", a, &rhs)
344 )
345 })?;
346
347 if let Some(shape) = restore_shape {
348 self.reshape_read(TensorRead::from_tensor(&result), &shape)
349 } else {
350 Ok(result)
351 }
352 }
353
354 fn svd(&mut self, input: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
355 let input = &host_operand("svd", input)?;
356 self.with_linalg_pool(|exec, buffers| svd_entered(exec, buffers, input))
357 }
358
359 fn svd_full(&mut self, input: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
360 let input = &host_operand("svd_full", input)?;
361 self.with_linalg_pool(|exec, buffers| svd_full_entered(exec, buffers, input))
362 }
363
364 fn svd_full_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Vec<Tensor>> {
365 let input = input.try_host("svd_full")?;
366 ensure_supported_linalg_dtype("svd_full", input.dtype())?;
367 self.with_linalg_pool(move |exec, buffers| {
368 if faer_strided_host_ok(&input) {
369 return svd_full_faer_view_entered(exec, buffers, input);
370 }
371 with_owned_layout(exec, buffers, "svd_full", input, |input, buffers, exec| {
372 svd_full_entered(exec, buffers, input)
373 })
374 })
375 }
376
377 fn svd_values(&mut self, input: &Tensor) -> tenferro_tensor::Result<Tensor> {
378 let input = &host_operand("svd_values", input)?;
379 self.with_linalg_pool(|exec, buffers| svd_values_entered(exec, buffers, input))
380 }
381
382 fn svd_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Vec<Tensor>> {
383 let input = input.try_host("svd")?;
384 ensure_supported_linalg_dtype("svd", input.dtype())?;
385 self.with_linalg_pool(move |exec, buffers| {
386 if faer_strided_host_ok(&input) {
387 return svd_faer_view_entered(exec, buffers, input);
388 }
389 with_owned_layout(exec, buffers, "svd", input, |input, buffers, exec| {
390 svd_entered(exec, buffers, input)
391 })
392 })
393 }
394
395 fn svd_values_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Tensor> {
396 let input = input.try_host("svd_values")?;
397 ensure_supported_linalg_dtype("svd_values", input.dtype())?;
398 self.with_linalg_pool(move |exec, buffers| {
399 if faer_strided_host_ok(&input) {
400 return svd_values_faer_view_entered(exec, buffers, input);
401 }
402 with_owned_layout(
403 exec,
404 buffers,
405 "svd_values",
406 input,
407 |input, buffers, exec| svd_values_entered(exec, buffers, input),
408 )
409 })
410 }
411
412 fn qr(&mut self, input: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
413 let input = &host_operand("qr", input)?;
414 self.with_linalg_pool(|exec, buffers| qr_entered(exec, buffers, input))
415 }
416
417 fn qr_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Vec<Tensor>> {
418 let input = input.try_host("qr")?;
419 ensure_supported_linalg_dtype("qr", input.dtype())?;
420 self.with_linalg_pool(move |exec, buffers| {
421 if faer_strided_host_ok(&input) {
422 return qr_faer_view_entered(exec, buffers, input);
423 }
424 with_owned_layout(exec, buffers, "qr", input, |input, buffers, exec| {
425 qr_entered(exec, buffers, input)
426 })
427 })
428 }
429
430 fn rank_revealing_qr(
431 &mut self,
432 input: &Tensor,
433 options: crate::RankRevealingQrOptions,
434 ) -> tenferro_tensor::Result<Vec<Tensor>> {
435 validate_rank_revealing_qr_options("rank_revealing_qr", options)?;
436 let input = &host_operand("rank_revealing_qr", input)?;
437 ensure_supported_linalg_dtype("rank_revealing_qr", input.dtype())?;
438 self.with_linalg_pool(|exec, buffers| {
439 rank_revealing_qr_entered(exec, buffers, input, options)
440 })
441 }
442
443 fn rank_revealing_qr_read(
444 &mut self,
445 input: TensorRead<'_>,
446 options: crate::RankRevealingQrOptions,
447 ) -> tenferro_tensor::Result<Vec<Tensor>> {
448 validate_rank_revealing_qr_options("rank_revealing_qr", options)?;
449 let input = input.try_host("rank_revealing_qr")?;
450 ensure_supported_linalg_dtype("rank_revealing_qr", input.dtype())?;
451 self.with_linalg_pool(move |exec, buffers| {
452 if faer_strided_host_ok(&input) {
453 return rank_revealing_qr_faer_view_entered(exec, buffers, input, options);
454 }
455 with_owned_layout(
456 exec,
457 buffers,
458 "rank_revealing_qr",
459 input,
460 |input, buffers, exec| rank_revealing_qr_entered(exec, buffers, input, options),
461 )
462 })
463 }
464
465 fn householder_qr(&mut self, input: &Tensor) -> tenferro_tensor::Result<CompactQrResult> {
466 let input = &host_operand("householder_qr", input)?;
467 self.with_linalg_pool(|exec, buffers| householder_qr_entered(exec, buffers, input))
468 }
469
470 fn householder_qr_from_factors(
471 &mut self,
472 q: &Tensor,
473 r: &Tensor,
474 ) -> tenferro_tensor::Result<CompactQrResult> {
475 let q = &host_operand("householder_qr_from_factors", q)?;
476 let r = &host_operand("householder_qr_from_factors", r)?;
477 self.with_linalg_pool(|exec, buffers| {
478 householder_qr_from_factors_entered(exec, buffers, q, r)
479 })
480 }
481
482 fn householder_qr_append(
483 &mut self,
484 packed: &Tensor,
485 coeff: &Tensor,
486 block: &Tensor,
487 ) -> tenferro_tensor::Result<CompactQrResult> {
488 let packed = &host_operand("householder_qr_append", packed)?;
489 let coeff = &host_operand("householder_qr_append", coeff)?;
490 let block = &host_operand("householder_qr_append", block)?;
491 self.with_linalg_pool(|exec, buffers| {
492 householder_qr_append_entered(exec, buffers, packed, coeff, block)
493 })
494 }
495
496 fn householder_qr_r(
497 &mut self,
498 packed: &Tensor,
499 coeff: &Tensor,
500 options: crate::QrOptions,
501 ) -> tenferro_tensor::Result<Tensor> {
502 let packed = &host_operand("householder_qr_r", packed)?;
503 let coeff = &host_operand("householder_qr_r", coeff)?;
504 self.with_linalg_pool(|exec, buffers| {
505 householder_qr_r_entered(exec, buffers, packed, coeff, options)
506 })
507 }
508
509 fn householder_qr_q_columns(
510 &mut self,
511 packed: &Tensor,
512 coeff: &Tensor,
513 columns: std::ops::Range<usize>,
514 options: crate::QrOptions,
515 ) -> tenferro_tensor::Result<Tensor> {
516 let packed = &host_operand("householder_qr_q_columns", packed)?;
517 let coeff = &host_operand("householder_qr_q_columns", coeff)?;
518 self.with_linalg_pool(|exec, buffers| {
519 householder_qr_q_columns_entered(exec, buffers, packed, coeff, columns, options)
520 })
521 }
522
523 fn eigh(&mut self, input: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
524 let input = &host_operand("eigh", input)?;
525 self.with_linalg_pool(|exec, buffers| eigh_entered(exec, buffers, input))
526 }
527
528 fn eigh_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Vec<Tensor>> {
529 let input = input.try_host("eigh")?;
530 ensure_supported_linalg_dtype("eigh", input.dtype())?;
531 self.with_linalg_pool(move |exec, buffers| {
532 if faer_strided_host_ok(&input) {
533 return eigh_faer_view_entered(exec, buffers, input);
534 }
535 with_owned_layout(exec, buffers, "eigh", input, |input, buffers, exec| {
536 eigh_entered(exec, buffers, input)
537 })
538 })
539 }
540
541 fn eigh_values_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Tensor> {
542 let input = input.try_host("eigh_values")?;
543 ensure_supported_linalg_dtype("eigh_values", input.dtype())?;
544 self.with_linalg_pool(move |exec, buffers| {
545 if faer_strided_host_ok(&input) {
546 return eigh_values_faer_view_entered(exec, buffers, input);
547 }
548 with_owned_layout(
549 exec,
550 buffers,
551 "eigh_values",
552 input,
553 |input, buffers, exec| eigh_values_entered(exec, buffers, input),
554 )
555 })
556 }
557
558 fn cholesky_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Tensor> {
559 let domain = self.shared_allocation_domain().cloned();
560 self.with_linalg_pool(move |exec, buffers| {
561 if let Some(domain) = domain.as_deref()
562 && input.backend_family().is_some()
563 {
564 return managed_cholesky(exec, buffers, input, domain);
565 }
566 let input = input.try_host("cholesky")?;
567 ensure_supported_linalg_dtype("cholesky", input.dtype())?;
568 if faer_strided_host_ok(&input) {
569 return cholesky_faer_view_entered(exec, buffers, input);
570 }
571 with_owned_layout(exec, buffers, "cholesky", input, |input, buffers, exec| {
572 cholesky_entered(exec, buffers, input)
573 })
574 })
575 }
576
577 fn lu_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Vec<Tensor>> {
578 let input = input.try_host("lu")?;
579 ensure_supported_linalg_dtype("lu", input.dtype())?;
580 self.with_linalg_pool(move |exec, buffers| {
581 if faer_strided_host_ok(&input) {
582 return lu_faer_view_entered(exec, buffers, input);
583 }
584 with_owned_layout(exec, buffers, "lu", input, |input, buffers, exec| {
585 lu_entered(exec, buffers, input)
586 })
587 })
588 }
589
590 fn full_piv_lu_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Vec<Tensor>> {
591 let input = input.try_host("full_piv_lu")?;
592 ensure_supported_linalg_dtype("full_piv_lu", input.dtype())?;
593 self.with_linalg_pool(move |exec, buffers| {
594 if faer_strided_host_ok(&input) {
595 return full_piv_lu_faer_view_entered(exec, buffers, input);
596 }
597 with_owned_layout(
598 exec,
599 buffers,
600 "full_piv_lu",
601 input,
602 |input, buffers, exec| full_piv_lu_entered(exec, buffers, input),
603 )
604 })
605 }
606
607 fn eig_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Vec<Tensor>> {
608 let input = input.try_host("eig")?;
609 ensure_supported_linalg_dtype("eig", input.dtype())?;
610 self.with_linalg_pool(move |exec, buffers| {
611 if faer_strided_host_ok(&input) {
612 return linalg::faer_linalg::eig(exec, buffers, &input);
613 }
614 with_owned_layout(exec, buffers, "eig", input, |input, buffers, exec| {
615 eig_entered(exec, buffers, input)
616 })
617 })
618 }
619
620 fn eig_values_read(&mut self, input: TensorRead<'_>) -> tenferro_tensor::Result<Tensor> {
621 let input = input.try_host("eig_values")?;
622 ensure_supported_linalg_dtype("eig_values", input.dtype())?;
623 self.with_linalg_pool(move |exec, buffers| {
624 if faer_strided_host_ok(&input) {
625 return linalg::faer_linalg::eig_values(exec, buffers, &input);
626 }
627 with_owned_layout(
628 exec,
629 buffers,
630 "eig_values",
631 input,
632 |input, buffers, exec| eig_values_entered(exec, buffers, input),
633 )
634 })
635 }
636
637 fn eigh_values(&mut self, input: &Tensor) -> tenferro_tensor::Result<Tensor> {
638 let input = &host_operand("eigh_values", input)?;
639 self.with_linalg_pool(|exec, buffers| eigh_values_entered(exec, buffers, input))
640 }
641
642 fn eig(&mut self, input: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
643 let input = &host_operand("eig", input)?;
644 ensure_supported_linalg_dtype("eig", input.dtype())?;
645 self.with_linalg_pool(|exec, buffers| eig_entered(exec, buffers, input))
646 }
647
648 fn eig_values(&mut self, input: &Tensor) -> tenferro_tensor::Result<Tensor> {
649 let input = &host_operand("eig_values", input)?;
650 ensure_supported_linalg_dtype("eig_values", input.dtype())?;
651 self.with_linalg_pool(|exec, buffers| eig_values_entered(exec, buffers, input))
652 }
653
654 fn lu_solve_prepared(
655 &mut self,
656 a: &Tensor,
657 packed_lu: &Tensor,
658 pivots: &Tensor,
659 b: &Tensor,
660 transpose_a: bool,
661 conjugate_a: bool,
662 ) -> tenferro_tensor::Result<Tensor> {
663 const OP: &str = "lu_solve_prepared";
664
665 let a = &host_operand(OP, a)?;
666 let packed_lu_tensor = packed_lu;
667 let packed_lu = &host_operand(OP, packed_lu)?;
668 let pivots = &host_operand(OP, pivots)?;
669 let b = &host_operand(OP, b)?;
670 ensure_supported_linalg_pair(OP, a, b)?;
671 ensure_supported_linalg_pair(OP, a, packed_lu)?;
672 if !matches!(pivots.dtype(), DType::I32) {
673 return Err(Error::dtype_mismatch(OP, DType::I32, pivots.dtype()));
674 }
675 if has_zero_dim(a.shape()) || has_zero_dim(b.shape()) {
676 return self.with_linalg_pool(|_, _| zeros_like_tensor(b.dtype(), b.shape()));
677 }
678
679 validate_lu_solve_prepared_shapes(
680 packed_lu.shape(),
681 pivots.shape(),
682 &packed_lu::rhs_matrix_shape(a.shape(), b.shape()),
683 )?;
684 validate_nonsingular_u(packed_lu_tensor)?;
685 self.with_linalg_pool(|exec, buffers| {
688 packed_lu::lu_solve_prepared_entered(
689 exec,
690 buffers,
691 a,
692 packed_lu,
693 pivots,
694 b,
695 transpose_a,
696 conjugate_a,
697 )
698 })
699 }
700
701 fn lu_factor_solve(&mut self, a: &Tensor, b: &Tensor) -> tenferro_tensor::Result<Vec<Tensor>> {
702 const OP: &str = "lu_factor_solve";
703
704 let a = &host_operand(OP, a)?;
705 let b = &host_operand(OP, b)?;
706 ensure_supported_linalg_pair(OP, a, b)?;
707 self.with_linalg_pool(|exec, buffers| {
708 packed_lu::lu_factor_solve_entered(exec, buffers, a, b)
709 })
710 }
711
712 fn solve(&mut self, a: &Tensor, b: &Tensor) -> tenferro_tensor::Result<Tensor> {
713 let a = &host_operand("solve", a)?;
714 let b = &host_operand("solve", b)?;
715 ensure_supported_linalg_pair("solve", a, b)?;
716 self.with_linalg_pool(|exec, buffers| solve_entered(exec, buffers, a, b))
717 }
718
719 fn solve_read(
720 &mut self,
721 a: TensorRead<'_>,
722 b: TensorRead<'_>,
723 ) -> tenferro_tensor::Result<Tensor> {
724 let a = a.try_host("solve")?;
725 let b = b.try_host("solve")?;
726 ensure_supported_linalg_dtypes("solve", a.dtype(), b.dtype())?;
727 let direct = !has_zero_dim(a.shape())
728 && !has_zero_dim(b.shape())
729 && solve_shape_direct_eligible(a.shape(), b.shape());
730 self.with_linalg_pool(move |exec, buffers| {
731 if direct {
732 solve_from_views_entered(exec, buffers, a, b)
733 } else {
734 with_owned_layout(exec, buffers, "solve", a, |a, buffers, exec| {
735 with_owned_layout(exec, buffers, "solve", b, |b, buffers, exec| {
736 solve_entered(exec, buffers, a, b)
737 })
738 })
739 }
740 })
741 }
742
743 fn solve_read_into(
744 &mut self,
745 a: TensorRead<'_>,
746 b: TensorRead<'_>,
747 out: TensorWrite<'_>,
748 ) -> tenferro_tensor::Result<()> {
749 crate::backend::validate_solve_read_into(&a, &b, &out)?;
750 let a = a.try_host("solve_read_into")?;
751 let b = b.try_host("solve_read_into")?;
752 let out = out.try_mut_host("solve_read_into")?;
753 ensure_supported_linalg_dtypes("solve_read_into", a.dtype(), b.dtype())?;
754
755 if has_zero_dim(a.shape())
756 || has_zero_dim(b.shape())
757 || !solve_read_into_direct_eligible(&a, &b, &out)
758 {
759 return crate::backend::solve_read_into_default(
761 self,
762 TensorRead::from_view(a.into_dynamic()),
763 TensorRead::from_view(b.into_dynamic()),
764 TensorWrite::from_view(out.into_dynamic()),
765 );
766 }
767
768 self.with_linalg_pool(move |exec, buffers| {
769 solve_read_into_entered(exec, buffers, a, b, out)
770 })
771 }
772}
773
774fn solve_read_into_direct_eligible(
775 a: &HostOperand<'_>,
776 b: &HostOperand<'_>,
777 out: &TensorViewMut<'_, Host>,
778) -> bool {
779 if !solve_shape_direct_eligible(a.shape(), b.shape()) || out.shape() != b.shape() {
780 return false;
781 }
782 let strides = out.strides();
783 match out.shape() {
784 [_] => strides == [1],
785 [rows, cols] => {
786 strides.first().copied() == Some(1)
787 && strides.get(1).copied().is_some_and(|stride| {
788 stride >= isize::try_from(*rows).unwrap_or(isize::MAX)
789 && (*cols <= 1 || stride > 0)
790 })
791 }
792 _ => false,
793 }
794}
795
796fn solve_shape_direct_eligible(a_shape: &[usize], b_shape: &[usize]) -> bool {
797 a_shape.len() == 2 && matches!(b_shape.len(), 1 | 2)
798}
799
800fn tensor_uses_backend_storage(input: &Tensor) -> bool {
801 input.is_backend_buffer()
802}
803
804fn managed_cholesky(
805 exec: &mut Exec<'_>,
806 buffers: &mut BufferPool,
807 input: TensorRead<'_>,
808 domain: &dyn SharedTensorAllocationDomain,
809) -> tenferro_tensor::Result<Tensor> {
810 let dtype = input.dtype();
811 ensure_supported_linalg_dtype("cholesky", dtype)?;
812 match linalg_view("cholesky", input)? {
813 TensorView::F32(view) => managed_cholesky_typed(exec, buffers, &view, domain),
814 TensorView::F64(view) => managed_cholesky_typed(exec, buffers, &view, domain),
815 TensorView::C32(view) => managed_cholesky_typed(exec, buffers, &view, domain),
816 TensorView::C64(view) => managed_cholesky_typed(exec, buffers, &view, domain),
817 _ => Err(unsupported_dtype("cholesky", dtype)),
818 }
819}
820
821trait ManagedCholeskyScalar: Copy + Send + Sync + TensorScalar + 'static {
822 const DTYPE: DType;
823
824 fn factor(
827 exec: &mut Exec<'_>,
828 buffers: &mut BufferPool,
829 data: &[Self],
830 n: usize,
831 ) -> tenferro_tensor::Result<TypedTensor<Self>>;
832
833 fn take_output(output: Tensor) -> tenferro_tensor::Result<TypedTensor<Self>>;
834 fn wrap(output: TypedTensor<Self>) -> Tensor;
835}
836
837macro_rules! impl_managed_cholesky_scalar {
838 ($scalar:ty, $dtype:ident, $variant:ident) => {
839 impl ManagedCholeskyScalar for $scalar {
840 const DTYPE: DType = DType::$dtype;
841
842 fn factor(
843 exec: &mut Exec<'_>,
844 buffers: &mut BufferPool,
845 data: &[Self],
846 n: usize,
847 ) -> tenferro_tensor::Result<TypedTensor<Self>> {
848 {
849 {
850 {
851 linalg::faer_linalg::cholesky_compact_data(exec, buffers, data, n)
852 }
853 }
854 }
855 }
856
857 fn take_output(output: Tensor) -> tenferro_tensor::Result<TypedTensor<Self>> {
858 let Ok(output) = output.into_typed::<preset_scalar!($variant)>() else {
859 return Err(tenferro_tensor::Error::runtime_state(
860 "cholesky",
861 concat!(
862 "shared allocation owner returned a non-",
863 stringify!($variant),
864 " output"
865 ),
866 ));
867 };
868 Ok(output)
869 }
870
871 fn wrap(output: TypedTensor<Self>) -> Tensor {
872 Tensor::from_typed::<preset_scalar!($variant)>(output)
873 }
874 }
875 };
876}
877
878impl_managed_cholesky_scalar!(f32, F32, F32);
879impl_managed_cholesky_scalar!(f64, F64, F64);
880impl_managed_cholesky_scalar!(Complex32, C32, C32);
881impl_managed_cholesky_scalar!(Complex64, C64, C64);
882
883fn managed_cholesky_typed<T>(
884 exec: &mut Exec<'_>,
885 buffers: &mut BufferPool,
886 input: &tenferro_tensor::TypedTensorView<'_, T>,
887 domain: &dyn SharedTensorAllocationDomain,
888) -> tenferro_tensor::Result<Tensor>
889where
890 T: ManagedCholeskyScalar,
891{
892 let n = validate_managed_cholesky_input(input, domain.id())?;
893 let values = if n == 0 {
896 None
897 } else {
898 Some(input.with_host_read(|read| T::factor(exec, buffers, read, n))??)
899 };
900 let mut typed = T::take_output(domain.allocate(T::DTYPE, &[n, n])?)?;
901 let staged: &[T] = match &values {
902 Some(values) => values.host_data()?,
903 None => &[],
904 };
905 write_managed_cholesky_output(&mut typed, domain.id(), staged)?;
906 Ok(T::wrap(typed))
907}
908
909fn validate_managed_cholesky_input<T: Copy + Send + Sync + TensorScalar + 'static>(
910 input: &tenferro_tensor::TypedTensorView<'_, T>,
911 expected_domain: AllocationDomainId,
912) -> tenferro_tensor::Result<usize> {
913 if input.rank() != 2 {
914 return Err(tenferro_tensor::Error::rank_mismatch(
915 "cholesky",
916 2,
917 input.rank(),
918 ));
919 }
920 let rows = input.shape()[0];
921 let cols = input.shape()[1];
922 if rows != cols {
923 return Err(tenferro_tensor::Error::shape_mismatch(
924 "cholesky",
925 vec![rows],
926 vec![cols],
927 ));
928 }
929 if !input.is_col_major_contiguous()? {
930 return Err(tenferro_tensor::Error::invalid_argument(
931 "cholesky",
932 "input layout",
933 "managed rank-2 Cholesky requires a compact column-major descriptor",
934 ));
935 }
936 rows.checked_mul(cols).ok_or_else(|| {
937 tenferro_tensor::Error::invalid_argument(
938 "cholesky",
939 "input shape",
940 "matrix element count overflows usize",
941 )
942 })?;
943 if input.placement().memory_kind != MemoryKind::Managed {
944 return Err(tenferro_tensor::Error::host_access(
945 "cholesky",
946 HostAccessError::Unsupported {
947 backend: if matches!(
948 input.placement().memory_kind,
949 MemoryKind::PinnedHost | MemoryKind::UnpinnedHost
950 ) {
951 "host"
952 } else {
953 "backend"
954 },
955 },
956 ));
957 }
958 match input.allocation_domain() {
959 Some(actual) if actual == expected_domain => {}
960 Some(actual) => {
961 return Err(tenferro_tensor::Error::host_access(
962 "cholesky",
963 HostAccessError::ForeignDomain {
964 expected: expected_domain,
965 actual,
966 },
967 ));
968 }
969 None => {
970 return Err(tenferro_tensor::Error::host_access(
971 "cholesky",
972 HostAccessError::Unsupported { backend: "backend" },
973 ));
974 }
975 }
976 Ok(rows)
977}
978
979fn write_managed_cholesky_output<T: TensorScalar + Copy + Send + Sync + 'static>(
980 output: &mut TypedTensor<T>,
981 expected_domain: AllocationDomainId,
982 values: &[T],
983) -> tenferro_tensor::Result<()> {
984 if output.allocation_domain() != Some(expected_domain)
985 || output.placement().memory_kind != MemoryKind::Managed
986 {
987 return Err(tenferro_tensor::Error::runtime_state(
988 "cholesky",
989 "shared allocation owner returned an output outside its managed domain",
990 ));
991 }
992 if let Some(buffer) = output.backend_buffer_mut() {
993 let mut write = buffer
994 .map_write()
995 .map_err(|source| tenferro_tensor::Error::host_access("cholesky", source))?;
996 return write
997 .copy_from_slice(values)
998 .map_err(|source| tenferro_tensor::Error::host_access("cholesky", source));
999 }
1000 output.with_host_write(|write| {
1001 if write.len() != values.len() {
1002 return Err(tenferro_tensor::Error::runtime_state(
1003 "cholesky",
1004 "shared allocation owner returned an output with the wrong length",
1005 ));
1006 }
1007 write.copy_from_slice(values);
1008 Ok(())
1009 })?
1010}
1011
1012fn q_columns_operands<'v, 'a, T: HostArm>(
1014 packed: &'v HostOperand<'a>,
1015 coeff: &'v HostOperand<'a>,
1016) -> Result<TypedOperandPair<'v, 'a, T>, tenferro_tensor::Error> {
1017 let p = packed.as_typed::<T>().ok_or_else(|| {
1018 Error::dtype_mismatch("householder_qr_q_columns", packed.dtype(), coeff.dtype())
1019 })?;
1020 let c = coeff.as_typed::<T>().ok_or_else(|| {
1021 Error::dtype_mismatch("householder_qr_q_columns", packed.dtype(), coeff.dtype())
1022 })?;
1023 Ok((p, c))
1024}
1025
1026type TypedOperandPair<'v, 'a, T> = (&'v HostView<'a, T>, &'v HostView<'a, T>);
1028
1029fn qr_import_operands<'v, 'a, T: HostArm>(
1034 q: &'v HostOperand<'a>,
1035 r: &'v HostOperand<'a>,
1036) -> Result<TypedOperandPair<'v, 'a, T>, tenferro_tensor::Error> {
1037 let q_t = q
1038 .as_typed::<T>()
1039 .ok_or_else(|| unsupported_dtype("householder_qr_from_factors", q.dtype()))?;
1040 let r_t = r
1041 .as_typed::<T>()
1042 .ok_or_else(|| unsupported_dtype("householder_qr_from_factors", r.dtype()))?;
1043 Ok((q_t, r_t))
1044}
1045
1046type TypedOperandTriple<'v, 'a, T> = (
1048 &'v HostView<'a, T>,
1049 &'v HostView<'a, T>,
1050 &'v HostView<'a, T>,
1051);
1052
1053fn append_operands<'v, 'a, T: HostArm>(
1055 packed: &'v HostOperand<'a>,
1056 coeff: &'v HostOperand<'a>,
1057 block: &'v HostOperand<'a>,
1058) -> Result<TypedOperandTriple<'v, 'a, T>, tenferro_tensor::Error> {
1059 let mismatch = || unsupported_dtype("householder_qr_append", packed.dtype());
1060 let packed_t = packed.as_typed::<T>().ok_or_else(mismatch)?;
1061 let coeff_t = coeff.as_typed::<T>().ok_or_else(mismatch)?;
1062 let block_t = block.as_typed::<T>().ok_or_else(mismatch)?;
1063 Ok((packed_t, coeff_t, block_t))
1064}
1065
1066fn triangular_solve_entered(
1067 exec: &mut Exec<'_>,
1068 buffers: &mut BufferPool,
1069 a: &HostOperand<'_>,
1070 b: &HostOperand<'_>,
1071 options: TriangularSolveOptions,
1072) -> tenferro_tensor::Result<Tensor> {
1073 {
1074 {
1075 {
1076 same_variant_pair!(
1077 a,
1078 b,
1079 |a, b| {
1080 linalg::faer_linalg::triangular_solve(
1081 exec,
1082 buffers,
1083 a,
1084 b,
1085 options.left_side,
1086 options.lower,
1087 options.transpose_a,
1088 options.unit_diagonal,
1089 )
1090 },
1091 unsupported_pair("triangular_solve", a, b)
1092 )
1093 }
1094 }
1095 }
1096}
1097
1098fn solve_entered(
1099 exec: &mut Exec<'_>,
1100 buffers: &mut BufferPool,
1101 a: &HostOperand<'_>,
1102 b: &HostOperand<'_>,
1103) -> tenferro_tensor::Result<Tensor> {
1104 validate_square_matrix("solve", a.shape())?;
1105 if has_zero_dim(a.shape()) || has_zero_dim(b.shape()) {
1106 return zeros_like_tensor(b.dtype(), b.shape());
1107 }
1108
1109 let (rhs, restore_shape) = match batched_vector_rhs_shape(a, b) {
1112 Some(matrix_rhs_shape) => (
1113 reshape_host(b, &matrix_rhs_shape)?,
1114 Some(b.shape().to_vec()),
1115 ),
1116 None => (b.clone(), None),
1117 };
1118
1119 let result = {
1120 {
1121 {
1122 same_variant_pair!(
1123 a,
1124 &rhs,
1125 |a, b| { linalg::faer_linalg::solve(exec, buffers, a, b, false) },
1126 unsupported_pair("solve", a, &rhs)
1127 )
1128 }
1129 }
1130 }?;
1131
1132 if let Some(shape) = restore_shape {
1133 tenferro_cpu::reshape_tensor(exec, buffers, &result, &shape)
1134 } else {
1135 Ok(result)
1136 }
1137}
1138
1139fn solve_from_views_entered(
1140 exec: &mut Exec<'_>,
1141 buffers: &mut BufferPool,
1142 a: HostOperand<'_>,
1143 b: HostOperand<'_>,
1144) -> tenferro_tensor::Result<Tensor> {
1145 let a = a.into_dynamic();
1146 let b = b.into_dynamic();
1147 {
1148 {
1149 {
1150 match (a, b) {
1151 (TensorView::F32(a), TensorView::F32(b)) => {
1152 linalg::faer_linalg::solve_from_views(exec, buffers, a, b, false)
1153 .map(Tensor::from_typed::<f32>)
1154 }
1155 (TensorView::F64(a), TensorView::F64(b)) => {
1156 linalg::faer_linalg::solve_from_views(exec, buffers, a, b, false)
1157 .map(Tensor::from_typed::<f64>)
1158 }
1159 (TensorView::C32(a), TensorView::C32(b)) => {
1160 linalg::faer_linalg::solve_from_views(exec, buffers, a, b, false)
1161 .map(Tensor::from_typed::<Complex32>)
1162 }
1163 (TensorView::C64(a), TensorView::C64(b)) => {
1164 linalg::faer_linalg::solve_from_views(exec, buffers, a, b, false)
1165 .map(Tensor::from_typed::<Complex64>)
1166 }
1167 _ => Err(Error::invalid_argument(
1168 "solve",
1169 "inputs",
1170 "solve inputs must have the same dtype",
1171 )),
1172 }
1173 }
1174 }
1175 }
1176}
1177
1178fn solve_read_into_entered(
1179 exec: &mut Exec<'_>,
1180 buffers: &mut BufferPool,
1181 a: HostOperand<'_>,
1182 b: HostOperand<'_>,
1183 out: TensorViewMut<'_, Host>,
1184) -> tenferro_tensor::Result<()> {
1185 let a = a.into_dynamic();
1186 let b = b.into_dynamic();
1187 let out = out.into_dynamic();
1188 {
1189 {
1190 {
1191 match (a, b, out) {
1192 (TensorView::F32(a), TensorView::F32(b), TensorViewMut::F32(mut out)) => {
1193 linalg::faer_linalg::solve_into(exec, buffers, a, b, &mut out, false)
1194 }
1195 (TensorView::F64(a), TensorView::F64(b), TensorViewMut::F64(mut out)) => {
1196 linalg::faer_linalg::solve_into(exec, buffers, a, b, &mut out, false)
1197 }
1198 (TensorView::C32(a), TensorView::C32(b), TensorViewMut::C32(mut out)) => {
1199 linalg::faer_linalg::solve_into(exec, buffers, a, b, &mut out, false)
1200 }
1201 (TensorView::C64(a), TensorView::C64(b), TensorViewMut::C64(mut out)) => {
1202 linalg::faer_linalg::solve_into(exec, buffers, a, b, &mut out, false)
1203 }
1204 _ => Err(Error::invalid_argument(
1205 "solve_read_into",
1206 "out",
1207 "destination dtype does not match the solve inputs",
1208 )),
1209 }
1210 }
1211 }
1212 }
1213}
1214
1215fn cholesky_entered(
1216 exec: &mut Exec<'_>,
1217 buffers: &mut BufferPool,
1218 input: &HostOperand<'_>,
1219) -> tenferro_tensor::Result<Tensor> {
1220 {
1221 {
1222 {
1223 match input.dtype() {
1224 DType::F32 => linalg::faer_linalg::cholesky(
1225 exec,
1226 buffers,
1227 input
1228 .as_typed::<f32>()
1229 .ok_or_else(|| unsupported_dtype("cholesky", input.dtype()))?,
1230 )
1231 .map(Tensor::from_typed::<f32>),
1232 DType::F64 => linalg::faer_linalg::cholesky(
1233 exec,
1234 buffers,
1235 input
1236 .as_typed::<f64>()
1237 .ok_or_else(|| unsupported_dtype("cholesky", input.dtype()))?,
1238 )
1239 .map(Tensor::from_typed::<f64>),
1240 DType::C32 => linalg::faer_linalg::cholesky(
1241 exec,
1242 buffers,
1243 input
1244 .as_typed::<Complex32>()
1245 .ok_or_else(|| unsupported_dtype("cholesky", input.dtype()))?,
1246 )
1247 .map(Tensor::from_typed::<Complex32>),
1248 DType::C64 => linalg::faer_linalg::cholesky(
1249 exec,
1250 buffers,
1251 input
1252 .as_typed::<Complex64>()
1253 .ok_or_else(|| unsupported_dtype("cholesky", input.dtype()))?,
1254 )
1255 .map(Tensor::from_typed::<Complex64>),
1256 _ => Err(unsupported_dtype("cholesky", input.dtype())),
1257 }
1258 }
1259 }
1260 }
1261}
1262
1263fn lu_entered(
1264 exec: &mut Exec<'_>,
1265 buffers: &mut BufferPool,
1266 input: &HostOperand<'_>,
1267) -> tenferro_tensor::Result<Vec<Tensor>> {
1268 {
1269 {
1270 {
1271 match input.dtype() {
1272 DType::F32 => linalg::faer_linalg::lu(
1273 exec,
1274 buffers,
1275 input
1276 .as_typed::<f32>()
1277 .ok_or_else(|| unsupported_dtype("lu", input.dtype()))?,
1278 )
1279 .map(|outputs| outputs.into_iter().map(Tensor::from_typed::<f32>).collect()),
1280 DType::F64 => linalg::faer_linalg::lu(
1281 exec,
1282 buffers,
1283 input
1284 .as_typed::<f64>()
1285 .ok_or_else(|| unsupported_dtype("lu", input.dtype()))?,
1286 )
1287 .map(|outputs| outputs.into_iter().map(Tensor::from_typed::<f64>).collect()),
1288 DType::C32 => linalg::faer_linalg::lu(
1289 exec,
1290 buffers,
1291 input
1292 .as_typed::<Complex32>()
1293 .ok_or_else(|| unsupported_dtype("lu", input.dtype()))?,
1294 )
1295 .map(|outputs| {
1296 outputs
1297 .into_iter()
1298 .map(Tensor::from_typed::<Complex32>)
1299 .collect()
1300 }),
1301 DType::C64 => linalg::faer_linalg::lu(
1302 exec,
1303 buffers,
1304 input
1305 .as_typed::<Complex64>()
1306 .ok_or_else(|| unsupported_dtype("lu", input.dtype()))?,
1307 )
1308 .map(|outputs| {
1309 outputs
1310 .into_iter()
1311 .map(Tensor::from_typed::<Complex64>)
1312 .collect()
1313 }),
1314 _ => Err(unsupported_dtype("lu", input.dtype())),
1315 }
1316 }
1317 }
1318 }
1319}
1320
1321fn full_piv_lu_entered(
1322 exec: &mut Exec<'_>,
1323 buffers: &mut BufferPool,
1324 input: &HostOperand<'_>,
1325) -> tenferro_tensor::Result<Vec<Tensor>> {
1326 match input.dtype() {
1330 DType::F32 => linalg::faer_linalg::full_piv_lu(
1331 exec,
1332 buffers,
1333 input
1334 .as_typed::<f32>()
1335 .ok_or_else(|| unsupported_dtype("full_piv_lu", input.dtype()))?,
1336 )
1337 .map(|outputs| outputs.into_iter().map(Tensor::from_typed::<f32>).collect()),
1338 DType::F64 => linalg::faer_linalg::full_piv_lu(
1339 exec,
1340 buffers,
1341 input
1342 .as_typed::<f64>()
1343 .ok_or_else(|| unsupported_dtype("full_piv_lu", input.dtype()))?,
1344 )
1345 .map(|outputs| outputs.into_iter().map(Tensor::from_typed::<f64>).collect()),
1346 DType::C32 => linalg::faer_linalg::full_piv_lu(
1347 exec,
1348 buffers,
1349 input
1350 .as_typed::<Complex32>()
1351 .ok_or_else(|| unsupported_dtype("full_piv_lu", input.dtype()))?,
1352 )
1353 .and_then(full_piv_lu_c32_outputs_to_public_tensors),
1354 DType::C64 => linalg::faer_linalg::full_piv_lu(
1355 exec,
1356 buffers,
1357 input
1358 .as_typed::<Complex64>()
1359 .ok_or_else(|| unsupported_dtype("full_piv_lu", input.dtype()))?,
1360 )
1361 .and_then(full_piv_lu_c64_outputs_to_public_tensors),
1362 _ => Err(unsupported_dtype("full_piv_lu", input.dtype())),
1363 }
1364}
1365
1366fn svd_entered(
1367 exec: &mut Exec<'_>,
1368 buffers: &mut BufferPool,
1369 input: &HostOperand<'_>,
1370) -> tenferro_tensor::Result<Vec<Tensor>> {
1371 {
1372 {
1373 {
1374 match input.dtype() {
1375 DType::F32 => linalg::faer_linalg::svd(
1376 exec,
1377 buffers,
1378 input
1379 .as_typed::<f32>()
1380 .ok_or_else(|| unsupported_dtype("svd", input.dtype()))?,
1381 )
1382 .map(svd_erased),
1383 DType::F64 => linalg::faer_linalg::svd(
1384 exec,
1385 buffers,
1386 input
1387 .as_typed::<f64>()
1388 .ok_or_else(|| unsupported_dtype("svd", input.dtype()))?,
1389 )
1390 .map(svd_erased),
1391 DType::C32 => linalg::faer_linalg::svd(
1392 exec,
1393 buffers,
1394 input
1395 .as_typed::<Complex32>()
1396 .ok_or_else(|| unsupported_dtype("svd", input.dtype()))?,
1397 )
1398 .map(svd_erased),
1399 DType::C64 => linalg::faer_linalg::svd(
1400 exec,
1401 buffers,
1402 input
1403 .as_typed::<Complex64>()
1404 .ok_or_else(|| unsupported_dtype("svd", input.dtype()))?,
1405 )
1406 .map(svd_erased),
1407 _ => Err(unsupported_dtype("svd", input.dtype())),
1408 }
1409 }
1410 }
1411 }
1412}
1413
1414fn svd_full_entered(
1415 exec: &mut Exec<'_>,
1416 buffers: &mut BufferPool,
1417 input: &HostOperand<'_>,
1418) -> tenferro_tensor::Result<Vec<Tensor>> {
1419 {
1420 {
1421 {
1422 match input.dtype() {
1423 DType::F32 => linalg::faer_linalg::svd_full(
1424 exec,
1425 buffers,
1426 input
1427 .as_typed::<f32>()
1428 .ok_or_else(|| unsupported_dtype("svd_full", input.dtype()))?,
1429 )
1430 .map(svd_erased),
1431 DType::F64 => linalg::faer_linalg::svd_full(
1432 exec,
1433 buffers,
1434 input
1435 .as_typed::<f64>()
1436 .ok_or_else(|| unsupported_dtype("svd_full", input.dtype()))?,
1437 )
1438 .map(svd_erased),
1439 DType::C32 => linalg::faer_linalg::svd_full(
1440 exec,
1441 buffers,
1442 input
1443 .as_typed::<Complex32>()
1444 .ok_or_else(|| unsupported_dtype("svd_full", input.dtype()))?,
1445 )
1446 .map(svd_erased),
1447 DType::C64 => linalg::faer_linalg::svd_full(
1448 exec,
1449 buffers,
1450 input
1451 .as_typed::<Complex64>()
1452 .ok_or_else(|| unsupported_dtype("svd_full", input.dtype()))?,
1453 )
1454 .map(svd_erased),
1455 _ => Err(unsupported_dtype("svd_full", input.dtype())),
1456 }
1457 }
1458 }
1459 }
1460}
1461
1462fn svd_values_entered(
1463 exec: &mut Exec<'_>,
1464 buffers: &mut BufferPool,
1465 input: &HostOperand<'_>,
1466) -> tenferro_tensor::Result<Tensor> {
1467 {
1468 {
1469 {
1470 match input.dtype() {
1471 DType::F32 => {
1472 let t = input
1473 .as_typed::<f32>()
1474 .ok_or_else(|| unsupported_dtype("svd_values", input.dtype()))?;
1475 linalg::faer_linalg::svd_values(exec, buffers, t)
1476 .map(Tensor::from_typed::<f32>)
1477 }
1478 DType::F64 => {
1479 let t = input
1480 .as_typed::<f64>()
1481 .ok_or_else(|| unsupported_dtype("svd_values", input.dtype()))?;
1482 linalg::faer_linalg::svd_values(exec, buffers, t)
1483 .map(Tensor::from_typed::<f64>)
1484 }
1485 DType::C32 => {
1486 let t = input
1487 .as_typed::<Complex32>()
1488 .ok_or_else(|| unsupported_dtype("svd_values", input.dtype()))?;
1489 linalg::faer_linalg::svd_values(exec, buffers, t)
1490 .map(Tensor::from_typed::<f32>)
1491 }
1492 DType::C64 => {
1493 let t = input
1494 .as_typed::<Complex64>()
1495 .ok_or_else(|| unsupported_dtype("svd_values", input.dtype()))?;
1496 linalg::faer_linalg::svd_values(exec, buffers, t)
1497 .map(Tensor::from_typed::<f64>)
1498 }
1499 _ => Err(unsupported_dtype("svd_values", input.dtype())),
1500 }
1501 }
1502 }
1503 }
1504}
1505
1506fn qr_entered(
1507 exec: &mut Exec<'_>,
1508 buffers: &mut BufferPool,
1509 input: &HostOperand<'_>,
1510) -> tenferro_tensor::Result<Vec<Tensor>> {
1511 {
1512 {
1513 {
1514 match input.dtype() {
1515 DType::F32 => linalg::faer_linalg::qr(
1516 exec,
1517 buffers,
1518 input
1519 .as_typed::<f32>()
1520 .ok_or_else(|| unsupported_dtype("qr", input.dtype()))?,
1521 )
1522 .map(pair_erased),
1523 DType::F64 => linalg::faer_linalg::qr(
1524 exec,
1525 buffers,
1526 input
1527 .as_typed::<f64>()
1528 .ok_or_else(|| unsupported_dtype("qr", input.dtype()))?,
1529 )
1530 .map(pair_erased),
1531 DType::C32 => linalg::faer_linalg::qr(
1532 exec,
1533 buffers,
1534 input
1535 .as_typed::<Complex32>()
1536 .ok_or_else(|| unsupported_dtype("qr", input.dtype()))?,
1537 )
1538 .map(pair_erased),
1539 DType::C64 => linalg::faer_linalg::qr(
1540 exec,
1541 buffers,
1542 input
1543 .as_typed::<Complex64>()
1544 .ok_or_else(|| unsupported_dtype("qr", input.dtype()))?,
1545 )
1546 .map(pair_erased),
1547 _ => Err(unsupported_dtype("qr", input.dtype())),
1548 }
1549 }
1550 }
1551 }
1552}
1553
1554fn rank_revealing_qr_entered(
1555 exec: &mut Exec<'_>,
1556 buffers: &mut BufferPool,
1557 input: &HostOperand<'_>,
1558 options: crate::RankRevealingQrOptions,
1559) -> tenferro_tensor::Result<Vec<Tensor>> {
1560 macro_rules! map_result {
1561 ($result:expr, $variant:ident) => {{
1562 $result.map(|result| {
1563 vec![
1564 Tensor::from_typed::<preset_scalar!($variant)>(result.q),
1565 Tensor::from_typed::<preset_scalar!($variant)>(result.r),
1566 Tensor::from_typed::<i64>(result.column_permutation),
1567 Tensor::from_typed::<i64>(result.rank),
1568 ]
1569 })
1570 }};
1571 }
1572 let mut outputs = {
1573 {
1574 {
1575 match input.dtype() {
1576 DType::F32 => map_result!(
1577 linalg::faer_linalg::rank_revealing_qr(
1578 exec,
1579 buffers,
1580 input.as_typed::<f32>().ok_or_else(|| unsupported_dtype(
1581 "rank_revealing_qr",
1582 input.dtype()
1583 ))?,
1584 options
1585 ),
1586 F32
1587 ),
1588 DType::F64 => map_result!(
1589 linalg::faer_linalg::rank_revealing_qr(
1590 exec,
1591 buffers,
1592 input.as_typed::<f64>().ok_or_else(|| unsupported_dtype(
1593 "rank_revealing_qr",
1594 input.dtype()
1595 ))?,
1596 options
1597 ),
1598 F64
1599 ),
1600 DType::C32 => {
1601 map_result!(
1602 linalg::faer_linalg::rank_revealing_qr(
1603 exec,
1604 buffers,
1605 input.as_typed::<Complex32>().ok_or_else(|| {
1606 unsupported_dtype("rank_revealing_qr", input.dtype())
1607 })?,
1608 options
1609 ),
1610 C32
1611 )
1612 }
1613 DType::C64 => {
1614 map_result!(
1615 linalg::faer_linalg::rank_revealing_qr(
1616 exec,
1617 buffers,
1618 input.as_typed::<Complex64>().ok_or_else(|| {
1619 unsupported_dtype("rank_revealing_qr", input.dtype())
1620 })?,
1621 options
1622 ),
1623 C64
1624 )
1625 }
1626 _ => Err(unsupported_dtype("rank_revealing_qr", input.dtype())),
1627 }
1628 }
1629 }
1630 }?;
1631 apply_qr_gauge(options.gauge, &mut outputs[..2])?;
1632 Ok(outputs)
1633}
1634
1635fn householder_qr_entered(
1636 exec: &mut Exec<'_>,
1637 buffers: &mut BufferPool,
1638 input: &HostOperand<'_>,
1639) -> tenferro_tensor::Result<CompactQrResult> {
1640 {
1641 {
1642 {
1643 macro_rules! factor {
1644 ($tensor:expr, $variant:ident) => {{
1645 let (packed, coeff) =
1646 linalg::faer_linalg::compact_factor_2d(exec, buffers, $tensor)?;
1647 Ok(CompactQrResult {
1648 packed: Tensor::from_typed::<preset_scalar!($variant)>(packed),
1649 coeff: Tensor::from_typed::<preset_scalar!($variant)>(coeff),
1650 })
1651 }};
1652 }
1653 match input.dtype() {
1654 DType::F32 => factor!(
1655 input
1656 .as_typed::<f32>()
1657 .ok_or_else(|| unsupported_dtype("householder_qr", input.dtype()))?,
1658 F32
1659 ),
1660 DType::F64 => factor!(
1661 input
1662 .as_typed::<f64>()
1663 .ok_or_else(|| unsupported_dtype("householder_qr", input.dtype()))?,
1664 F64
1665 ),
1666 DType::C32 => factor!(
1667 input
1668 .as_typed::<Complex32>()
1669 .ok_or_else(|| unsupported_dtype("householder_qr", input.dtype()))?,
1670 C32
1671 ),
1672 DType::C64 => factor!(
1673 input
1674 .as_typed::<Complex64>()
1675 .ok_or_else(|| unsupported_dtype("householder_qr", input.dtype()))?,
1676 C64
1677 ),
1678 _ => Err(unsupported_dtype("householder_qr", input.dtype())),
1679 }
1680 }
1681 }
1682 }
1683}
1684
1685fn householder_qr_from_factors_entered(
1686 exec: &mut Exec<'_>,
1687 buffers: &mut BufferPool,
1688 q: &HostOperand<'_>,
1689 r: &HostOperand<'_>,
1690) -> tenferro_tensor::Result<CompactQrResult> {
1691 if q.dtype() != r.dtype() {
1692 return Err(Error::dtype_mismatch(
1693 "householder_qr_from_factors",
1694 q.dtype(),
1695 r.dtype(),
1696 ));
1697 }
1698 {
1699 {
1700 macro_rules! import {
1701 ($q:expr, $r:expr, $variant:ident) => {{
1702 let (packed, coeff) =
1703 linalg::faer_linalg::from_factors_2d(exec, buffers, $q, $r)?;
1704 return Ok(CompactQrResult {
1705 packed: Tensor::from_typed::<preset_scalar!($variant)>(packed),
1706 coeff: Tensor::from_typed::<preset_scalar!($variant)>(coeff),
1707 });
1708 }};
1709 }
1710 match (q.dtype(), r.dtype()) {
1711 (DType::F32, DType::F32) => {
1712 let (q_t, r_t) = qr_import_operands::<f32>(q, r)?;
1713 import!(q_t, r_t, F32)
1714 }
1715 (DType::F64, DType::F64) => {
1716 let (q_t, r_t) = qr_import_operands::<f64>(q, r)?;
1717 import!(q_t, r_t, F64)
1718 }
1719 (DType::C32, DType::C32) => {
1720 let (q_t, r_t) = qr_import_operands::<Complex32>(q, r)?;
1721 import!(q_t, r_t, C32)
1722 }
1723 (DType::C64, DType::C64) => {
1724 let (q_t, r_t) = qr_import_operands::<Complex64>(q, r)?;
1725 import!(q_t, r_t, C64)
1726 }
1727 _ => Err(unsupported_dtype("householder_qr_from_factors", q.dtype())),
1728 }
1729 }
1730 }
1731}
1732
1733fn householder_qr_append_entered(
1734 exec: &mut Exec<'_>,
1735 buffers: &mut BufferPool,
1736 packed: &HostOperand<'_>,
1737 coeff: &HostOperand<'_>,
1738 block: &HostOperand<'_>,
1739) -> tenferro_tensor::Result<CompactQrResult> {
1740 if packed.dtype() != coeff.dtype() || packed.dtype() != block.dtype() {
1741 return Err(Error::dtype_mismatch(
1742 "householder_qr_append",
1743 packed.dtype(),
1744 block.dtype(),
1745 ));
1746 }
1747 {
1748 {
1749 macro_rules! append {
1750 ($packed:expr, $coeff:expr, $block:expr, $variant:ident) => {{
1751 let (packed, coeff) =
1752 linalg::faer_linalg::append_2d(exec, buffers, $packed, $coeff, $block)?;
1753 return Ok(CompactQrResult {
1754 packed: Tensor::from_typed::<preset_scalar!($variant)>(packed),
1755 coeff: Tensor::from_typed::<preset_scalar!($variant)>(coeff),
1756 });
1757 }};
1758 }
1759 match (packed.dtype(), coeff.dtype(), block.dtype()) {
1760 (DType::F32, DType::F32, DType::F32) => {
1761 let (p, c, b) = append_operands::<f32>(packed, coeff, block)?;
1762
1763 append!(p, c, b, F32)
1764 }
1765 (DType::F64, DType::F64, DType::F64) => {
1766 let (p, c, b) = append_operands::<f64>(packed, coeff, block)?;
1767
1768 append!(p, c, b, F64)
1769 }
1770 (DType::C32, DType::C32, DType::C32) => {
1771 let (p, c, b) = append_operands::<Complex32>(packed, coeff, block)?;
1772
1773 append!(p, c, b, C32)
1774 }
1775 (DType::C64, DType::C64, DType::C64) => {
1776 let (p, c, b) = append_operands::<Complex64>(packed, coeff, block)?;
1777
1778 append!(p, c, b, C64)
1779 }
1780 _ => Err(unsupported_dtype("householder_qr_append", packed.dtype())),
1781 }
1782 }
1783 }
1784}
1785
1786fn householder_qr_r_entered(
1787 _context: &mut Exec<'_>,
1788 buffers: &mut BufferPool,
1789 packed: &HostOperand<'_>,
1790 coeff: &HostOperand<'_>,
1791 options: crate::QrOptions,
1792) -> tenferro_tensor::Result<Tensor> {
1793 let positive = options.gauge == crate::QrGauge::PositiveDiagonal;
1794 same_variant_pair!(
1795 packed,
1796 coeff,
1797 |p, c| linalg::faer_linalg::raw_r_2d(buffers, p, c, positive),
1798 Err(Error::dtype_mismatch(
1799 "householder_qr_r",
1800 packed.dtype(),
1801 coeff.dtype(),
1802 ))
1803 )
1804}
1805
1806fn householder_qr_q_columns_entered(
1807 _context: &mut Exec<'_>,
1808 buffers: &mut BufferPool,
1809 packed: &HostOperand<'_>,
1810 coeff: &HostOperand<'_>,
1811 columns: std::ops::Range<usize>,
1812 options: crate::QrOptions,
1813) -> tenferro_tensor::Result<Tensor> {
1814 let positive = options.gauge == crate::QrGauge::PositiveDiagonal;
1815 match (packed.dtype(), coeff.dtype()) {
1816 (DType::F32, DType::F32) => {
1817 let (p, c) = q_columns_operands::<f32>(packed, coeff)?;
1818 linalg::faer_linalg::q_columns_2d(
1819 _context,
1820 buffers,
1821 p,
1822 c,
1823 columns.start,
1824 columns.end,
1825 positive,
1826 )
1827 .map(Tensor::from_typed::<f32>)
1828 }
1829 (DType::F64, DType::F64) => {
1830 let (p, c) = q_columns_operands::<f64>(packed, coeff)?;
1831 linalg::faer_linalg::q_columns_2d(
1832 _context,
1833 buffers,
1834 p,
1835 c,
1836 columns.start,
1837 columns.end,
1838 positive,
1839 )
1840 .map(Tensor::from_typed::<f64>)
1841 }
1842 (DType::C32, DType::C32) => {
1843 let (p, c) = q_columns_operands::<Complex32>(packed, coeff)?;
1844 linalg::faer_linalg::q_columns_2d(
1845 _context,
1846 buffers,
1847 p,
1848 c,
1849 columns.start,
1850 columns.end,
1851 positive,
1852 )
1853 .map(Tensor::from_typed::<Complex32>)
1854 }
1855 (DType::C64, DType::C64) => {
1856 let (p, c) = q_columns_operands::<Complex64>(packed, coeff)?;
1857 linalg::faer_linalg::q_columns_2d(
1858 _context,
1859 buffers,
1860 p,
1861 c,
1862 columns.start,
1863 columns.end,
1864 positive,
1865 )
1866 .map(Tensor::from_typed::<Complex64>)
1867 }
1868 _ => Err(Error::dtype_mismatch(
1869 "householder_qr_q_columns",
1870 packed.dtype(),
1871 coeff.dtype(),
1872 )),
1873 }
1874}
1875
1876fn eigh_entered(
1877 exec: &mut Exec<'_>,
1878 buffers: &mut BufferPool,
1879 input: &HostOperand<'_>,
1880) -> tenferro_tensor::Result<Vec<Tensor>> {
1881 {
1882 {
1883 {
1884 match input.dtype() {
1885 DType::F32 => linalg::faer_linalg::eigh(
1886 exec,
1887 buffers,
1888 input
1889 .as_typed::<f32>()
1890 .ok_or_else(|| unsupported_dtype("eigh", input.dtype()))?,
1891 )
1892 .map(pair_erased),
1893 DType::F64 => linalg::faer_linalg::eigh(
1894 exec,
1895 buffers,
1896 input
1897 .as_typed::<f64>()
1898 .ok_or_else(|| unsupported_dtype("eigh", input.dtype()))?,
1899 )
1900 .map(pair_erased),
1901 DType::C32 => linalg::faer_linalg::eigh(
1902 exec,
1903 buffers,
1904 input
1905 .as_typed::<Complex32>()
1906 .ok_or_else(|| unsupported_dtype("eigh", input.dtype()))?,
1907 )
1908 .map(pair_erased),
1909 DType::C64 => linalg::faer_linalg::eigh(
1910 exec,
1911 buffers,
1912 input
1913 .as_typed::<Complex64>()
1914 .ok_or_else(|| unsupported_dtype("eigh", input.dtype()))?,
1915 )
1916 .map(pair_erased),
1917 _ => Err(unsupported_dtype("eigh", input.dtype())),
1918 }
1919 }
1920 }
1921 }
1922}
1923
1924fn eigh_values_entered(
1925 exec: &mut Exec<'_>,
1926 buffers: &mut BufferPool,
1927 input: &HostOperand<'_>,
1928) -> tenferro_tensor::Result<Tensor> {
1929 {
1930 {
1931 {
1932 match input.dtype() {
1933 DType::F32 => {
1934 let t = input
1935 .as_typed::<f32>()
1936 .ok_or_else(|| unsupported_dtype("eigh_values", input.dtype()))?;
1937 linalg::faer_linalg::eigh_values(exec, buffers, t)
1938 .map(Tensor::from_typed::<f32>)
1939 }
1940 DType::F64 => {
1941 let t = input
1942 .as_typed::<f64>()
1943 .ok_or_else(|| unsupported_dtype("eigh_values", input.dtype()))?;
1944 linalg::faer_linalg::eigh_values(exec, buffers, t)
1945 .map(Tensor::from_typed::<f64>)
1946 }
1947 DType::C32 => {
1948 let t = input
1949 .as_typed::<Complex32>()
1950 .ok_or_else(|| unsupported_dtype("eigh_values", input.dtype()))?;
1951 linalg::faer_linalg::eigh_values(exec, buffers, t)
1952 .map(Tensor::from_typed::<f32>)
1953 }
1954 DType::C64 => {
1955 let t = input
1956 .as_typed::<Complex64>()
1957 .ok_or_else(|| unsupported_dtype("eigh_values", input.dtype()))?;
1958 linalg::faer_linalg::eigh_values(exec, buffers, t)
1959 .map(Tensor::from_typed::<f64>)
1960 }
1961 _ => Err(unsupported_dtype("eigh_values", input.dtype())),
1962 }
1963 }
1964 }
1965 }
1966}
1967
1968fn eig_values_entered(
1969 exec: &mut Exec<'_>,
1970 buffers: &mut BufferPool,
1971 input: &HostOperand<'_>,
1972) -> tenferro_tensor::Result<Tensor> {
1973 {
1974 {
1975 {
1976 linalg::faer_linalg::eig_values(exec, buffers, input)
1977 }
1978 }
1979 }
1980}
1981
1982fn eig_entered(
1983 exec: &mut Exec<'_>,
1984 buffers: &mut BufferPool,
1985 input: &HostOperand<'_>,
1986) -> tenferro_tensor::Result<Vec<Tensor>> {
1987 {
1988 {
1989 {
1990 linalg::faer_linalg::eig(exec, buffers, input)
1991 }
1992 }
1993 }
1994}
1995
1996fn faer_strided_host_ok(input: &HostOperand<'_>) -> bool {
1997 match input {
1998 TensorView::F32(view) => linalg::faer_linalg::faer_strided_ok(view),
1999 TensorView::F64(view) => linalg::faer_linalg::faer_strided_ok(view),
2000 TensorView::C32(view) => linalg::faer_linalg::faer_strided_ok(view),
2001 TensorView::C64(view) => linalg::faer_linalg::faer_strided_ok(view),
2002 TensorView::I32(_) | TensorView::I64(_) | TensorView::Bool(_) => false,
2004 }
2005}
2006
2007fn faer_rhs_host_ok(input: &HostOperand<'_>) -> bool {
2013 input.shape().len() == 2
2014 && matches!(
2015 input.dtype(),
2016 DType::F32 | DType::F64 | DType::C32 | DType::C64
2017 )
2018}
2019
2020fn rank_revealing_qr_faer_view_entered(
2021 exec: &mut Exec<'_>,
2022 buffers: &mut BufferPool,
2023 input: HostOperand<'_>,
2024 options: crate::RankRevealingQrOptions,
2025) -> tenferro_tensor::Result<Vec<Tensor>> {
2026 macro_rules! factor {
2027 ($view:expr, $scalar:ty) => {
2028 linalg::faer_linalg::rank_revealing_qr(exec, buffers, &$view, options).map(|result| {
2029 vec![
2030 Tensor::from_typed::<$scalar>(result.q),
2031 Tensor::from_typed::<$scalar>(result.r),
2032 Tensor::from_typed::<i64>(result.column_permutation),
2033 Tensor::from_typed::<i64>(result.rank),
2034 ]
2035 })
2036 };
2037 }
2038 let mut outputs = match input {
2039 TensorView::F32(view) => factor!(view, f32),
2040 TensorView::F64(view) => factor!(view, f64),
2041 TensorView::C32(view) => factor!(view, Complex32),
2042 TensorView::C64(view) => factor!(view, Complex64),
2043 unsupported => Err(unsupported_dtype("rank_revealing_qr", unsupported.dtype())),
2044 }?;
2045 apply_qr_gauge(options.gauge, &mut outputs[..2])?;
2046 Ok(outputs)
2047}
2048
2049fn triangular_solve_faer_view_entered(
2050 exec: &mut Exec<'_>,
2051 buffers: &mut BufferPool,
2052 a: HostOperand<'_>,
2053 b: HostOperand<'_>,
2054 options: TriangularSolveOptions,
2055) -> tenferro_tensor::Result<Tensor> {
2056 let TriangularSolveOptions {
2057 left_side,
2058 lower,
2059 transpose_a,
2060 unit_diagonal,
2061 } = options;
2062 macro_rules! solve {
2063 ($a:expr, $b:expr, $scalar:ty) => {
2064 linalg::faer_linalg::triangular_solve(
2065 exec,
2066 buffers,
2067 &$a,
2068 &$b,
2069 left_side,
2070 lower,
2071 transpose_a,
2072 unit_diagonal,
2073 )
2074 .map(Tensor::from_typed::<$scalar>)
2075 };
2076 }
2077 match (a, b) {
2078 (TensorView::F32(a), TensorView::F32(b)) => solve!(a, b, f32),
2079 (TensorView::F64(a), TensorView::F64(b)) => solve!(a, b, f64),
2080 (TensorView::C32(a), TensorView::C32(b)) => solve!(a, b, Complex32),
2081 (TensorView::C64(a), TensorView::C64(b)) => solve!(a, b, Complex64),
2082 (a, b) => Err(Error::dtype_mismatch(
2083 "triangular_solve",
2084 a.dtype(),
2085 b.dtype(),
2086 )),
2087 }
2088}
2089
2090fn svd_faer_view_entered(
2091 exec: &mut Exec<'_>,
2092 buffers: &mut BufferPool,
2093 input: HostOperand<'_>,
2094) -> tenferro_tensor::Result<Vec<Tensor>> {
2095 match input {
2096 TensorView::F32(view) => linalg::faer_linalg::svd(exec, buffers, &view).map(svd_erased),
2097 TensorView::F64(view) => linalg::faer_linalg::svd(exec, buffers, &view).map(svd_erased),
2098 TensorView::C32(view) => linalg::faer_linalg::svd(exec, buffers, &view).map(svd_erased),
2099 TensorView::C64(view) => linalg::faer_linalg::svd(exec, buffers, &view).map(svd_erased),
2100 unsupported => Err(unsupported_dtype("svd", unsupported.dtype())),
2101 }
2102}
2103
2104fn svd_full_faer_view_entered(
2105 exec: &mut Exec<'_>,
2106 buffers: &mut BufferPool,
2107 input: HostOperand<'_>,
2108) -> tenferro_tensor::Result<Vec<Tensor>> {
2109 match input {
2110 TensorView::F32(view) => {
2111 linalg::faer_linalg::svd_full(exec, buffers, &view).map(svd_erased)
2112 }
2113 TensorView::F64(view) => {
2114 linalg::faer_linalg::svd_full(exec, buffers, &view).map(svd_erased)
2115 }
2116 TensorView::C32(view) => {
2117 linalg::faer_linalg::svd_full(exec, buffers, &view).map(svd_erased)
2118 }
2119 TensorView::C64(view) => {
2120 linalg::faer_linalg::svd_full(exec, buffers, &view).map(svd_erased)
2121 }
2122 unsupported => Err(unsupported_dtype("svd_full", unsupported.dtype())),
2123 }
2124}
2125
2126fn svd_values_faer_view_entered(
2127 exec: &mut Exec<'_>,
2128 buffers: &mut BufferPool,
2129 input: HostOperand<'_>,
2130) -> tenferro_tensor::Result<Tensor> {
2131 match input {
2132 TensorView::F32(view) => {
2133 linalg::faer_linalg::svd_values(exec, buffers, &view).map(Tensor::from_typed::<f32>)
2134 }
2135 TensorView::F64(view) => {
2136 linalg::faer_linalg::svd_values(exec, buffers, &view).map(Tensor::from_typed::<f64>)
2137 }
2138 TensorView::C32(view) => {
2139 linalg::faer_linalg::svd_values(exec, buffers, &view).map(Tensor::from_typed::<f32>)
2140 }
2141 TensorView::C64(view) => {
2142 linalg::faer_linalg::svd_values(exec, buffers, &view).map(Tensor::from_typed::<f64>)
2143 }
2144 unsupported => Err(unsupported_dtype("svd_values", unsupported.dtype())),
2145 }
2146}
2147
2148fn qr_faer_view_entered(
2149 exec: &mut Exec<'_>,
2150 buffers: &mut BufferPool,
2151 input: HostOperand<'_>,
2152) -> tenferro_tensor::Result<Vec<Tensor>> {
2153 match input {
2154 TensorView::F32(view) => linalg::faer_linalg::qr(exec, buffers, &view).map(pair_erased),
2155 TensorView::F64(view) => linalg::faer_linalg::qr(exec, buffers, &view).map(pair_erased),
2156 TensorView::C32(view) => linalg::faer_linalg::qr(exec, buffers, &view).map(pair_erased),
2157 TensorView::C64(view) => linalg::faer_linalg::qr(exec, buffers, &view).map(pair_erased),
2158 unsupported => Err(unsupported_dtype("qr", unsupported.dtype())),
2159 }
2160}
2161
2162fn eigh_faer_view_entered(
2163 exec: &mut Exec<'_>,
2164 buffers: &mut BufferPool,
2165 input: HostOperand<'_>,
2166) -> tenferro_tensor::Result<Vec<Tensor>> {
2167 match input {
2168 TensorView::F32(view) => linalg::faer_linalg::eigh(exec, buffers, &view).map(pair_erased),
2169 TensorView::F64(view) => linalg::faer_linalg::eigh(exec, buffers, &view).map(pair_erased),
2170 TensorView::C32(view) => linalg::faer_linalg::eigh(exec, buffers, &view).map(pair_erased),
2171 TensorView::C64(view) => linalg::faer_linalg::eigh(exec, buffers, &view).map(pair_erased),
2172 unsupported => Err(unsupported_dtype("eigh", unsupported.dtype())),
2173 }
2174}
2175
2176fn eigh_values_faer_view_entered(
2177 exec: &mut Exec<'_>,
2178 buffers: &mut BufferPool,
2179 input: HostOperand<'_>,
2180) -> tenferro_tensor::Result<Tensor> {
2181 match input {
2182 TensorView::F32(view) => {
2183 linalg::faer_linalg::eigh_values(exec, buffers, &view).map(Tensor::from_typed::<f32>)
2184 }
2185 TensorView::F64(view) => {
2186 linalg::faer_linalg::eigh_values(exec, buffers, &view).map(Tensor::from_typed::<f64>)
2187 }
2188 TensorView::C32(view) => {
2189 linalg::faer_linalg::eigh_values(exec, buffers, &view).map(Tensor::from_typed::<f32>)
2190 }
2191 TensorView::C64(view) => {
2192 linalg::faer_linalg::eigh_values(exec, buffers, &view).map(Tensor::from_typed::<f64>)
2193 }
2194 unsupported => Err(unsupported_dtype("eigh_values", unsupported.dtype())),
2195 }
2196}
2197
2198fn cholesky_faer_view_entered(
2199 exec: &mut Exec<'_>,
2200 buffers: &mut BufferPool,
2201 input: HostOperand<'_>,
2202) -> tenferro_tensor::Result<Tensor> {
2203 match input {
2204 TensorView::F32(view) => {
2205 linalg::faer_linalg::cholesky(exec, buffers, &view).map(Tensor::from_typed::<f32>)
2206 }
2207 TensorView::F64(view) => {
2208 linalg::faer_linalg::cholesky(exec, buffers, &view).map(Tensor::from_typed::<f64>)
2209 }
2210 TensorView::C32(view) => {
2211 linalg::faer_linalg::cholesky(exec, buffers, &view).map(Tensor::from_typed::<Complex32>)
2212 }
2213 TensorView::C64(view) => {
2214 linalg::faer_linalg::cholesky(exec, buffers, &view).map(Tensor::from_typed::<Complex64>)
2215 }
2216 unsupported => Err(unsupported_dtype("cholesky", unsupported.dtype())),
2217 }
2218}
2219
2220fn lu_faer_view_entered(
2221 exec: &mut Exec<'_>,
2222 buffers: &mut BufferPool,
2223 input: HostOperand<'_>,
2224) -> tenferro_tensor::Result<Vec<Tensor>> {
2225 match input {
2226 TensorView::F32(view) => linalg::faer_linalg::lu(exec, buffers, &view)
2227 .map(|outputs| outputs.into_iter().map(Tensor::from_typed::<f32>).collect()),
2228 TensorView::F64(view) => linalg::faer_linalg::lu(exec, buffers, &view)
2229 .map(|outputs| outputs.into_iter().map(Tensor::from_typed::<f64>).collect()),
2230 TensorView::C32(view) => linalg::faer_linalg::lu(exec, buffers, &view).map(|outputs| {
2231 outputs
2232 .into_iter()
2233 .map(Tensor::from_typed::<Complex32>)
2234 .collect()
2235 }),
2236 TensorView::C64(view) => linalg::faer_linalg::lu(exec, buffers, &view).map(|outputs| {
2237 outputs
2238 .into_iter()
2239 .map(Tensor::from_typed::<Complex64>)
2240 .collect()
2241 }),
2242 unsupported => Err(unsupported_dtype("lu", unsupported.dtype())),
2243 }
2244}
2245
2246fn full_piv_lu_faer_view_entered(
2247 exec: &mut Exec<'_>,
2248 buffers: &mut BufferPool,
2249 input: HostOperand<'_>,
2250) -> tenferro_tensor::Result<Vec<Tensor>> {
2251 match input {
2252 TensorView::F32(view) => linalg::faer_linalg::full_piv_lu(exec, buffers, &view)
2253 .map(|outputs| outputs.into_iter().map(Tensor::from_typed::<f32>).collect()),
2254 TensorView::F64(view) => linalg::faer_linalg::full_piv_lu(exec, buffers, &view)
2255 .map(|outputs| outputs.into_iter().map(Tensor::from_typed::<f64>).collect()),
2256 TensorView::C32(view) => {
2257 linalg::faer_linalg::full_piv_lu(exec, buffers, &view).map(|outputs| {
2258 outputs
2259 .into_iter()
2260 .map(Tensor::from_typed::<Complex32>)
2261 .collect()
2262 })
2263 }
2264 TensorView::C64(view) => {
2265 linalg::faer_linalg::full_piv_lu(exec, buffers, &view).map(|outputs| {
2266 outputs
2267 .into_iter()
2268 .map(Tensor::from_typed::<Complex64>)
2269 .collect()
2270 })
2271 }
2272 unsupported => Err(unsupported_dtype("full_piv_lu", unsupported.dtype())),
2273 }
2274}
2275
2276fn ensure_supported_linalg_pair(
2277 op: &'static str,
2278 lhs: &HostOperand<'_>,
2279 rhs: &HostOperand<'_>,
2280) -> tenferro_tensor::Result<()> {
2281 ensure_supported_linalg_dtypes(op, lhs.dtype(), rhs.dtype())
2282}
2283
2284fn ensure_supported_linalg_dtypes(
2285 op: &'static str,
2286 lhs: DType,
2287 rhs: DType,
2288) -> tenferro_tensor::Result<()> {
2289 if lhs != rhs {
2290 return Err(Error::dtype_mismatch(op, lhs, rhs));
2291 }
2292 ensure_supported_linalg_dtype(op, lhs)
2293}
2294
2295fn linalg_view<'a>(
2302 op: &'static str,
2303 read: TensorRead<'a>,
2304) -> tenferro_tensor::Result<TensorView<'a>> {
2305 let dtype = read.dtype();
2306 read.tensor_view()
2307 .ok_or_else(|| unsupported_dtype(op, dtype))
2308}
2309
2310fn ensure_supported_linalg_dtype(op: &'static str, dtype: DType) -> tenferro_tensor::Result<()> {
2311 match dtype {
2312 DType::F32 | DType::F64 | DType::C32 | DType::C64 => Ok(()),
2313 DType::I32 | DType::I64 | DType::Bool | DType::External(_) => {
2314 Err(unsupported_dtype(op, dtype))
2315 }
2316 }
2317}
2318
2319fn has_zero_dim(shape: &[usize]) -> bool {
2320 shape.contains(&0)
2321}
2322
2323fn batched_vector_rhs_shape(a: &HostOperand<'_>, b: &HostOperand<'_>) -> Option<Vec<usize>> {
2324 batched_vector_rhs_shape_of(a.shape(), b.shape())
2325}
2326
2327fn batched_vector_rhs_shape_of(a_shape: &[usize], b_shape: &[usize]) -> Option<Vec<usize>> {
2329 if b_shape.len() == 1 {
2330 return Some(vec![b_shape[0], 1]);
2331 }
2332
2333 let is_batched_vector_rhs = a_shape.len() == b_shape.len() + 1
2334 && !b_shape.is_empty()
2335 && b_shape[0] == a_shape[0]
2336 && b_shape[1..] == a_shape[2..];
2337 if !is_batched_vector_rhs {
2338 return None;
2339 }
2340
2341 let mut rhs_shape = vec![b_shape[0], 1];
2342 rhs_shape.extend_from_slice(&b_shape[1..]);
2343 Some(rhs_shape)
2344}
2345
2346fn zeros_like_tensor(dtype: DType, shape: &[usize]) -> tenferro_tensor::Result<Tensor> {
2347 Ok(match dtype {
2348 DType::External(type_id) => {
2350 return Err(unsupported_dtype(
2351 "zeros_like_tensor",
2352 DType::External(type_id),
2353 ));
2354 }
2355 DType::F32 => Tensor::from_typed::<f32>(TypedTensor::zeros(shape.to_vec())?),
2356 DType::F64 => Tensor::from_typed::<f64>(TypedTensor::zeros(shape.to_vec())?),
2357 DType::I32 => Tensor::from_typed::<i32>(TypedTensor::zeros(shape.to_vec())?),
2358 DType::I64 => Tensor::from_typed::<i64>(TypedTensor::zeros(shape.to_vec())?),
2359 DType::Bool => {
2360 let len = shape
2361 .iter()
2362 .try_fold(1usize, |acc, &dim| acc.checked_mul(dim))
2363 .ok_or_else(|| {
2364 Error::invalid_argument(
2365 "zeros_like_tensor",
2366 "shape",
2367 "element count overflows usize",
2368 )
2369 })?;
2370 Tensor::from_typed::<bool>(TypedTensor::from_vec_col_major(
2371 shape.to_vec(),
2372 vec![false; len],
2373 )?)
2374 }
2375 DType::C32 => Tensor::from_typed::<Complex32>(TypedTensor::zeros(shape.to_vec())?),
2376 DType::C64 => Tensor::from_typed::<Complex64>(TypedTensor::zeros(shape.to_vec())?),
2377 })
2378}
2379
2380fn complex32_real_part_tensor(
2381 values: TypedTensor<Complex32>,
2382) -> tenferro_tensor::Result<TypedTensor<f32>> {
2383 let mut out = TypedTensor::from_vec_col_major(
2384 values.shape().to_vec(),
2385 values.host_data()?.iter().map(|value| value.re).collect(),
2386 )?;
2387 out.set_placement(values.placement().clone())?;
2388 Ok(out)
2389}
2390
2391fn complex64_real_part_tensor(
2392 values: TypedTensor<Complex64>,
2393) -> tenferro_tensor::Result<TypedTensor<f64>> {
2394 let mut out = TypedTensor::from_vec_col_major(
2395 values.shape().to_vec(),
2396 values.host_data()?.iter().map(|value| value.re).collect(),
2397 )?;
2398 out.set_placement(values.placement().clone())?;
2399 Ok(out)
2400}
2401
2402fn svd_erased<T: TensorScalar, R: TensorScalar>(
2404 (u, s, vt): (TypedTensor<T>, TypedTensor<R>, TypedTensor<T>),
2405) -> Vec<Tensor> {
2406 vec![
2407 Tensor::from_typed(u),
2408 Tensor::from_typed(s),
2409 Tensor::from_typed(vt),
2410 ]
2411}
2412
2413fn pair_erased<A: TensorScalar, B: TensorScalar>(
2415 (first, second): (TypedTensor<A>, TypedTensor<B>),
2416) -> Vec<Tensor> {
2417 vec![Tensor::from_typed(first), Tensor::from_typed(second)]
2418}
2419
2420fn full_piv_lu_output_count_error(count: usize) -> Error {
2421 Error::Internal(format!(
2422 "full_piv_lu produced an invalid output count: expected 5, got {count}"
2423 ))
2424}
2425
2426fn full_piv_lu_c32_outputs_to_public_tensors(
2427 outputs: Vec<TypedTensor<Complex32>>,
2428) -> tenferro_tensor::Result<Vec<Tensor>> {
2429 let count = outputs.len();
2430 let mut outputs = outputs.into_iter();
2431 match (
2432 outputs.next(),
2433 outputs.next(),
2434 outputs.next(),
2435 outputs.next(),
2436 outputs.next(),
2437 outputs.next(),
2438 ) {
2439 (Some(p), Some(l), Some(u), Some(q), Some(parity), None) => Ok(vec![
2440 Tensor::from_typed::<Complex32>(p),
2441 Tensor::from_typed::<Complex32>(l),
2442 Tensor::from_typed::<Complex32>(u),
2443 Tensor::from_typed::<Complex32>(q),
2444 Tensor::from_typed::<f32>(complex32_real_part_tensor(parity)?),
2445 ]),
2446 _ => Err(full_piv_lu_output_count_error(count)),
2447 }
2448}
2449
2450fn full_piv_lu_c64_outputs_to_public_tensors(
2451 outputs: Vec<TypedTensor<Complex64>>,
2452) -> tenferro_tensor::Result<Vec<Tensor>> {
2453 let count = outputs.len();
2454 let mut outputs = outputs.into_iter();
2455 match (
2456 outputs.next(),
2457 outputs.next(),
2458 outputs.next(),
2459 outputs.next(),
2460 outputs.next(),
2461 outputs.next(),
2462 ) {
2463 (Some(p), Some(l), Some(u), Some(q), Some(parity), None) => Ok(vec![
2464 Tensor::from_typed::<Complex64>(p),
2465 Tensor::from_typed::<Complex64>(l),
2466 Tensor::from_typed::<Complex64>(u),
2467 Tensor::from_typed::<Complex64>(q),
2468 Tensor::from_typed::<f64>(complex64_real_part_tensor(parity)?),
2469 ]),
2470 _ => Err(full_piv_lu_output_count_error(count)),
2471 }
2472}
2473
2474fn validate_lu_solve_prepared_shapes(
2475 lu_shape: &[usize],
2476 pivots_shape: &[usize],
2477 b_shape: &[usize],
2478) -> tenferro_tensor::Result<()> {
2479 let n = square_matrix_dim("lu_solve_prepared", lu_shape)?;
2480 let (b_rows, _) = matrix_dims("lu_solve_prepared", b_shape)?;
2481 if b_rows != n {
2482 return Err(Error::invalid_argument(
2483 "lu_solve_prepared",
2484 "rhs rows",
2485 format!("expected {n}, got {b_rows}"),
2486 ));
2487 }
2488 if lu_shape[2..] != b_shape[2..] {
2489 return Err(Error::shape_mismatch(
2490 "lu_solve_prepared",
2491 lu_shape.to_vec(),
2492 b_shape.to_vec(),
2493 ));
2494 }
2495 let mut expected_pivots = vec![n];
2496 expected_pivots.extend_from_slice(&lu_shape[2..]);
2497 if pivots_shape != expected_pivots {
2498 return Err(Error::shape_mismatch(
2499 "lu_solve_prepared",
2500 expected_pivots,
2501 pivots_shape.to_vec(),
2502 ));
2503 }
2504 Ok(())
2505}
2506
2507fn matrix_dims(op: &'static str, shape: &[usize]) -> tenferro_tensor::Result<(usize, usize)> {
2508 if shape.len() < 2 {
2509 return Err(Error::rank_mismatch(op, 2, shape.len()));
2510 }
2511 Ok((shape[0], shape[1]))
2512}
2513
2514fn square_matrix_dim(op: &'static str, shape: &[usize]) -> tenferro_tensor::Result<usize> {
2515 let (rows, cols) = matrix_dims(op, shape)?;
2516 if rows != cols {
2517 return Err(Error::shape_mismatch(op, vec![rows], vec![cols]));
2518 }
2519 Ok(rows)
2520}
2521
2522fn unsupported_pair(
2523 op: &'static str,
2524 lhs: &HostOperand<'_>,
2525 rhs: &HostOperand<'_>,
2526) -> tenferro_tensor::Result<Tensor> {
2527 if lhs.dtype() != rhs.dtype() {
2528 Err(Error::dtype_mismatch(op, lhs.dtype(), rhs.dtype()))
2529 } else {
2530 Err(unsupported_dtype(op, lhs.dtype()))
2531 }
2532}
2533
2534mod packed_lu;
2535
2536#[cfg(test)]
2537mod tests;