Skip to main content

tensor4all_core/
chain_compression.rs

1//! Two-sweep compression of chains of rank-3 cores.
2//!
3//! [`compress_chain_lu_mut`] and [`compress_chain_lu_on_session`] compress a
4//! chain with pivoted LU factorizations. [`compress_chain_cores_mut`] runs the
5//! same two sweeps with a caller-supplied factorization and multiplication,
6//! and [`ChainCore`] adapts a storage type to the sweeps.
7
8use std::collections::TryReserveError;
9
10use tensor4all_tensorbackend::{
11    explicit::Session, mat_mul, Matrix, MatrixMulError, MatrixShapeError,
12};
13
14use crate::col_major_array::ColMajorArrayError;
15use crate::{
16    matrix_luci_factors_from_matrix_owned, matrix_luci_factors_from_matrix_owned_on_session,
17    rrlu_mut, ColMajorArray, ColMajorArrayRef, MatrixCIError, MatrixLuciScalar, RrLUOptions,
18};
19
20/// Relative pivot threshold used by the forward sweep, and by the backward
21/// sweep when [`ChainLuOptions::tolerance`] is zero.
22const PIVOT_FLOOR: f64 = 1e-14;
23
24/// Failure to validate or compress a chain of rank-3 cores.
25///
26/// # Examples
27///
28/// ```
29/// use tensor4all_core::{compress_chain_lu_mut, ChainCompressionError, ChainLuOptions, ColMajorArray};
30/// let core = ColMajorArray::new(vec![1.0_f64], vec![1, 1])?;
31/// let error = compress_chain_lu_mut(&mut [core], &ChainLuOptions::default()).unwrap_err();
32/// assert!(matches!(error, ChainCompressionError::CoreRank { site: 0, rank: 2 }));
33/// # Ok::<(), Box<dyn std::error::Error>>(())
34/// ```
35#[derive(Debug, thiserror::Error)]
36#[non_exhaustive]
37pub enum ChainCompressionError {
38    /// A core has a rank other than three.
39    #[error("core at site {site} must have three axes, got {rank}")]
40    CoreRank {
41        /// Zero-based site of the invalid core.
42        site: usize,
43        /// Number of axes of the supplied core.
44        rank: usize,
45    },
46    /// The right bond of a site disagrees with the next site's left bond.
47    #[error(
48        "core at site {site} has right bond {right}, but the next core has left bond {next_left}"
49    )]
50    BondMismatch {
51        /// Zero-based site to the left of the mismatched bond.
52        site: usize,
53        /// Dimension of the bond at `site`.
54        right: usize,
55        /// Dimension of the same bond at `site + 1`.
56        next_left: usize,
57    },
58    /// The left boundary bond must have dimension one.
59    #[error("First tensor must have left dimension 1")]
60    LeftBoundary,
61    /// The right boundary bond must have dimension one.
62    #[error("Last tensor must have right dimension 1")]
63    RightBoundary,
64    /// The pivot tolerance must be finite and nonnegative.
65    #[error("LU compression tolerance must be finite and nonnegative, got {0}")]
66    InvalidTolerance(f64),
67    /// A rank cap of zero would discard every bond.
68    #[error("LU compression max_bond_dim must be positive when specified")]
69    ZeroBondCap,
70    /// Fusing two core axes would overflow `usize`.
71    #[error("unfolding core shape {0:?} overflows usize")]
72    ShapeOverflow([usize; 3]),
73    /// A flat buffer does not hold a whole number of unfolding blocks.
74    #[error("flat buffer of length {len} is not a whole number of {block}-element blocks")]
75    BufferLength {
76        /// Length of the supplied buffer.
77        len: usize,
78        /// Element count of one block, or zero when a block dimension is zero.
79        block: usize,
80    },
81    /// The flat buffer could not be allocated.
82    #[error("cannot allocate a compression buffer: {0}")]
83    Allocation(#[from] TryReserveError),
84    /// A column-major array could not be constructed.
85    #[error(transparent)]
86    Array(#[from] ColMajorArrayError),
87    /// An unfolding or factor had incompatible matrix dimensions.
88    #[error(transparent)]
89    MatrixShape(#[from] MatrixShapeError),
90    /// A matrix factorization failed.
91    #[error(transparent)]
92    Factorization(#[from] MatrixCIError),
93    /// A matrix factor could not be absorbed into the adjacent core.
94    #[error("{direction} compression multiply failed: {source}")]
95    Multiply {
96        /// Sweep direction in which the multiplication failed.
97        direction: &'static str,
98        /// Underlying backend multiplication failure.
99        #[source]
100        source: MatrixMulError,
101    },
102}
103
104/// Pivoted factorization applied at each bond by [`compress_chain_lu_mut`].
105///
106/// # Examples
107///
108/// ```
109/// use tensor4all_core::ChainLuFactorization;
110/// assert_eq!(ChainLuFactorization::default(), ChainLuFactorization::RrLu);
111/// ```
112#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
113#[non_exhaustive]
114pub enum ChainLuFactorization {
115    /// Rank-revealing LU with full pivoting ([`rrlu_mut`]).
116    #[default]
117    RrLu,
118    /// LU-based cross interpolation ([`crate::MatrixLUCI`]), which selects the
119    /// same pivots and assembles the factors with backend products and solves.
120    Luci,
121}
122
123/// Truncation settings for [`compress_chain_lu_mut`] and
124/// [`compress_chain_lu_on_session`].
125///
126/// The forward sweep applies a fixed `1e-14` relative pivot cutoff and no
127/// rank cap. The truncation settings below apply only to the backward sweep,
128/// where `tolerance` and `max_bond_dim` both limit the retained rank of each bond
129/// and the smaller limit wins. `normalize_error` selects whether `tolerance`
130/// is relative to the largest pivot of that bond or absolute.
131///
132/// | Field | Default | Meaning |
133/// |-------|---------|---------|
134/// | `tolerance` | `1e-12` | Pivot threshold; zero selects the `1e-14` relative floor |
135/// | `max_bond_dim` | `None` | Positive cap on each internal bond, or uncapped |
136/// | `normalize_error` | `true` | Relative (`true`) or absolute (`false`) `tolerance` |
137/// | `factorization` | `RrLu` | Pivoted factorization at each bond |
138///
139/// Start with the defaults. Larger tolerances can retain fewer pivots, but a
140/// local pivot threshold does not bound the approximation error of the chain.
141///
142/// # Examples
143///
144/// ```
145/// use tensor4all_core::{ChainLuFactorization, ChainLuOptions};
146/// let options = ChainLuOptions {
147///     tolerance: 1e-8,
148///     max_bond_dim: Some(16),
149///     ..ChainLuOptions::default()
150/// };
151/// assert!(options.normalize_error);
152/// assert_eq!(options.factorization, ChainLuFactorization::RrLu);
153/// ```
154#[derive(Clone, Copy, Debug, PartialEq)]
155pub struct ChainLuOptions {
156    /// Finite nonnegative pivot threshold for the backward sweep. Zero selects
157    /// the `1e-14` relative floor regardless of `normalize_error`.
158    pub tolerance: f64,
159    /// Positive cap on each internal bond in the backward sweep. `None` leaves
160    /// the rank limited only by `tolerance`.
161    pub max_bond_dim: Option<usize>,
162    /// Whether a positive `tolerance` is relative to the largest pivot of each
163    /// bond (`true`) or an absolute pivot threshold (`false`).
164    pub normalize_error: bool,
165    /// Pivoted factorization applied at each bond.
166    pub factorization: ChainLuFactorization,
167}
168
169impl Default for ChainLuOptions {
170    fn default() -> Self {
171        Self {
172            tolerance: 1e-12,
173            max_bond_dim: None,
174            normalize_error: true,
175            factorization: ChainLuFactorization::RrLu,
176        }
177    }
178}
179
180impl ChainLuOptions {
181    fn validate(&self) -> Result<(), ChainCompressionError> {
182        if !self.tolerance.is_finite() || self.tolerance < 0.0 {
183            return Err(ChainCompressionError::InvalidTolerance(self.tolerance));
184        }
185        if self.max_bond_dim == Some(0) {
186            return Err(ChainCompressionError::ZeroBondCap);
187        }
188        Ok(())
189    }
190
191    fn rrlu_options(&self, left_orthogonal: bool) -> RrLUOptions {
192        if left_orthogonal {
193            return RrLUOptions {
194                max_bond_dim: usize::MAX,
195                rel_tol: PIVOT_FLOOR,
196                abs_tol: 0.0,
197                left_orthogonal,
198            };
199        }
200        let (rel_tol, abs_tol) = if self.tolerance == 0.0 {
201            (PIVOT_FLOOR, 0.0)
202        } else if self.normalize_error {
203            (self.tolerance, 0.0)
204        } else {
205            (0.0, self.tolerance)
206        };
207        RrLUOptions {
208            max_bond_dim: self.max_bond_dim.unwrap_or(usize::MAX),
209            rel_tol,
210            abs_tol,
211            left_orthogonal,
212        }
213    }
214
215    fn factorize<T: MatrixLuciScalar>(
216        &self,
217        mut matrix: Matrix<T>,
218        left_orthogonal: bool,
219        session: Option<&mut Session<'_>>,
220    ) -> Result<(Matrix<T>, Matrix<T>), ChainCompressionError> {
221        let options = Some(self.rrlu_options(left_orthogonal));
222        match self.factorization {
223            ChainLuFactorization::RrLu => {
224                let lu = rrlu_mut(&mut matrix, options)?;
225                Ok((lu.left(true), lu.right(true)))
226            }
227            ChainLuFactorization::Luci => {
228                let factors = match session {
229                    Some(session) => {
230                        matrix_luci_factors_from_matrix_owned_on_session(matrix, options, session)
231                    }
232                    None => matrix_luci_factors_from_matrix_owned(matrix, options),
233                }?;
234                Ok((factors.left, factors.right))
235            }
236        }
237    }
238}
239
240/// Compress a chain of rank-3 cores in place with pivoted LU.
241///
242/// Each core has shape `[left_bond, site, right_bond]`. Neighboring bond
243/// dimensions must match and the outer bonds must have dimension one. An
244/// empty or single-core chain is validated and left unchanged. The forward
245/// sweep applies a fixed `1e-14` relative pivot cutoff and no rank cap. The
246/// backward sweep truncates according to `options`. Backend operations use
247/// the process-global CPU context. [`compress_chain_lu_on_session`] runs them
248/// on a caller-held session.
249///
250/// # Errors
251///
252/// Returns [`ChainCompressionError::InvalidTolerance`] for a negative or
253/// nonfinite tolerance and [`ChainCompressionError::ZeroBondCap`] for
254/// `max_bond_dim: Some(0)`, before touching any core. Invalid core shapes and
255/// factorization, multiplication or allocation failures return the other
256/// [`ChainCompressionError`] variants, converted into the storage type's error.
257/// If a numerical operation fails, bonds updated before the failure keep
258/// their compressed values.
259///
260/// # Examples
261///
262/// ```
263/// use tensor4all_core::{compress_chain_lu_mut, ChainLuOptions, ColMajorArray};
264/// let mut cores = [
265///     ColMajorArray::new(vec![2.0_f64, 2.0], vec![1, 1, 2])?,
266///     ColMajorArray::new(vec![3.0, 3.0], vec![2, 1, 1])?,
267/// ];
268/// compress_chain_lu_mut(&mut cores, &ChainLuOptions::default())?;
269/// assert_eq!(cores[0].shape(), &[1, 1, 1]);
270/// assert_eq!(cores[0].data()[0] * cores[1].data()[0], 12.0);
271/// # Ok::<(), Box<dyn std::error::Error>>(())
272/// ```
273pub fn compress_chain_lu_mut<T: MatrixLuciScalar, C: ChainCore<T>>(
274    cores: &mut [C],
275    options: &ChainLuOptions,
276) -> Result<(), C::Error> {
277    options.validate()?;
278    compress_chain_cores_mut(
279        cores,
280        &mut (),
281        |(), matrix, left_orthogonal| Ok(options.factorize(matrix, left_orthogonal, None)?),
282        |(), left, right| mat_mul(left, right),
283    )
284}
285
286/// Compress a chain of rank-3 cores in place with pivoted LU on a caller-held
287/// session.
288///
289/// Identical to [`compress_chain_lu_mut`], including validation and retained
290/// ranks, with all matrix products and LUCI triangular solves running on
291/// `session`.
292///
293/// # Errors
294///
295/// Returns [`ChainCompressionError::InvalidTolerance`] for a negative or
296/// nonfinite tolerance and [`ChainCompressionError::ZeroBondCap`] for
297/// `max_bond_dim: Some(0)`, before touching any core. Returns
298/// [`ChainCompressionError::Multiply`] when a product fails on `session`.
299/// Invalid core shapes and factorization or allocation failures return the
300/// other [`ChainCompressionError`] variants, converted into the storage
301/// type's error. If a numerical operation fails, bonds updated before the
302/// failure keep their compressed values.
303///
304/// # Examples
305///
306/// ```
307/// use tensor4all_core::{compress_chain_lu_on_session, ChainLuOptions, ColMajorArray};
308/// use tensor4all_tensorbackend::{explicit::Session, CpuBackend};
309/// let mut backend = CpuBackend::with_threads(1)?;
310/// let mut session = Session::new(backend.engine_mut());
311/// let mut cores = [
312///     ColMajorArray::new(vec![2.0_f64, 2.0], vec![1, 1, 2])?,
313///     ColMajorArray::new(vec![3.0, 3.0], vec![2, 1, 1])?,
314/// ];
315/// compress_chain_lu_on_session(&mut session, &mut cores, &ChainLuOptions::default())?;
316/// assert_eq!(cores[1].shape(), &[1, 1, 1]);
317/// assert_eq!(cores[0].data()[0] * cores[1].data()[0], 12.0);
318/// # Ok::<(), Box<dyn std::error::Error>>(())
319/// ```
320pub fn compress_chain_lu_on_session<T: MatrixLuciScalar, C: ChainCore<T>>(
321    session: &mut Session<'_>,
322    cores: &mut [C],
323    options: &ChainLuOptions,
324) -> Result<(), C::Error> {
325    options.validate()?;
326    compress_chain_cores_mut(
327        cores,
328        session,
329        |session, matrix, left_orthogonal| {
330            Ok(options.factorize(matrix, left_orthogonal, Some(session))?)
331        },
332        |session, left, right| session.mat_mul(left, right),
333    )
334}
335
336/// Storage adapter that lets the chain sweeps read and rebuild rank-3 cores.
337///
338/// [`ColMajorArray`] implements it directly; other crates implement it for
339/// their own core types so that [`compress_chain_lu_mut`] and
340/// [`compress_chain_cores_mut`] can update them in place.
341///
342/// # Examples
343///
344/// ```
345/// use tensor4all_core::{ChainCore, ColMajorArray};
346/// let core = <ColMajorArray<f64> as ChainCore<f64>>::from_col_major([1, 2, 1], vec![1.0, 2.0])?;
347/// let view = ChainCore::<f64>::view(&core)?;
348/// assert_eq!(view.shape(), &[1, 2, 1]);
349/// assert_eq!(view.data(), &[1.0, 2.0]);
350/// # Ok::<(), Box<dyn std::error::Error>>(())
351/// ```
352pub trait ChainCore<T>: Sized {
353    /// Error returned when reading or rebuilding a core, and by the sweeps.
354    type Error: From<ChainCompressionError>;
355
356    /// Borrow the shape and flat data in column-major order.
357    ///
358    /// # Errors
359    /// Returns the adapter's error for noncompact or device-only storage,
360    /// or a shape mismatch between the declared extents and borrowed data.
361    fn view(&self) -> Result<ColMajorArrayRef<'_, T>, Self::Error>;
362
363    /// Rebuild a core from column-major data.
364    ///
365    /// # Errors
366    /// Returns the adapter's error for a shape mismatch with the data length
367    /// or a shape or stride overflow.
368    fn from_col_major(shape: [usize; 3], data: Vec<T>) -> Result<Self, Self::Error>;
369}
370
371impl<T> ChainCore<T> for ColMajorArray<T> {
372    type Error = ChainCompressionError;
373
374    fn view(&self) -> Result<ColMajorArrayRef<'_, T>, Self::Error> {
375        Ok(self.as_ref())
376    }
377
378    fn from_col_major(shape: [usize; 3], data: Vec<T>) -> Result<Self, Self::Error> {
379        Ok(Self::new(data, shape.to_vec())?)
380    }
381}
382
383/// Run the two compression sweeps with a caller-supplied factorization.
384///
385/// The cores are validated as in [`compress_chain_lu_mut`]. The forward sweep
386/// then visits each bond from left to right, unfolds the left core to a
387/// `(left_bond * site) x bond` matrix and calls `factorize(state, matrix,
388/// true)`. The backward sweep visits the bonds from right to left, unfolds the
389/// right core to a `bond x (site * right_bond)` matrix and calls
390/// `factorize(state, matrix, false)`. `factorize` returns factors `(L, R)`
391/// with `L * R` approximating the matrix; their shared dimension becomes the
392/// new bond. `multiply` absorbs the factor that does not replace the visited
393/// core into its neighbor. `state` is passed to both closures, for example a
394/// backend session.
395///
396/// Unfoldings index rows as `l * site + s` and columns as `s * right + r`.
397///
398/// # Errors
399///
400/// Returns invalid core shapes, factor shapes that disagree with the
401/// unfolding, multiplication and allocation failures as
402/// [`ChainCompressionError`] converted into `C::Error`, and passes through
403/// any error from `factorize`. If a step fails, bonds updated before the
404/// failure keep their new values.
405///
406/// # Examples
407///
408/// ```
409/// use tensor4all_core::{compress_chain_cores_mut, rrlu_mut, ChainCompressionError, ColMajorArray};
410/// use tensor4all_tensorbackend::mat_mul;
411/// // A rank-one chain stored with bond dimension two.
412/// let mut cores = [
413///     ColMajorArray::new(vec![1.0_f64, 1.0, 2.0, 2.0], vec![1, 2, 2])?,
414///     ColMajorArray::new(vec![1.0, 1.0, 3.0, 3.0], vec![2, 2, 1])?,
415/// ];
416/// compress_chain_cores_mut(
417///     &mut cores,
418///     &mut (),
419///     |(), mut matrix, _| {
420///         let lu = rrlu_mut(&mut matrix, None).map_err(ChainCompressionError::from)?;
421///         Ok((lu.left(true), lu.right(true)))
422///     },
423///     |(), left, right| mat_mul(left, right),
424/// )?;
425/// assert_eq!(cores[0].shape(), &[1, 2, 1]);
426/// assert_eq!(cores[1].shape(), &[1, 2, 1]);
427/// # Ok::<(), Box<dyn std::error::Error>>(())
428/// ```
429pub fn compress_chain_cores_mut<T: Copy, C: ChainCore<T>, S>(
430    cores: &mut [C],
431    state: &mut S,
432    mut factorize: impl FnMut(&mut S, Matrix<T>, bool) -> Result<(Matrix<T>, Matrix<T>), C::Error>,
433    mut multiply: impl FnMut(&mut S, &Matrix<T>, &Matrix<T>) -> Result<Matrix<T>, MatrixMulError>,
434) -> Result<(), C::Error> {
435    let n = cores.len();
436    let mut first_left = 1;
437    let mut previous_right = 1;
438    for (site, core) in cores.iter().enumerate() {
439        let [left, _, right] = core_shape(core.view()?, site)?;
440        if site == 0 {
441            first_left = left;
442        } else {
443            check_bond(site - 1, previous_right, left)?;
444        }
445        previous_right = right;
446    }
447    if first_left != 1 {
448        return Err(ChainCompressionError::LeftBoundary.into());
449    }
450    if previous_right != 1 {
451        return Err(ChainCompressionError::RightBoundary.into());
452    }
453
454    let bonds = 0..n.saturating_sub(1);
455    let forward = bonds.clone().map(|site| (site, true));
456    let backward = bonds.rev().map(|site| (site, false));
457    for (site, left_orthogonal) in forward.chain(backward) {
458        let left = cores[site].view()?;
459        let right = cores[site + 1].view()?;
460        let [l, left_site, bond] = core_shape(left, site)?;
461        let [next_bond, right_site, r] = core_shape(right, site + 1)?;
462        check_bond(site, bond, next_bond)?;
463        let rows = fused_dim(l, left_site, [l, left_site, bond])?;
464        let cols = fused_dim(right_site, r, [bond, right_site, r])?;
465        let left_matrix = || -> Result<_, ChainCompressionError> {
466            Ok(Matrix::try_from_col_major_vec(
467                rows,
468                bond,
469                swap_blocks(left.data(), 1, l, left_site)?,
470            )?)
471        };
472        let right_matrix = || -> Result<_, ChainCompressionError> {
473            Ok(Matrix::try_from_col_major_vec(
474                bond,
475                cols,
476                swap_blocks(right.data(), bond, right_site, r)?,
477            )?)
478        };
479        let matrix = if left_orthogonal {
480            left_matrix()?
481        } else {
482            right_matrix()?
483        };
484        let (matrix_rows, matrix_cols) = (matrix.nrows(), matrix.ncols());
485        let (left_factor, right_factor) = factorize(state, matrix, left_orthogonal)?;
486        let rank = left_factor.ncols();
487        matrix_data(&left_factor, matrix_rows, rank)?;
488        matrix_data(&right_factor, rank, matrix_cols)?;
489
490        let (left, right) = if left_orthogonal {
491            let right = multiply(state, &right_factor, &right_matrix()?).map_err(|source| {
492                ChainCompressionError::Multiply {
493                    direction: "left-to-right",
494                    source,
495                }
496            })?;
497            (left_factor, right)
498        } else {
499            let left = multiply(state, &left_matrix()?, &left_factor).map_err(|source| {
500                ChainCompressionError::Multiply {
501                    direction: "right-to-left",
502                    source,
503                }
504            })?;
505            (left, right_factor)
506        };
507        let left = C::from_col_major(
508            [l, left_site, rank],
509            swap_blocks(matrix_data(&left, rows, rank)?, 1, left_site, l)?,
510        )?;
511        let right = C::from_col_major(
512            [rank, right_site, r],
513            swap_blocks(matrix_data(&right, rank, cols)?, rank, r, right_site)?,
514        )?;
515        cores[site] = left;
516        cores[site + 1] = right;
517    }
518    Ok(())
519}
520
521fn core_shape<T>(
522    core: ColMajorArrayRef<'_, T>,
523    site: usize,
524) -> Result<[usize; 3], ChainCompressionError> {
525    core.shape()
526        .try_into()
527        .map_err(|_| ChainCompressionError::CoreRank {
528            site,
529            rank: core.ndim(),
530        })
531}
532
533fn check_bond(site: usize, right: usize, next_left: usize) -> Result<(), ChainCompressionError> {
534    if right != next_left {
535        return Err(ChainCompressionError::BondMismatch {
536            site,
537            right,
538            next_left,
539        });
540    }
541    Ok(())
542}
543
544fn fused_dim(a: usize, b: usize, shape: [usize; 3]) -> Result<usize, ChainCompressionError> {
545    a.checked_mul(b)
546        .ok_or(ChainCompressionError::ShapeOverflow(shape))
547}
548
549fn matrix_data<T>(
550    matrix: &Matrix<T>,
551    rows: usize,
552    cols: usize,
553) -> Result<&[T], ChainCompressionError> {
554    if matrix.nrows() != rows || matrix.ncols() != cols {
555        return Err(MatrixCIError::DimensionMismatch {
556            expected_rows: rows,
557            expected_cols: cols,
558            actual_rows: matrix.nrows(),
559            actual_cols: matrix.ncols(),
560        }
561        .into());
562    }
563    Ok(matrix.as_col_major_slice())
564}
565
566// The fused row index is l * site + s and the fused column index is
567// s * right + r. Exact ties in pivot magnitude are broken in this order, so it
568// determines which pivots the sweeps retain; core storage is column-major, so
569// each unfolding swaps the two fused axes. `data` is a sequence of blocks of
570// `inner * first * second` elements; within each block, chunks of `inner`
571// elements are reordered from `first`-fastest to `second`-fastest.
572fn swap_blocks<T: Copy>(
573    data: &[T],
574    inner: usize,
575    first: usize,
576    second: usize,
577) -> Result<Vec<T>, ChainCompressionError> {
578    let mut result = Vec::new();
579    result.try_reserve_exact(data.len())?;
580    if data.is_empty() {
581        return Ok(result);
582    }
583    let block = inner
584        .checked_mul(first)
585        .and_then(|stride| stride.checked_mul(second))
586        .filter(|&block| block != 0 && data.len().is_multiple_of(block))
587        .ok_or(ChainCompressionError::BufferLength {
588            len: data.len(),
589            block: inner.saturating_mul(first).saturating_mul(second),
590        })?;
591    if first == 1 || second == 1 {
592        result.extend_from_slice(data);
593        return Ok(result);
594    }
595    let stride = inner * first;
596    for chunk in data.chunks_exact(block) {
597        for k in 0..first {
598            for j in 0..second {
599                let start = k * inner + j * stride;
600                result.extend_from_slice(&chunk[start..start + inner]);
601            }
602        }
603    }
604    Ok(result)
605}
606
607#[cfg(test)]
608mod tests;