Skip to main content

tenferro_cpu_basic/
lib.rs

1#![doc(hidden)]
2
3//! Shared host CPU resources and low-level adapters.
4
5/// Internal result alias for shared CPU adapters.
6///
7/// # Examples
8///
9/// ```rust
10/// use tenferro_cpu_basic::Result;
11/// let result: Result<()> = Ok(());
12/// assert!(result.is_ok());
13/// ```
14pub type Result<T> = tenferro_tensor::Result<T>;
15use tenferro_tensor::CompareDir;
16pub use tenferro_tensor::{
17    CacheStats, DType, Error, ErrorKind, TensorRank, TensorScalar, TypedTensor, TypedTensorView,
18};
19
20pub mod buffer_pool;
21pub use buffer_pool::{BufferPool, PoolScalar};
22mod pooled_uninit_output;
23pub use pooled_uninit_output::PooledUninitOutput;
24
25use num_complex::{Complex32, Complex64};
26use std::mem::MaybeUninit;
27use std::ptr::NonNull;
28use strided_basic::{
29    ErasedRawStridedMut, ErasedRawStridedPtr, ErasedRawStridedRef, ErasedRawStridedUninitMut,
30    KernelDType, StridedView,
31};
32
33// Both sides own the same inline-capacity contract: tenferro's `ShapeVec`/
34// `StrideVec` and strided-view's metadata must spill at the same rank, or a
35// view would allocate where the metadata budget assumes it cannot.
36const _: () = assert!(tenferro_tensor::INLINE_RANK == strided_basic::INLINE_RANK);
37
38/// Error returned when a CPU-only operation receives backend storage.
39#[doc(hidden)]
40pub fn cpu_backend_buffer_error(op: &'static str) -> Error {
41    Error::runtime_state(
42        op,
43        "CPU backend received backend buffer; download to host before CPU execution",
44    )
45}
46
47/// Error returned when a CPU operand's payload is not host memory.
48///
49/// `role` names the operand (`lhs`, `destination`, ...). The verdict is the
50/// payload predicate, so this error never accuses a host tensor of being a
51/// device buffer because a label said so.
52#[doc(hidden)]
53pub fn cpu_non_host_operand_error(op: &'static str, role: &'static str) -> Error {
54    Error::runtime_state(
55        op,
56        format!(
57            "CPU backend requires {role} host placement for {op}: the payload is a backend buffer; download to host before CPU execution"
58        ),
59    )
60}
61
62#[derive(Debug, thiserror::Error)]
63#[doc(hidden)]
64pub enum CpuNumericalError {
65    #[error("{op} detected division by zero for dtype {dtype:?}")]
66    DivisionByZero { op: &'static str, dtype: DType },
67}
68
69#[doc(hidden)]
70pub fn cpu_division_by_zero(op: &'static str, dtype: DType) -> Error {
71    Error::extension(
72        op,
73        "cpu",
74        ErrorKind::NumericalFailure,
75        CpuNumericalError::DivisionByZero { op, dtype },
76    )
77}
78
79#[doc(hidden)]
80pub trait ConjElem {
81    /// Apply complex conjugation, or leave a real scalar unchanged.
82    ///
83    /// # Examples
84    ///
85    /// ```rust
86    /// use tenferro_cpu_basic::ConjElem;
87    /// assert_eq!(ConjElem::conj_elem(2.0_f64), 2.0);
88    /// ```
89    fn conj_elem(self) -> Self;
90}
91
92impl ConjElem for f32 {
93    fn conj_elem(self) -> Self {
94        self
95    }
96}
97impl ConjElem for f64 {
98    fn conj_elem(self) -> Self {
99        self
100    }
101}
102impl ConjElem for Complex32 {
103    fn conj_elem(self) -> Self {
104        self.conj()
105    }
106}
107impl ConjElem for Complex64 {
108    fn conj_elem(self) -> Self {
109        self.conj()
110    }
111}
112
113#[doc(hidden)]
114pub fn is_complex_dtype(dtype: DType) -> bool {
115    matches!(dtype, DType::C32 | DType::C64)
116}
117
118#[doc(hidden)]
119pub fn ordered_complex_error(op: &'static str) -> Error {
120    Error::unsupported(
121        op,
122        "complex tensors do not have a total order; compute abs/norm explicitly before ordered operations",
123    )
124}
125
126#[doc(hidden)]
127pub fn reject_complex_ordered_dtypes(op: &'static str, dtypes: &[DType]) -> Result<()> {
128    if dtypes.iter().copied().any(is_complex_dtype) {
129        return Err(ordered_complex_error(op));
130    }
131    Ok(())
132}
133
134#[doc(hidden)]
135pub fn reject_complex_unsupported_compare_dtypes(dir: &CompareDir, dtypes: &[DType]) -> Result<()> {
136    if *dir != CompareDir::Eq {
137        reject_complex_ordered_dtypes("compare", dtypes)?;
138    }
139    Ok(())
140}
141
142#[doc(hidden)]
143pub fn typed_host_data<'a, T: TensorScalar>(
144    op: &'static str,
145    tensor: &'a TypedTensor<T>,
146) -> Result<&'a [T]> {
147    if let Some(buffer) = tensor.backend_buffer() {
148        let expected =
149            tenferro_tensor::validate::checked_shape_product(op, "shape", tensor.shape())?;
150        return shared_read_slice(op, expected, buffer.shared_host_slice());
151    }
152    tensor.host_data()
153}
154
155/// Admit one backend allocation the CPU can read, or keep the CPU refusal.
156///
157/// A shared allocation domain hands out allocations the CPU can read at the
158/// same address as the device, so the provider's borrow (`shared_host_slice`,
159/// which waits for pending device writes) is the admission. Every other
160/// backend payload keeps the historical "download to host" refusal, and a
161/// provider's own typed failure survives as [`Error::host_access`].
162#[doc(hidden)]
163pub fn shared_read_slice<'a, T>(
164    op: &'static str,
165    expected: usize,
166    slice: std::result::Result<&'a [T], tenferro_tensor::HostAccessError>,
167) -> Result<&'a [T]> {
168    let slice = admit_shared_slice(op, slice)?;
169    slice
170        .get(..expected)
171        .ok_or_else(|| Error::validation(op, tenferro_tensor::ValidationError::ViewOutOfBounds))
172}
173
174/// Turn the provider's borrow into a slice, keeping every non-admission a typed refusal.
175fn admit_shared_slice<'a, T>(
176    op: &'static str,
177    slice: std::result::Result<&'a [T], tenferro_tensor::HostAccessError>,
178) -> Result<&'a [T]> {
179    match slice {
180        Ok(slice) => Ok(slice),
181        Err(tenferro_tensor::HostAccessError::Unsupported { .. }) => {
182            Err(cpu_backend_buffer_error(op))
183        }
184        Err(error) => Err(Error::host_access(op, error)),
185    }
186}
187
188#[doc(hidden)]
189pub fn typed_view<'a, T: Copy + TensorScalar>(
190    op: &'static str,
191    tensor: &'a TypedTensor<T>,
192) -> Result<StridedView<'a, T>> {
193    let data = typed_host_data(op, tensor)?;
194    // The tensor caches its compact layout, so the view borrows both metadata
195    // vectors instead of allocating a pair per call.
196    StridedView::new(data, tensor.shape(), tensor.strides(), 0)
197        .map_err(|err| Error::backend_source(op, err))
198}
199
200/// Admit one borrowed view the CPU can read, keeping its strides and offset.
201///
202/// The allocation, not the view, bounds the borrow: a sliced or strided view
203/// reaches outside its own `n_elements`, so the extent check belongs to the
204/// caller's `StridedView::new`, as it already does for a host view.
205#[doc(hidden)]
206pub fn shared_view_read_slice<'a, T>(
207    op: &'static str,
208    buffer: &'a dyn tenferro_tensor::BackendStorage<T>,
209) -> Result<&'a [T]>
210where
211    T: 'static,
212{
213    admit_shared_slice(op, buffer.shared_host_slice())
214}
215
216#[doc(hidden)]
217pub fn typed_view_from_view<'a, T: Copy + 'static, R: TensorRank>(
218    op: &'static str,
219    view: &TypedTensorView<'a, T, R>,
220) -> Result<StridedView<'a, T>> {
221    let data = match view.shared_backend_storage() {
222        Some(buffer) => shared_view_read_slice(op, buffer)?,
223        None => view.host_storage()?,
224    };
225    StridedView::new(data, view.shape(), view.strides(), view.offset())
226        .map_err(|err| Error::backend_source(op, err))
227}
228
229/// Construct an erased read-only strided view over initialized typed storage.
230///
231/// # Safety
232/// `dtype` must match the aligned initialized storage. All reachable offsets
233/// must be in bounds and the storage/metadata must remain valid for `'a`.
234#[doc(hidden)]
235pub unsafe fn erased_raw_strided_ref<'a>(
236    dtype: KernelDType,
237    data: &'a [u8],
238    dims: &'a [usize],
239    strides: &'a [isize],
240    offset: isize,
241) -> strided_basic::Result<ErasedRawStridedRef<'a>> {
242    let data_ptr = NonNull::new(data.as_ptr().cast_mut()).unwrap_or_else(NonNull::dangling);
243    // SAFETY: the caller supplies the documented dtype, bounds, alignment and lifetime invariants.
244    unsafe {
245        ErasedRawStridedRef::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
246    }
247}
248
249/// Construct an erased pointer view over initialized storage.
250///
251/// # Safety
252/// The caller must uphold the dtype, alignment, bounds, lifetime and
253/// initialization requirements of the returned descriptor.
254#[doc(hidden)]
255pub unsafe fn erased_raw_strided_ptr<'a>(
256    dtype: KernelDType,
257    data: &'a [u8],
258    dims: &'a [usize],
259    strides: &'a [isize],
260    offset: isize,
261) -> strided_basic::Result<ErasedRawStridedPtr<'a>> {
262    let data_ptr = NonNull::new(data.as_ptr().cast_mut()).unwrap_or_else(NonNull::dangling);
263    // SAFETY: the caller supplies the documented dtype, bounds, alignment and lifetime invariants.
264    unsafe {
265        ErasedRawStridedPtr::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
266    }
267}
268
269/// Construct an erased writable view over exclusively owned initialized storage.
270///
271/// # Safety
272/// The caller must uphold the dtype, alignment, bounds, lifetime and exclusive
273/// access requirements of the returned descriptor.
274#[doc(hidden)]
275pub unsafe fn erased_raw_strided_mut<'a>(
276    dtype: KernelDType,
277    data: &'a mut [u8],
278    dims: &'a [usize],
279    strides: &'a [isize],
280    offset: isize,
281) -> strided_basic::Result<ErasedRawStridedMut<'a>> {
282    let data_ptr = NonNull::new(data.as_mut_ptr()).unwrap_or_else(NonNull::dangling);
283    // SAFETY: the caller supplies the documented dtype, bounds, alignment and exclusive-lifetime invariants.
284    unsafe {
285        ErasedRawStridedMut::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
286    }
287}
288
289/// Construct an erased writable strided view over exclusively owned uninitialized storage.
290///
291/// # Safety
292/// `dtype` must match the aligned storage; reachable offsets must be in bounds;
293/// the borrow must remain exclusive; and storage must stay uninitialized until
294/// every reachable element has been written.
295#[doc(hidden)]
296pub unsafe fn erased_raw_strided_uninit_mut<'a>(
297    dtype: KernelDType,
298    data: &'a mut [MaybeUninit<u8>],
299    dims: &'a [usize],
300    strides: &'a [isize],
301    offset: isize,
302) -> strided_basic::Result<ErasedRawStridedUninitMut<'a>> {
303    let data_ptr = NonNull::new(data.as_mut_ptr().cast::<u8>()).unwrap_or_else(NonNull::dangling);
304    // SAFETY: the caller supplies the documented dtype, bounds, alignment and exclusive-lifetime invariants.
305    unsafe {
306        ErasedRawStridedUninitMut::from_raw_parts(
307            dtype,
308            data_ptr,
309            data.len(),
310            dims,
311            strides,
312            offset,
313        )
314    }
315}