1#![doc(hidden)]
2
3pub type Result<T> = tenferro_tensor::Result<T>;
15use tenferro_tensor::CompareDir;
16pub use tenferro_tensor::{
17 CacheStats, DType, Error, ErrorKind, TensorRank, TensorScalar, TypedTensor, TypedTensorView,
18};
19
20pub mod buffer_pool;
21pub use buffer_pool::{BufferPool, PoolScalar};
22mod pooled_uninit_output;
23pub use pooled_uninit_output::PooledUninitOutput;
24
25use num_complex::{Complex32, Complex64};
26use std::mem::MaybeUninit;
27use std::ptr::NonNull;
28use strided_basic::{
29 ErasedRawStridedMut, ErasedRawStridedPtr, ErasedRawStridedRef, ErasedRawStridedUninitMut,
30 KernelDType, StridedView,
31};
32
33const _: () = assert!(tenferro_tensor::INLINE_RANK == strided_basic::INLINE_RANK);
37
38#[doc(hidden)]
40pub fn cpu_backend_buffer_error(op: &'static str) -> Error {
41 Error::runtime_state(
42 op,
43 "CPU backend received backend buffer; download to host before CPU execution",
44 )
45}
46
47#[doc(hidden)]
53pub fn cpu_non_host_operand_error(op: &'static str, role: &'static str) -> Error {
54 Error::runtime_state(
55 op,
56 format!(
57 "CPU backend requires {role} host placement for {op}: the payload is a backend buffer; download to host before CPU execution"
58 ),
59 )
60}
61
62#[derive(Debug, thiserror::Error)]
63#[doc(hidden)]
64pub enum CpuNumericalError {
65 #[error("{op} detected division by zero for dtype {dtype:?}")]
66 DivisionByZero { op: &'static str, dtype: DType },
67}
68
69#[doc(hidden)]
70pub fn cpu_division_by_zero(op: &'static str, dtype: DType) -> Error {
71 Error::extension(
72 op,
73 "cpu",
74 ErrorKind::NumericalFailure,
75 CpuNumericalError::DivisionByZero { op, dtype },
76 )
77}
78
79#[doc(hidden)]
80pub trait ConjElem {
81 fn conj_elem(self) -> Self;
90}
91
92impl ConjElem for f32 {
93 fn conj_elem(self) -> Self {
94 self
95 }
96}
97impl ConjElem for f64 {
98 fn conj_elem(self) -> Self {
99 self
100 }
101}
102impl ConjElem for Complex32 {
103 fn conj_elem(self) -> Self {
104 self.conj()
105 }
106}
107impl ConjElem for Complex64 {
108 fn conj_elem(self) -> Self {
109 self.conj()
110 }
111}
112
113#[doc(hidden)]
114pub fn is_complex_dtype(dtype: DType) -> bool {
115 matches!(dtype, DType::C32 | DType::C64)
116}
117
118#[doc(hidden)]
119pub fn ordered_complex_error(op: &'static str) -> Error {
120 Error::unsupported(
121 op,
122 "complex tensors do not have a total order; compute abs/norm explicitly before ordered operations",
123 )
124}
125
126#[doc(hidden)]
127pub fn reject_complex_ordered_dtypes(op: &'static str, dtypes: &[DType]) -> Result<()> {
128 if dtypes.iter().copied().any(is_complex_dtype) {
129 return Err(ordered_complex_error(op));
130 }
131 Ok(())
132}
133
134#[doc(hidden)]
135pub fn reject_complex_unsupported_compare_dtypes(dir: &CompareDir, dtypes: &[DType]) -> Result<()> {
136 if *dir != CompareDir::Eq {
137 reject_complex_ordered_dtypes("compare", dtypes)?;
138 }
139 Ok(())
140}
141
142#[doc(hidden)]
143pub fn typed_host_data<'a, T: TensorScalar>(
144 op: &'static str,
145 tensor: &'a TypedTensor<T>,
146) -> Result<&'a [T]> {
147 if let Some(buffer) = tensor.backend_buffer() {
148 let expected =
149 tenferro_tensor::validate::checked_shape_product(op, "shape", tensor.shape())?;
150 return shared_read_slice(op, expected, buffer.shared_host_slice());
151 }
152 tensor.host_data()
153}
154
155#[doc(hidden)]
163pub fn shared_read_slice<'a, T>(
164 op: &'static str,
165 expected: usize,
166 slice: std::result::Result<&'a [T], tenferro_tensor::HostAccessError>,
167) -> Result<&'a [T]> {
168 let slice = admit_shared_slice(op, slice)?;
169 slice
170 .get(..expected)
171 .ok_or_else(|| Error::validation(op, tenferro_tensor::ValidationError::ViewOutOfBounds))
172}
173
174fn admit_shared_slice<'a, T>(
176 op: &'static str,
177 slice: std::result::Result<&'a [T], tenferro_tensor::HostAccessError>,
178) -> Result<&'a [T]> {
179 match slice {
180 Ok(slice) => Ok(slice),
181 Err(tenferro_tensor::HostAccessError::Unsupported { .. }) => {
182 Err(cpu_backend_buffer_error(op))
183 }
184 Err(error) => Err(Error::host_access(op, error)),
185 }
186}
187
188#[doc(hidden)]
189pub fn typed_view<'a, T: Copy + TensorScalar>(
190 op: &'static str,
191 tensor: &'a TypedTensor<T>,
192) -> Result<StridedView<'a, T>> {
193 let data = typed_host_data(op, tensor)?;
194 StridedView::new(data, tensor.shape(), tensor.strides(), 0)
197 .map_err(|err| Error::backend_source(op, err))
198}
199
200#[doc(hidden)]
206pub fn shared_view_read_slice<'a, T>(
207 op: &'static str,
208 buffer: &'a dyn tenferro_tensor::BackendStorage<T>,
209) -> Result<&'a [T]>
210where
211 T: 'static,
212{
213 admit_shared_slice(op, buffer.shared_host_slice())
214}
215
216#[doc(hidden)]
217pub fn typed_view_from_view<'a, T: Copy + 'static, R: TensorRank>(
218 op: &'static str,
219 view: &TypedTensorView<'a, T, R>,
220) -> Result<StridedView<'a, T>> {
221 let data = match view.shared_backend_storage() {
222 Some(buffer) => shared_view_read_slice(op, buffer)?,
223 None => view.host_storage()?,
224 };
225 StridedView::new(data, view.shape(), view.strides(), view.offset())
226 .map_err(|err| Error::backend_source(op, err))
227}
228
229#[doc(hidden)]
235pub unsafe fn erased_raw_strided_ref<'a>(
236 dtype: KernelDType,
237 data: &'a [u8],
238 dims: &'a [usize],
239 strides: &'a [isize],
240 offset: isize,
241) -> strided_basic::Result<ErasedRawStridedRef<'a>> {
242 let data_ptr = NonNull::new(data.as_ptr().cast_mut()).unwrap_or_else(NonNull::dangling);
243 unsafe {
245 ErasedRawStridedRef::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
246 }
247}
248
249#[doc(hidden)]
255pub unsafe fn erased_raw_strided_ptr<'a>(
256 dtype: KernelDType,
257 data: &'a [u8],
258 dims: &'a [usize],
259 strides: &'a [isize],
260 offset: isize,
261) -> strided_basic::Result<ErasedRawStridedPtr<'a>> {
262 let data_ptr = NonNull::new(data.as_ptr().cast_mut()).unwrap_or_else(NonNull::dangling);
263 unsafe {
265 ErasedRawStridedPtr::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
266 }
267}
268
269#[doc(hidden)]
275pub unsafe fn erased_raw_strided_mut<'a>(
276 dtype: KernelDType,
277 data: &'a mut [u8],
278 dims: &'a [usize],
279 strides: &'a [isize],
280 offset: isize,
281) -> strided_basic::Result<ErasedRawStridedMut<'a>> {
282 let data_ptr = NonNull::new(data.as_mut_ptr()).unwrap_or_else(NonNull::dangling);
283 unsafe {
285 ErasedRawStridedMut::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
286 }
287}
288
289#[doc(hidden)]
296pub unsafe fn erased_raw_strided_uninit_mut<'a>(
297 dtype: KernelDType,
298 data: &'a mut [MaybeUninit<u8>],
299 dims: &'a [usize],
300 strides: &'a [isize],
301 offset: isize,
302) -> strided_basic::Result<ErasedRawStridedUninitMut<'a>> {
303 let data_ptr = NonNull::new(data.as_mut_ptr().cast::<u8>()).unwrap_or_else(NonNull::dangling);
304 unsafe {
306 ErasedRawStridedUninitMut::from_raw_parts(
307 dtype,
308 data_ptr,
309 data.len(),
310 dims,
311 strides,
312 offset,
313 )
314 }
315}