1use cpueinsum::Exec;
14use tenferro_tensor::backend::Engine;
15use tenferro_tensor::{
16 CompareDir, DType, DotGeneralConfig, ElementwiseReadOp, GatherConfig, NativeSessionRef,
17 PadConfig, ScatterConfig, SliceConfig, TensorRead, TensorWrite,
18};
19use tenferro_tensor::{
20 SessionCachedDot, TensorAnalytic, TensorBuffer, TensorDeviceTransfer, TensorDot,
21 TensorElementwise, TensorFusion, TensorIndexing, TensorReduction, TensorStructural,
22};
23
24use crate::buffer_pool::BufferPool;
25use crate::CpuEngine;
26
27struct BorrowedCpuMarker;
29
30pub(crate) struct BorrowedCpuEngine<'a, 'e> {
36 exec: &'a mut Exec<'e>,
37 pool: &'a mut BufferPool,
38}
39
40impl<'a, 'e> BorrowedCpuEngine<'a, 'e> {
41 pub(crate) fn new(exec: &'a mut Exec<'e>, pool: &'a mut BufferPool) -> Self {
43 Self { exec, pool }
44 }
45}
46
47impl std::fmt::Debug for BorrowedCpuEngine<'_, '_> {
48 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
49 f.debug_struct("BorrowedCpuEngine")
50 .field("budget", &self.exec.budget())
51 .field("pool", &self.pool.stats())
52 .finish()
53 }
54}
55
56#[doc(hidden)]
87pub fn with_borrowed_cpu_engine<B, R>(
88 engine: &mut B,
89 f: impl FnOnce(&mut Exec<'_>, &mut BufferPool) -> R,
90) -> Option<R>
91where
92 B: Engine + ?Sized,
93{
94 let data = engine
95 .native_engine()?
96 .into_marked_ptr::<BorrowedCpuMarker>()?;
97 let this = unsafe { data.cast::<BorrowedCpuEngine<'_, '_>>().as_mut() };
107 Some(f(&mut *this.exec, &mut *this.pool))
108}
109
110fn refused<T>(op: &'static str) -> tenferro_tensor::Result<T> {
111 Err(tenferro_tensor::Error::unsupported(
112 op,
113 "a lane of CpuEngine::with_lanes runs only the linalg factorizations \
114 svd/svdvals/qr/eigh/eigvalsh/eigvals; this operation needs a CpuEngine",
115 ))
116}
117impl SessionCachedDot for BorrowedCpuEngine<'_, '_> {}
118impl TensorFusion for BorrowedCpuEngine<'_, '_> {}
119impl TensorBuffer for BorrowedCpuEngine<'_, '_> {}
120
121impl Engine for BorrowedCpuEngine<'_, '_> {
122 fn native_engine(&mut self) -> Option<NativeSessionRef<'_>> {
123 Some(unsafe { NativeSessionRef::new::<BorrowedCpuMarker, _>(self) })
126 }
127}
128
129impl TensorStructural for BorrowedCpuEngine<'_, '_> {
130 fn transpose_read(
131 &mut self,
132 _a0: TensorRead<'_>,
133 _a1: &[usize],
134 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
135 refused("transpose_read")
136 }
137
138 fn reshape_read(
139 &mut self,
140 _a0: TensorRead<'_>,
141 _a1: &[usize],
142 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
143 refused("reshape_read")
144 }
145
146 fn broadcast_in_dim_read(
147 &mut self,
148 _a0: TensorRead<'_>,
149 _a1: &[usize],
150 _a2: &[usize],
151 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
152 refused("broadcast_in_dim_read")
153 }
154
155 fn cast(
156 &mut self,
157 _a0: &tenferro_tensor::Tensor,
158 _a1: DType,
159 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
160 refused("cast")
161 }
162
163 fn extract_diagonal(
164 &mut self,
165 _a0: &tenferro_tensor::Tensor,
166 _a1: usize,
167 _a2: usize,
168 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
169 refused("extract_diagonal")
170 }
171
172 fn embed_diagonal(
173 &mut self,
174 _a0: &tenferro_tensor::Tensor,
175 _a1: usize,
176 _a2: usize,
177 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
178 refused("embed_diagonal")
179 }
180
181 fn tril(
182 &mut self,
183 _a0: &tenferro_tensor::Tensor,
184 _a1: i64,
185 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
186 refused("tril")
187 }
188
189 fn triu(
190 &mut self,
191 _a0: &tenferro_tensor::Tensor,
192 _a1: i64,
193 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
194 refused("triu")
195 }
196}
197
198impl TensorElementwise for BorrowedCpuEngine<'_, '_> {
199 fn elementwise_read_into(
200 &mut self,
201 _a0: ElementwiseReadOp,
202 _a1: &[TensorRead<'_>],
203 _a2: TensorWrite<'_>,
204 ) -> Result<(), tenferro_tensor::Error> {
205 refused("elementwise_read_into")
206 }
207
208 fn add_read(
209 &mut self,
210 _a0: TensorRead<'_>,
211 _a1: TensorRead<'_>,
212 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
213 refused("add_read")
214 }
215
216 fn sub_read(
217 &mut self,
218 _a0: TensorRead<'_>,
219 _a1: TensorRead<'_>,
220 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
221 refused("sub_read")
222 }
223
224 fn mul_read(
225 &mut self,
226 _a0: TensorRead<'_>,
227 _a1: TensorRead<'_>,
228 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
229 refused("mul_read")
230 }
231
232 fn neg_read(
233 &mut self,
234 _a0: TensorRead<'_>,
235 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
236 refused("neg_read")
237 }
238
239 fn conj_read(
240 &mut self,
241 _a0: TensorRead<'_>,
242 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
243 refused("conj_read")
244 }
245
246 fn div_read(
247 &mut self,
248 _a0: TensorRead<'_>,
249 _a1: TensorRead<'_>,
250 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
251 refused("div_read")
252 }
253
254 fn abs_read(
255 &mut self,
256 _a0: TensorRead<'_>,
257 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
258 refused("abs_read")
259 }
260
261 fn sign_read(
262 &mut self,
263 _a0: TensorRead<'_>,
264 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
265 refused("sign_read")
266 }
267
268 fn maximum_read(
269 &mut self,
270 _a0: TensorRead<'_>,
271 _a1: TensorRead<'_>,
272 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
273 refused("maximum_read")
274 }
275
276 fn minimum_read(
277 &mut self,
278 _a0: TensorRead<'_>,
279 _a1: TensorRead<'_>,
280 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
281 refused("minimum_read")
282 }
283
284 fn compare_read(
285 &mut self,
286 _a0: TensorRead<'_>,
287 _a1: TensorRead<'_>,
288 _a2: &CompareDir,
289 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
290 refused("compare_read")
291 }
292
293 fn select_read(
294 &mut self,
295 _a0: TensorRead<'_>,
296 _a1: TensorRead<'_>,
297 _a2: TensorRead<'_>,
298 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
299 refused("select_read")
300 }
301
302 fn clamp_read(
303 &mut self,
304 _a0: TensorRead<'_>,
305 _a1: TensorRead<'_>,
306 _a2: TensorRead<'_>,
307 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
308 refused("clamp_read")
309 }
310}
311
312impl TensorAnalytic for BorrowedCpuEngine<'_, '_> {
313 fn exp_read(
314 &mut self,
315 _a0: TensorRead<'_>,
316 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
317 refused("exp_read")
318 }
319
320 fn log_read(
321 &mut self,
322 _a0: TensorRead<'_>,
323 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
324 refused("log_read")
325 }
326
327 fn sin_read(
328 &mut self,
329 _a0: TensorRead<'_>,
330 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
331 refused("sin_read")
332 }
333
334 fn cos_read(
335 &mut self,
336 _a0: TensorRead<'_>,
337 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
338 refused("cos_read")
339 }
340
341 fn tanh_read(
342 &mut self,
343 _a0: TensorRead<'_>,
344 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
345 refused("tanh_read")
346 }
347
348 fn sqrt_read(
349 &mut self,
350 _a0: TensorRead<'_>,
351 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
352 refused("sqrt_read")
353 }
354
355 fn rsqrt_read(
356 &mut self,
357 _a0: TensorRead<'_>,
358 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
359 refused("rsqrt_read")
360 }
361
362 fn pow_read(
363 &mut self,
364 _a0: TensorRead<'_>,
365 _a1: TensorRead<'_>,
366 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
367 refused("pow_read")
368 }
369
370 fn expm1_read(
371 &mut self,
372 _a0: TensorRead<'_>,
373 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
374 refused("expm1_read")
375 }
376
377 fn log1p_read(
378 &mut self,
379 _a0: TensorRead<'_>,
380 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
381 refused("log1p_read")
382 }
383
384 fn erf_read(
385 &mut self,
386 _a0: TensorRead<'_>,
387 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
388 refused("erf_read")
389 }
390}
391
392impl TensorReduction for BorrowedCpuEngine<'_, '_> {
393 fn reduce_sum_read(
394 &mut self,
395 _a0: TensorRead<'_>,
396 _a1: &[usize],
397 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
398 refused("reduce_sum_read")
399 }
400
401 fn reduce_prod_read(
402 &mut self,
403 _a0: TensorRead<'_>,
404 _a1: &[usize],
405 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
406 refused("reduce_prod_read")
407 }
408
409 fn reduce_max_read(
410 &mut self,
411 _a0: TensorRead<'_>,
412 _a1: &[usize],
413 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
414 refused("reduce_max_read")
415 }
416
417 fn reduce_min_read(
418 &mut self,
419 _a0: TensorRead<'_>,
420 _a1: &[usize],
421 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
422 refused("reduce_min_read")
423 }
424}
425
426impl TensorIndexing for BorrowedCpuEngine<'_, '_> {
427 fn gather(
428 &mut self,
429 _a0: &tenferro_tensor::Tensor,
430 _a1: &tenferro_tensor::Tensor,
431 _a2: &GatherConfig,
432 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
433 refused("gather")
434 }
435
436 fn scatter(
437 &mut self,
438 _a0: &tenferro_tensor::Tensor,
439 _a1: &tenferro_tensor::Tensor,
440 _a2: &tenferro_tensor::Tensor,
441 _a3: &ScatterConfig,
442 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
443 refused("scatter")
444 }
445
446 fn slice(
447 &mut self,
448 _a0: &tenferro_tensor::Tensor,
449 _a1: &SliceConfig,
450 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
451 refused("slice")
452 }
453
454 fn dynamic_slice(
455 &mut self,
456 _a0: &tenferro_tensor::Tensor,
457 _a1: &tenferro_tensor::Tensor,
458 _a2: &[usize],
459 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
460 refused("dynamic_slice")
461 }
462
463 fn dynamic_update_slice(
464 &mut self,
465 _a0: &tenferro_tensor::Tensor,
466 _a1: &tenferro_tensor::Tensor,
467 _a2: &tenferro_tensor::Tensor,
468 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
469 refused("dynamic_update_slice")
470 }
471
472 fn pad(
473 &mut self,
474 _a0: &tenferro_tensor::Tensor,
475 _a1: &PadConfig,
476 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
477 refused("pad")
478 }
479
480 fn concatenate(
481 &mut self,
482 _a0: &[&tenferro_tensor::Tensor],
483 _a1: usize,
484 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
485 refused("concatenate")
486 }
487
488 fn reverse(
489 &mut self,
490 _a0: &tenferro_tensor::Tensor,
491 _a1: &[usize],
492 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
493 refused("reverse")
494 }
495}
496
497impl TensorDot for BorrowedCpuEngine<'_, '_> {
498 fn dot_general_read(
499 &mut self,
500 _a0: TensorRead<'_>,
501 _a1: TensorRead<'_>,
502 _a2: &DotGeneralConfig,
503 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
504 refused("dot_general_read")
505 }
506}
507
508impl TensorDeviceTransfer for BorrowedCpuEngine<'_, '_> {
509 fn download_to_host(
510 &mut self,
511 _a0: TensorRead<'_>,
512 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
513 refused("download_to_host")
514 }
515
516 fn upload_host_tensor(
517 &mut self,
518 _a0: TensorRead<'_>,
519 ) -> Result<tenferro_tensor::Tensor, tenferro_tensor::Error> {
520 refused("upload_host_tensor")
521 }
522}
523
524struct Lane<'p, R> {
526 pool: &'p mut BufferPool,
527 out: Option<R>,
528}
529
530impl CpuEngine {
531 pub fn with_lanes<R: Send>(
610 &mut self,
611 shares: &[usize],
612 pools: &mut [BufferPool],
613 f: impl Fn(usize, &mut dyn Engine) -> R + Sync,
614 ) -> crate::Result<Vec<R>> {
615 const OP: &str = "CpuEngine::with_lanes";
616 if shares.len() != pools.len() {
617 return Err(tenferro_tensor::Error::invalid_argument(
618 OP,
619 "pools",
620 format!(
621 "need one pool per lane: {} shares but {} pools",
622 shares.len(),
623 pools.len()
624 ),
625 ));
626 }
627 if shares.contains(&0) {
628 return Err(tenferro_tensor::Error::invalid_argument(
629 OP,
630 "shares",
631 format!("every lane needs at least one thread, got {shares:?}"),
632 ));
633 }
634 let budget = self.num_threads();
635 let total = shares
636 .iter()
637 .try_fold(0_usize, |sum, &share| sum.checked_add(share));
638 if total.is_none_or(|total| total > budget) {
639 return Err(tenferro_tensor::Error::invalid_argument(
640 OP,
641 "shares",
642 format!("the shares {shares:?} exceed the engine's thread budget of {budget}"),
643 ));
644 }
645 self.with_exec(|exec| {
646 let workspace = exec.workspace();
647 let policy = *exec.batch_policy();
648 let mut lanes: Vec<Lane<'_, R>> = pools
649 .iter_mut()
650 .map(|pool| Lane { pool, out: None })
651 .collect();
652 exec.token()
653 .fork_sized_slots(shares, &mut lanes, |index, lane, child| {
654 let mut bundle = Exec::from_token(child, workspace, policy);
655 let mut engine = BorrowedCpuEngine::new(&mut bundle, &mut *lane.pool);
656 lane.out = Some(f(index, &mut engine));
657 })
658 .map_err(|error| {
659 tenferro_tensor::Error::invalid_argument(OP, "shares", error.to_string())
660 })?;
661 lanes
662 .into_iter()
663 .map(|lane| {
664 lane.out.ok_or_else(|| {
665 tenferro_tensor::Error::Internal("a lane produced no result".into())
666 })
667 })
668 .collect()
669 })?
670 }
671}
672
673#[cfg(test)]
674mod tests;