tenferro_cpu_basic/buffer_pool.rs
1//! Typed host buffer pooling for reusable tensor allocations.
2//!
3//! # Examples
4//!
5//! ```rust
6//! use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
7//!
8//! let mut pool = BufferPool::new();
9//! let mut buf = pool.acquire_zeroed::<f64>(4);
10//! buf.fill(1.0);
11//! <f64 as PoolScalar>::pool_release(&mut pool, buf);
12//! assert_eq!(pool.len(), 1);
13//! ```
14
15use std::collections::BTreeMap;
16use std::env;
17use std::ffi::OsString;
18use std::fmt;
19use std::mem::{size_of, ManuallyDrop, MaybeUninit};
20use std::sync::{Arc, Mutex, MutexGuard, OnceLock, Weak};
21use tenferro_tensor::HostBufferRecycler;
22
23use num_complex::{Complex32, Complex64};
24
25use crate::CacheStats;
26
27/// Non-Copy proof of a tracked pool checkout.
28#[derive(Debug)]
29pub(crate) enum UninitCheckoutToken {
30 /// The allocation was freshly allocated.
31 Fresh { actual_capacity: usize },
32 /// Retained storage was removed, with its actual capacity.
33 Reused { actual_capacity: usize },
34}
35
36/// Environment variable overriding the CPU buffer-pool retention cap in bytes.
37///
38/// The value is parsed as an unsigned integer. Invalid values fall back to
39/// [`DEFAULT_MAX_RETAINED_CAPACITY_BYTES`].
40pub const BUFFER_POOL_MAX_RETAINED_BYTES_ENV: &str = "TENFERRO_BUFFER_POOL_MAX_RETAINED_BYTES";
41
42/// Default retained CPU buffer capacity per backend.
43///
44/// The cap keeps long-running workloads from accumulating obsolete buffer
45/// sizes as tensor shapes grow while still preserving reuse for hot working
46/// sets.
47pub const DEFAULT_MAX_RETAINED_CAPACITY_BYTES: usize = 100 * 1024 * 1024;
48
49static DEFAULT_MAX_RETAINED_CAPACITY_FROM_ENV: OnceLock<usize> = OnceLock::new();
50
51/// Snapshot of typed host buffers retained by a [`BufferPool`].
52///
53/// `buffers` counts retained `Vec` allocations, while `capacity_bytes` counts
54/// their total element capacity in bytes. Allocators may keep freed memory in
55/// process-local arenas after a pool is cleared, so this reports memory that is
56/// still live in the pool rather than operating-system RSS.
57#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
58pub struct BufferPoolStats {
59 /// Number of retained vector allocations.
60 pub buffers: usize,
61 /// Total retained vector capacity in bytes.
62 pub capacity_bytes: usize,
63}
64
65/// Typed buffer pool keyed by element capacity and separated by scalar type.
66///
67/// Each supported dtype has an independent best-fit pool. Acquired buffers are
68/// returned without zero-initialization so kernels can avoid redundant writes
69/// when they fully overwrite the output. Use [`PoolScalar::pool_acquire_zeroed`]
70/// when the caller may read the buffer before writing every element.
71///
72/// # Examples
73///
74/// ```rust
75/// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
76///
77/// let mut pool = BufferPool::new();
78/// let buf = pool.acquire_zeroed::<f32>(8);
79/// <f32 as PoolScalar>::pool_release(&mut pool, buf);
80/// assert_eq!(pool.len(), 1);
81/// ```
82pub struct BufferPool {
83 state: Arc<SharedPool>,
84}
85
86#[derive(Debug)]
87struct SharedPool(Mutex<PoolState>);
88
89#[derive(Debug)]
90struct PoolState {
91 f64_pool: BTreeMap<usize, Vec<Vec<f64>>>,
92 f32_pool: BTreeMap<usize, Vec<Vec<f32>>>,
93 i32_pool: BTreeMap<usize, Vec<Vec<i32>>>,
94 i64_pool: BTreeMap<usize, Vec<Vec<i64>>>,
95 bool_pool: BTreeMap<usize, Vec<Vec<bool>>>,
96 c64_pool: BTreeMap<usize, Vec<Vec<Complex64>>>,
97 c32_pool: BTreeMap<usize, Vec<Vec<Complex32>>>,
98 retained_capacity_bytes: usize,
99 max_retained_capacity_bytes: usize,
100}
101
102impl fmt::Debug for BufferPool {
103 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
104 f.debug_struct("BufferPool")
105 .field("stats", &self.stats())
106 .field(
107 "max_retained_capacity_bytes",
108 &self.max_retained_capacity_bytes(),
109 )
110 .finish_non_exhaustive()
111 }
112}
113
114/// Scalar types supported by [`BufferPool`].
115///
116/// The trait is sealed to the scalar dtypes that tenferro currently pools for
117/// CPU execution.
118///
119/// # Examples
120///
121/// ```rust
122/// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
123///
124/// let mut pool = BufferPool::new();
125/// let mut buf = pool.acquire_zeroed::<f64>(2);
126/// buf.copy_from_slice(&[3.0, 4.0]);
127/// <f64 as PoolScalar>::pool_release(&mut pool, buf);
128/// ```
129pub trait PoolScalar:
130 Copy + Sized + Send + Sync + tenferro_tensor::TensorScalar + private::Sealed
131{
132 /// Zero value used to initialize acquired buffers.
133 fn pool_zero() -> Self;
134
135 /// Acquire a buffer with length `len` and every element set to zero.
136 ///
137 /// This is the safe path for callers that may read the buffer before every
138 /// element is overwritten. Full-overwrite kernels should use
139 /// [`crate::PooledUninitOutput`] or an operation-specific uninitialized
140 /// destination guard.
141 ///
142 /// # Examples
143 ///
144 /// ```rust
145 /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
146 ///
147 /// let mut pool = BufferPool::new();
148 /// let buf = <f64 as PoolScalar>::pool_acquire_zeroed(&mut pool, 2);
149 /// assert_eq!(buf, vec![0.0, 0.0]);
150 /// ```
151 fn pool_acquire_zeroed(pool: &mut BufferPool, len: usize) -> Vec<Self>;
152
153 /// Return a buffer to the typed pool for later reuse.
154 ///
155 /// Zero-capacity buffers are ignored.
156 ///
157 /// # Examples
158 ///
159 /// ```rust
160 /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
161 ///
162 /// let mut pool = BufferPool::new();
163 /// let buf = vec![1.0_f32; 4];
164 /// <f32 as PoolScalar>::pool_release(&mut pool, buf);
165 /// assert_eq!(pool.len(), 1);
166 /// ```
167 fn pool_release(pool: &mut BufferPool, buf: Vec<Self>);
168}
169
170pub(crate) mod private {
171 use std::mem::MaybeUninit;
172
173 // INVARIANT: this sealed crate-private trait is the only implementation
174 // boundary for tracked guard tokens; it is not part of the public API.
175 #[allow(private_interfaces)]
176 pub trait Sealed {
177 /// Checks out uninitialized storage with an exact cleanup token.
178 ///
179 /// # Errors
180 /// Returns `Error::Validation` if retained-capacity byte accounting
181 /// overflows, or `Error::BackendSource` if fresh allocation fails.
182 fn pool_acquire_uninit_tracked(
183 pool: &mut super::BufferPool,
184 len: usize,
185 ) -> crate::Result<(Vec<MaybeUninit<Self>>, super::UninitCheckoutToken)>
186 where
187 Self: Sized;
188 fn pool_finish_recycled(
189 pool: &mut super::BufferPool,
190 checkout: super::UninitCheckoutToken,
191 ) -> std::sync::Weak<dyn tenferro_tensor::HostBufferRecycler<Self>>
192 where
193 Self: Sized;
194 }
195}
196
197/// How many emptied bins one acquisition may walk before it starts dropping them.
198///
199/// A bin that empties while another capacity is taken is usually transient: a kernel
200/// that cycles a handful of scratch sizes keeps them in flight in turn, and
201/// re-inserting their map nodes is what the pool exists to avoid. `tenferro-linalg`'s
202/// allocation ceilings count exactly that and are the guard: they hold at this budget
203/// of 8 or 16 and break at 4, so 16 leaves margin above the empties an SVD/QR call
204/// crosses.
205///
206/// A varying-shape run instead leaves one emptied bin per capacity it has ever used,
207/// and then every acquisition walks all of them (#2170: 41.8 us per acquire after
208/// 10 000 distinct capacities against 21 ns with none). Past this budget the emptied
209/// bins the walk passes are capacities the run has stopped using, so they go, and the
210/// walk stays bounded by the budget instead of by the history.
211const EMPTIED_BIN_SCAN_BUDGET: usize = 16;
212
213fn take_best_fit<T>(pool: &mut BTreeMap<usize, Vec<Vec<T>>>, len: usize) -> Option<Vec<T>> {
214 // Walk upward from `len` to the smallest retained capacity that can serve it,
215 // counting the emptied bins passed and dropping them once past the budget. The
216 // walk is one pass and allocates nothing until the budget is exceeded, so a hot
217 // loop that finds its own bin immediately pays nothing for it.
218 let mut cursor = len;
219 let mut emptied = 0usize;
220 let mut stale: Vec<usize> = Vec::new();
221 let serving = loop {
222 match pool
223 .range(cursor..)
224 .next()
225 .map(|(&key, vecs)| (key, vecs.is_empty()))
226 {
227 None => break None,
228 Some((key, false)) => break Some(key),
229 Some((key, true)) => {
230 emptied += 1;
231 if emptied > EMPTIED_BIN_SCAN_BUDGET {
232 stale.push(key);
233 }
234 cursor = key.saturating_add(1);
235 }
236 }
237 };
238 for key in stale {
239 pool.remove(&key);
240 }
241 let key = serving?;
242 let vecs = pool.get_mut(&key)?;
243 vecs.pop()
244}
245
246fn pool_len<T>(pool: &BTreeMap<usize, Vec<Vec<T>>>) -> usize {
247 pool.values().map(Vec::len).sum()
248}
249
250#[cfg(test)]
251fn empty_bins<T>(pool: &BTreeMap<usize, Vec<Vec<T>>>) -> usize {
252 pool.values().filter(|vecs| vecs.is_empty()).count()
253}
254
255fn evict_one_from_pool<T>(pool: &mut BTreeMap<usize, Vec<Vec<T>>>) -> Option<usize> {
256 let key = *pool.iter().find(|(_, vecs)| !vecs.is_empty())?.0;
257 let vecs = pool.get_mut(&key)?;
258 let _ = vecs.pop()?;
259 if vecs.is_empty() {
260 // `take_best_fit` drops an emptied bin too; eviction cannot leave one
261 // behind either.
262 pool.remove(&key);
263 }
264 Some(key.saturating_mul(size_of::<T>()))
265}
266
267#[derive(Clone, Copy)]
268enum TypedPoolKind {
269 F64,
270 F32,
271 I32,
272 I64,
273 Bool,
274 C64,
275 C32,
276}
277
278fn smallest_pool_candidate<T>(
279 pool: &BTreeMap<usize, Vec<Vec<T>>>,
280 kind: TypedPoolKind,
281) -> Option<(usize, TypedPoolKind)> {
282 // Bins are dropped when they empty (see `take_best_fit`), so the first
283 // entry is the smallest retained capacity; the emptiness check stays as a
284 // guard for a bin a future path could leave behind.
285 pool.iter()
286 .find(|(_, vecs)| !vecs.is_empty())
287 .map(|(&capacity, _)| (capacity.saturating_mul(size_of::<T>()), kind))
288}
289
290macro_rules! impl_pool_scalar {
291 ($ty:ty, $field:ident, $zero:expr) => {
292 // INVARIANT: this implementation is reachable only through the sealed
293 // crate-private helper and cannot be named by sibling crates.
294 #[allow(private_interfaces)]
295 impl private::Sealed for $ty {
296 fn pool_acquire_uninit_tracked(
297 pool: &mut BufferPool,
298 len: usize,
299 ) -> crate::Result<(Vec<MaybeUninit<Self>>, UninitCheckoutToken)> {
300 let mut state = lock_pool(&pool.state);
301 let pool = &mut *state;
302 match take_best_fit(&mut pool.$field, len) {
303 Some(buf) => {
304 let cap = buf.capacity();
305 let bytes = cap.checked_mul(size_of::<Self>()).ok_or_else(|| {
306 crate::Error::invalid_argument(
307 "pooled_uninit_output",
308 "length",
309 "pool capacity byte length overflow",
310 )
311 })?;
312 pool.retained_capacity_bytes -= bytes;
313 let mut buf = ManuallyDrop::new(buf);
314 // SAFETY: MaybeUninit<Self> has the same layout as Self and len <= cap.
315 let buf = unsafe { Vec::from_raw_parts(buf.as_mut_ptr().cast(), len, cap) };
316 Ok((
317 buf,
318 UninitCheckoutToken::Reused {
319 actual_capacity: cap,
320 },
321 ))
322 }
323 None => {
324 let mut buf = Vec::new();
325 buf.try_reserve_exact(len).map_err(|err| {
326 crate::Error::backend_source("pooled_uninit_output", err)
327 })?;
328 // SAFETY: every bit pattern is valid for MaybeUninit.
329 unsafe { buf.set_len(len) };
330 let actual_capacity = buf.capacity();
331 Ok((buf, UninitCheckoutToken::Fresh { actual_capacity }))
332 }
333 }
334 }
335
336 fn pool_finish_recycled(
337 pool: &mut BufferPool,
338 checkout: UninitCheckoutToken,
339 ) -> Weak<dyn HostBufferRecycler<Self>> {
340 let _ = checkout;
341 let owner: Arc<dyn HostBufferRecycler<Self>> = pool.state.clone();
342 Arc::downgrade(&owner)
343 }
344 }
345
346 impl PoolScalar for $ty {
347 fn pool_zero() -> Self {
348 $zero
349 }
350
351 fn pool_acquire_zeroed(pool: &mut BufferPool, len: usize) -> Vec<Self> {
352 let mut state = lock_pool(&pool.state);
353 let pool = &mut *state;
354 match take_best_fit(&mut pool.$field, len) {
355 Some(mut buf) => {
356 pool.retained_capacity_bytes = pool
357 .retained_capacity_bytes
358 .saturating_sub(buf.capacity().saturating_mul(size_of::<Self>()));
359 buf.resize(len, Self::pool_zero());
360 buf.fill(Self::pool_zero());
361 buf
362 }
363 None => vec![Self::pool_zero(); len],
364 }
365 }
366
367 fn pool_release(pool: &mut BufferPool, buf: Vec<Self>) {
368 pool.state.recycle(buf);
369 }
370 }
371
372 impl HostBufferRecycler<$ty> for SharedPool {
373 fn recycle(&self, buf: Vec<$ty>) {
374 let mut state = lock_pool(self);
375 let pool = &mut *state;
376 let cap = buf.capacity();
377 if cap > 0 {
378 pool.retained_capacity_bytes = pool
379 .retained_capacity_bytes
380 .saturating_add(cap.saturating_mul(size_of::<$ty>()));
381 pool.$field.entry(cap).or_default().push(buf);
382 pool.enforce_retention_limit();
383 }
384 }
385 }
386 };
387}
388
389impl_pool_scalar!(f64, f64_pool, 0.0);
390impl_pool_scalar!(f32, f32_pool, 0.0);
391impl_pool_scalar!(i32, i32_pool, 0);
392impl_pool_scalar!(i64, i64_pool, 0);
393impl_pool_scalar!(bool, bool_pool, false);
394impl_pool_scalar!(Complex64, c64_pool, Complex64::new(0.0, 0.0));
395impl_pool_scalar!(Complex32, c32_pool, Complex32::new(0.0, 0.0));
396
397impl BufferPool {
398 pub(crate) fn checkout_handle(&self) -> Self {
399 Self {
400 state: Arc::clone(&self.state),
401 }
402 }
403
404 /// Number of retained capacity bins, one per capacity the pool still holds.
405 #[cfg(test)]
406 pub(crate) fn bin_count(&self) -> usize {
407 let state = lock_pool(&self.state);
408 state.f64_pool.len()
409 + state.f32_pool.len()
410 + state.i32_pool.len()
411 + state.i64_pool.len()
412 + state.bool_pool.len()
413 + state.c64_pool.len()
414 + state.c32_pool.len()
415 }
416
417 /// Number of retained capacity bins that hold no buffer.
418 #[cfg(test)]
419 pub(crate) fn empty_bin_count(&self) -> usize {
420 let state = lock_pool(&self.state);
421 empty_bins(&state.f64_pool)
422 + empty_bins(&state.f32_pool)
423 + empty_bins(&state.i32_pool)
424 + empty_bins(&state.i64_pool)
425 + empty_bins(&state.bool_pool)
426 + empty_bins(&state.c64_pool)
427 + empty_bins(&state.c32_pool)
428 }
429
430 /// Create an empty typed buffer pool.
431 ///
432 /// # Examples
433 ///
434 /// ```rust
435 /// use tenferro_cpu_basic::buffer_pool::BufferPool;
436 ///
437 /// let pool = BufferPool::new();
438 /// assert!(pool.is_empty());
439 /// ```
440 pub fn new() -> Self {
441 Self::with_max_retained_capacity_bytes(default_max_retained_capacity_bytes())
442 }
443
444 /// Create an empty typed buffer pool with a specific retention cap.
445 ///
446 /// A cap of zero disables retention. Use [`BufferPool::unbounded`] only for
447 /// diagnostics or workloads that are externally memory-limited.
448 ///
449 /// # Examples
450 ///
451 /// ```rust
452 /// use tenferro_cpu_basic::buffer_pool::BufferPool;
453 ///
454 /// let pool = BufferPool::with_max_retained_capacity_bytes(1024);
455 /// assert_eq!(pool.max_retained_capacity_bytes(), 1024);
456 /// ```
457 pub fn with_max_retained_capacity_bytes(max_retained_capacity_bytes: usize) -> Self {
458 Self {
459 state: Arc::new(SharedPool(Mutex::new(PoolState {
460 f64_pool: BTreeMap::new(),
461 f32_pool: BTreeMap::new(),
462 i32_pool: BTreeMap::new(),
463 i64_pool: BTreeMap::new(),
464 bool_pool: BTreeMap::new(),
465 c64_pool: BTreeMap::new(),
466 c32_pool: BTreeMap::new(),
467 retained_capacity_bytes: 0,
468 max_retained_capacity_bytes,
469 }))),
470 }
471 }
472
473 /// Create an empty typed buffer pool without a retention cap.
474 ///
475 /// This preserves the historical behavior and is mainly useful for
476 /// diagnostics or controlled benchmarks.
477 ///
478 /// # Examples
479 ///
480 /// ```rust
481 /// use tenferro_cpu_basic::buffer_pool::BufferPool;
482 ///
483 /// let pool = BufferPool::unbounded();
484 /// assert_eq!(pool.max_retained_capacity_bytes(), usize::MAX);
485 /// ```
486 pub fn unbounded() -> Self {
487 Self::with_max_retained_capacity_bytes(usize::MAX)
488 }
489
490 /// Maximum retained typed host-buffer capacity in bytes.
491 ///
492 /// # Examples
493 ///
494 /// ```rust
495 /// use tenferro_cpu_basic::buffer_pool::BufferPool;
496 ///
497 /// let pool = BufferPool::with_max_retained_capacity_bytes(4096);
498 /// assert_eq!(pool.max_retained_capacity_bytes(), 4096);
499 /// ```
500 pub fn max_retained_capacity_bytes(&self) -> usize {
501 lock_pool(&self.state).max_retained_capacity_bytes
502 }
503
504 /// Update the maximum retained typed host-buffer capacity in bytes.
505 ///
506 /// Shrinking below the currently retained capacity immediately evicts
507 /// retained buffers until the new cap is satisfied. A cap of zero disables
508 /// retention.
509 ///
510 /// # Examples
511 ///
512 /// ```
513 /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
514 ///
515 /// let mut pool = BufferPool::with_max_retained_capacity_bytes(1024);
516 /// <f64 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(128));
517 /// pool.set_max_retained_capacity_bytes(0);
518 /// assert_eq!(pool.max_retained_capacity_bytes(), 0);
519 /// assert!(pool.is_empty());
520 /// ```
521 pub fn set_max_retained_capacity_bytes(&mut self, max_retained_capacity_bytes: usize) {
522 let mut state = lock_pool(&self.state);
523 state.max_retained_capacity_bytes = max_retained_capacity_bytes;
524 state.enforce_retention_limit();
525 }
526
527 /// Number of retained buffers across all typed pools.
528 ///
529 /// # Examples
530 ///
531 /// ```rust
532 /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
533 ///
534 /// let mut pool = BufferPool::new();
535 /// <f64 as PoolScalar>::pool_release(&mut pool, vec![0.0; 2]);
536 /// assert_eq!(pool.len(), 1);
537 /// ```
538 pub fn len(&self) -> usize {
539 self.stats().buffers
540 }
541
542 /// Total retained typed host-buffer capacity in bytes.
543 ///
544 /// This counts capacity that is still live in the pool. The operating
545 /// system RSS may remain high after clearing the pool because the process
546 /// allocator can keep freed pages for future allocations.
547 ///
548 /// # Examples
549 ///
550 /// ```rust
551 /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
552 ///
553 /// let mut pool = BufferPool::new();
554 /// <f64 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(2));
555 /// assert_eq!(pool.retained_capacity_bytes(), 16);
556 /// ```
557 pub fn retained_capacity_bytes(&self) -> usize {
558 self.stats().capacity_bytes
559 }
560
561 /// Snapshot retained-buffer count and capacity.
562 ///
563 /// # Examples
564 ///
565 /// ```rust
566 /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
567 ///
568 /// let mut pool = BufferPool::new();
569 /// <f32 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(4));
570 /// let stats = pool.stats();
571 /// assert_eq!(stats.buffers, 1);
572 /// assert_eq!(stats.capacity_bytes, 16);
573 /// ```
574 pub fn stats(&self) -> BufferPoolStats {
575 let state = lock_pool(&self.state);
576 BufferPoolStats {
577 buffers: pool_len(&state.f64_pool)
578 + pool_len(&state.f32_pool)
579 + pool_len(&state.i32_pool)
580 + pool_len(&state.i64_pool)
581 + pool_len(&state.bool_pool)
582 + pool_len(&state.c64_pool)
583 + pool_len(&state.c32_pool),
584 capacity_bytes: state.retained_capacity_bytes,
585 }
586 }
587
588 /// Return cache-style stats for the buffers retained by this pool.
589 ///
590 /// `entries` is the number of retained buffers, and `retained_bytes` is the
591 /// total retained vector capacity in bytes.
592 ///
593 /// # Examples
594 ///
595 /// ```
596 /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
597 ///
598 /// let mut pool = BufferPool::new();
599 /// <f32 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(4));
600 /// let stats = pool.cache_stats();
601 /// assert_eq!(stats.entries, 1);
602 /// assert_eq!(stats.retained_bytes, 16);
603 /// ```
604 pub fn cache_stats(&self) -> CacheStats {
605 let stats = self.stats();
606 CacheStats {
607 entries: stats.buffers,
608 retained_bytes: stats.capacity_bytes,
609 hits: 0,
610 misses: 0,
611 evictions: 0,
612 clears: 0,
613 }
614 }
615
616 /// Acquire a typed vector with length 0 and at least `cap` capacity.
617 ///
618 /// Returned buffers come from the typed pool when possible and are ready
619 /// for push-based population.
620 ///
621 /// # Examples
622 ///
623 /// ```rust
624 /// use tenferro_cpu_basic::buffer_pool::BufferPool;
625 ///
626 /// let mut pool = BufferPool::new();
627 /// let mut buf = pool.acquire_with_capacity::<f64>(4);
628 /// buf.extend_from_slice(&[1.0, 2.0]);
629 /// assert_eq!(buf.len(), 2);
630 /// assert!(buf.capacity() >= 4);
631 /// ```
632 pub fn acquire_with_capacity<T: PoolScalar>(&mut self, cap: usize) -> Vec<T> {
633 if cap == 0 {
634 return Vec::new();
635 }
636
637 let (data, _checkout) = <T as private::Sealed>::pool_acquire_uninit_tracked(self, cap)
638 .expect("validated typed pool capacity must be acquirable");
639 let mut data = ManuallyDrop::new(data);
640 let ptr = data.as_mut_ptr().cast::<T>();
641 let capacity = data.capacity();
642 // SAFETY: `MaybeUninit<T>` and `T` have identical layouts, and the
643 // returned vector has length zero, so no element is read before push.
644 unsafe { Vec::from_raw_parts(ptr, 0, capacity) }
645 }
646
647 /// Acquire a typed vector with length `len` initialized to zero.
648 ///
649 /// Use this only when the caller may read elements before overwriting the
650 /// entire buffer. Full-overwrite kernels should use
651 /// [`crate::PooledUninitOutput`] to avoid the initialization cost.
652 ///
653 /// # Examples
654 ///
655 /// ```rust
656 /// use tenferro_cpu_basic::buffer_pool::BufferPool;
657 ///
658 /// let mut pool = BufferPool::new();
659 /// let buf = pool.acquire_zeroed::<f32>(3);
660 /// assert_eq!(buf, vec![0.0, 0.0, 0.0]);
661 /// ```
662 pub fn acquire_zeroed<T: PoolScalar>(&mut self, len: usize) -> Vec<T> {
663 T::pool_acquire_zeroed(self, len)
664 }
665
666 /// Whether all typed pools are empty.
667 ///
668 /// # Examples
669 ///
670 /// ```rust
671 /// use tenferro_cpu_basic::buffer_pool::BufferPool;
672 ///
673 /// let pool = BufferPool::new();
674 /// assert!(pool.is_empty());
675 /// ```
676 pub fn is_empty(&self) -> bool {
677 self.len() == 0
678 }
679
680 /// Drop all retained buffers from the pool.
681 ///
682 /// This releases the vectors owned by the pool. The process allocator may
683 /// still keep freed pages mapped for reuse, so operating-system RSS is not
684 /// guaranteed to fall immediately.
685 ///
686 /// # Examples
687 ///
688 /// ```rust
689 /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
690 ///
691 /// let mut pool = BufferPool::new();
692 /// <f64 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(8));
693 /// pool.clear();
694 /// assert!(pool.is_empty());
695 /// ```
696 pub fn clear(&mut self) {
697 let mut state = lock_pool(&self.state);
698 state.clear();
699 }
700}
701
702// INVARIANT: no user callback or tensor destructor runs under this lock; it
703// protects only scalar Vec bins and their accounting, never execution admission.
704fn lock_pool(state: &SharedPool) -> MutexGuard<'_, PoolState> {
705 state
706 .0
707 .lock()
708 .unwrap_or_else(std::sync::PoisonError::into_inner)
709}
710
711impl PoolState {
712 fn clear(&mut self) {
713 self.f64_pool.clear();
714 self.f32_pool.clear();
715 self.i32_pool.clear();
716 self.i64_pool.clear();
717 self.bool_pool.clear();
718 self.c64_pool.clear();
719 self.c32_pool.clear();
720 self.retained_capacity_bytes = 0;
721 }
722
723 fn enforce_retention_limit(&mut self) {
724 while self.retained_capacity_bytes > self.max_retained_capacity_bytes {
725 let Some(evicted_bytes) = self.evict_smallest_retained_buffer() else {
726 self.retained_capacity_bytes = 0;
727 return;
728 };
729 if evicted_bytes == 0 {
730 if self.retained_capacity_bytes == 0 {
731 self.retained_capacity_bytes = 0;
732 return;
733 }
734 continue;
735 }
736 self.retained_capacity_bytes =
737 self.retained_capacity_bytes.saturating_sub(evicted_bytes);
738 }
739 }
740
741 fn evict_smallest_retained_buffer(&mut self) -> Option<usize> {
742 let candidates = [
743 smallest_pool_candidate(&self.f64_pool, TypedPoolKind::F64),
744 smallest_pool_candidate(&self.f32_pool, TypedPoolKind::F32),
745 smallest_pool_candidate(&self.i32_pool, TypedPoolKind::I32),
746 smallest_pool_candidate(&self.i64_pool, TypedPoolKind::I64),
747 smallest_pool_candidate(&self.bool_pool, TypedPoolKind::Bool),
748 smallest_pool_candidate(&self.c64_pool, TypedPoolKind::C64),
749 smallest_pool_candidate(&self.c32_pool, TypedPoolKind::C32),
750 ];
751 let (_, kind) = candidates
752 .into_iter()
753 .flatten()
754 .min_by_key(|(bytes, _)| *bytes)?;
755 match kind {
756 TypedPoolKind::F64 => evict_one_from_pool(&mut self.f64_pool),
757 TypedPoolKind::F32 => evict_one_from_pool(&mut self.f32_pool),
758 TypedPoolKind::I32 => evict_one_from_pool(&mut self.i32_pool),
759 TypedPoolKind::I64 => evict_one_from_pool(&mut self.i64_pool),
760 TypedPoolKind::Bool => evict_one_from_pool(&mut self.bool_pool),
761 TypedPoolKind::C64 => evict_one_from_pool(&mut self.c64_pool),
762 TypedPoolKind::C32 => evict_one_from_pool(&mut self.c32_pool),
763 }
764 }
765}
766
767fn default_max_retained_capacity_bytes() -> usize {
768 *DEFAULT_MAX_RETAINED_CAPACITY_FROM_ENV.get_or_init(|| {
769 parse_default_max_retained_capacity_bytes(env::var_os(BUFFER_POOL_MAX_RETAINED_BYTES_ENV))
770 })
771}
772
773fn parse_default_max_retained_capacity_bytes(value: Option<OsString>) -> usize {
774 value
775 .and_then(|value| value.into_string().ok())
776 .and_then(|value| value.parse().ok())
777 .unwrap_or(DEFAULT_MAX_RETAINED_CAPACITY_BYTES)
778}
779
780impl Default for BufferPool {
781 fn default() -> Self {
782 Self::new()
783 }
784}
785
786#[cfg(test)]
787mod tests;