Skip to main content

tenferro_cpu_basic/
buffer_pool.rs

1//! Typed host buffer pooling for reusable tensor allocations.
2//!
3//! # Examples
4//!
5//! ```rust
6//! use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
7//!
8//! let mut pool = BufferPool::new();
9//! let mut buf = pool.acquire_zeroed::<f64>(4);
10//! buf.fill(1.0);
11//! <f64 as PoolScalar>::pool_release(&mut pool, buf);
12//! assert_eq!(pool.len(), 1);
13//! ```
14
15use std::collections::BTreeMap;
16use std::env;
17use std::ffi::OsString;
18use std::fmt;
19use std::mem::{size_of, ManuallyDrop, MaybeUninit};
20use std::sync::{Arc, Mutex, MutexGuard, OnceLock, Weak};
21use tenferro_tensor::HostBufferRecycler;
22
23use num_complex::{Complex32, Complex64};
24
25use crate::CacheStats;
26
27/// Non-Copy proof of a tracked pool checkout.
28#[derive(Debug)]
29pub(crate) enum UninitCheckoutToken {
30    /// The allocation was freshly allocated.
31    Fresh { actual_capacity: usize },
32    /// Retained storage was removed, with its actual capacity.
33    Reused { actual_capacity: usize },
34}
35
36/// Environment variable overriding the CPU buffer-pool retention cap in bytes.
37///
38/// The value is parsed as an unsigned integer. Invalid values fall back to
39/// [`DEFAULT_MAX_RETAINED_CAPACITY_BYTES`].
40pub const BUFFER_POOL_MAX_RETAINED_BYTES_ENV: &str = "TENFERRO_BUFFER_POOL_MAX_RETAINED_BYTES";
41
42/// Default retained CPU buffer capacity per backend.
43///
44/// The cap keeps long-running workloads from accumulating obsolete buffer
45/// sizes as tensor shapes grow while still preserving reuse for hot working
46/// sets.
47pub const DEFAULT_MAX_RETAINED_CAPACITY_BYTES: usize = 100 * 1024 * 1024;
48
49static DEFAULT_MAX_RETAINED_CAPACITY_FROM_ENV: OnceLock<usize> = OnceLock::new();
50
51/// Snapshot of typed host buffers retained by a [`BufferPool`].
52///
53/// `buffers` counts retained `Vec` allocations, while `capacity_bytes` counts
54/// their total element capacity in bytes. Allocators may keep freed memory in
55/// process-local arenas after a pool is cleared, so this reports memory that is
56/// still live in the pool rather than operating-system RSS.
57#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
58pub struct BufferPoolStats {
59    /// Number of retained vector allocations.
60    pub buffers: usize,
61    /// Total retained vector capacity in bytes.
62    pub capacity_bytes: usize,
63}
64
65/// Typed buffer pool keyed by element capacity and separated by scalar type.
66///
67/// Each supported dtype has an independent best-fit pool. Acquired buffers are
68/// returned without zero-initialization so kernels can avoid redundant writes
69/// when they fully overwrite the output. Use [`PoolScalar::pool_acquire_zeroed`]
70/// when the caller may read the buffer before writing every element.
71///
72/// # Examples
73///
74/// ```rust
75/// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
76///
77/// let mut pool = BufferPool::new();
78/// let buf = pool.acquire_zeroed::<f32>(8);
79/// <f32 as PoolScalar>::pool_release(&mut pool, buf);
80/// assert_eq!(pool.len(), 1);
81/// ```
82pub struct BufferPool {
83    state: Arc<SharedPool>,
84}
85
86#[derive(Debug)]
87struct SharedPool(Mutex<PoolState>);
88
89#[derive(Debug)]
90struct PoolState {
91    f64_pool: BTreeMap<usize, Vec<Vec<f64>>>,
92    f32_pool: BTreeMap<usize, Vec<Vec<f32>>>,
93    i32_pool: BTreeMap<usize, Vec<Vec<i32>>>,
94    i64_pool: BTreeMap<usize, Vec<Vec<i64>>>,
95    bool_pool: BTreeMap<usize, Vec<Vec<bool>>>,
96    c64_pool: BTreeMap<usize, Vec<Vec<Complex64>>>,
97    c32_pool: BTreeMap<usize, Vec<Vec<Complex32>>>,
98    retained_capacity_bytes: usize,
99    max_retained_capacity_bytes: usize,
100}
101
102impl fmt::Debug for BufferPool {
103    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
104        f.debug_struct("BufferPool")
105            .field("stats", &self.stats())
106            .field(
107                "max_retained_capacity_bytes",
108                &self.max_retained_capacity_bytes(),
109            )
110            .finish_non_exhaustive()
111    }
112}
113
114/// Scalar types supported by [`BufferPool`].
115///
116/// The trait is sealed to the scalar dtypes that tenferro currently pools for
117/// CPU execution.
118///
119/// # Examples
120///
121/// ```rust
122/// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
123///
124/// let mut pool = BufferPool::new();
125/// let mut buf = pool.acquire_zeroed::<f64>(2);
126/// buf.copy_from_slice(&[3.0, 4.0]);
127/// <f64 as PoolScalar>::pool_release(&mut pool, buf);
128/// ```
129pub trait PoolScalar:
130    Copy + Sized + Send + Sync + tenferro_tensor::TensorScalar + private::Sealed
131{
132    /// Zero value used to initialize acquired buffers.
133    fn pool_zero() -> Self;
134
135    /// Acquire a buffer with length `len` and every element set to zero.
136    ///
137    /// This is the safe path for callers that may read the buffer before every
138    /// element is overwritten. Full-overwrite kernels should use
139    /// [`crate::PooledUninitOutput`] or an operation-specific uninitialized
140    /// destination guard.
141    ///
142    /// # Examples
143    ///
144    /// ```rust
145    /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
146    ///
147    /// let mut pool = BufferPool::new();
148    /// let buf = <f64 as PoolScalar>::pool_acquire_zeroed(&mut pool, 2);
149    /// assert_eq!(buf, vec![0.0, 0.0]);
150    /// ```
151    fn pool_acquire_zeroed(pool: &mut BufferPool, len: usize) -> Vec<Self>;
152
153    /// Return a buffer to the typed pool for later reuse.
154    ///
155    /// Zero-capacity buffers are ignored.
156    ///
157    /// # Examples
158    ///
159    /// ```rust
160    /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
161    ///
162    /// let mut pool = BufferPool::new();
163    /// let buf = vec![1.0_f32; 4];
164    /// <f32 as PoolScalar>::pool_release(&mut pool, buf);
165    /// assert_eq!(pool.len(), 1);
166    /// ```
167    fn pool_release(pool: &mut BufferPool, buf: Vec<Self>);
168}
169
170pub(crate) mod private {
171    use std::mem::MaybeUninit;
172
173    // INVARIANT: this sealed crate-private trait is the only implementation
174    // boundary for tracked guard tokens; it is not part of the public API.
175    #[allow(private_interfaces)]
176    pub trait Sealed {
177        /// Checks out uninitialized storage with an exact cleanup token.
178        ///
179        /// # Errors
180        /// Returns `Error::Validation` if retained-capacity byte accounting
181        /// overflows, or `Error::BackendSource` if fresh allocation fails.
182        fn pool_acquire_uninit_tracked(
183            pool: &mut super::BufferPool,
184            len: usize,
185        ) -> crate::Result<(Vec<MaybeUninit<Self>>, super::UninitCheckoutToken)>
186        where
187            Self: Sized;
188        fn pool_finish_recycled(
189            pool: &mut super::BufferPool,
190            checkout: super::UninitCheckoutToken,
191        ) -> std::sync::Weak<dyn tenferro_tensor::HostBufferRecycler<Self>>
192        where
193            Self: Sized;
194    }
195}
196
197/// How many emptied bins one acquisition may walk before it starts dropping them.
198///
199/// A bin that empties while another capacity is taken is usually transient: a kernel
200/// that cycles a handful of scratch sizes keeps them in flight in turn, and
201/// re-inserting their map nodes is what the pool exists to avoid. `tenferro-linalg`'s
202/// allocation ceilings count exactly that and are the guard: they hold at this budget
203/// of 8 or 16 and break at 4, so 16 leaves margin above the empties an SVD/QR call
204/// crosses.
205///
206/// A varying-shape run instead leaves one emptied bin per capacity it has ever used,
207/// and then every acquisition walks all of them (#2170: 41.8 us per acquire after
208/// 10 000 distinct capacities against 21 ns with none). Past this budget the emptied
209/// bins the walk passes are capacities the run has stopped using, so they go, and the
210/// walk stays bounded by the budget instead of by the history.
211const EMPTIED_BIN_SCAN_BUDGET: usize = 16;
212
213fn take_best_fit<T>(pool: &mut BTreeMap<usize, Vec<Vec<T>>>, len: usize) -> Option<Vec<T>> {
214    // Walk upward from `len` to the smallest retained capacity that can serve it,
215    // counting the emptied bins passed and dropping them once past the budget. The
216    // walk is one pass and allocates nothing until the budget is exceeded, so a hot
217    // loop that finds its own bin immediately pays nothing for it.
218    let mut cursor = len;
219    let mut emptied = 0usize;
220    let mut stale: Vec<usize> = Vec::new();
221    let serving = loop {
222        match pool
223            .range(cursor..)
224            .next()
225            .map(|(&key, vecs)| (key, vecs.is_empty()))
226        {
227            None => break None,
228            Some((key, false)) => break Some(key),
229            Some((key, true)) => {
230                emptied += 1;
231                if emptied > EMPTIED_BIN_SCAN_BUDGET {
232                    stale.push(key);
233                }
234                cursor = key.saturating_add(1);
235            }
236        }
237    };
238    for key in stale {
239        pool.remove(&key);
240    }
241    let key = serving?;
242    let vecs = pool.get_mut(&key)?;
243    vecs.pop()
244}
245
246fn pool_len<T>(pool: &BTreeMap<usize, Vec<Vec<T>>>) -> usize {
247    pool.values().map(Vec::len).sum()
248}
249
250#[cfg(test)]
251fn empty_bins<T>(pool: &BTreeMap<usize, Vec<Vec<T>>>) -> usize {
252    pool.values().filter(|vecs| vecs.is_empty()).count()
253}
254
255fn evict_one_from_pool<T>(pool: &mut BTreeMap<usize, Vec<Vec<T>>>) -> Option<usize> {
256    let key = *pool.iter().find(|(_, vecs)| !vecs.is_empty())?.0;
257    let vecs = pool.get_mut(&key)?;
258    let _ = vecs.pop()?;
259    if vecs.is_empty() {
260        // `take_best_fit` drops an emptied bin too; eviction cannot leave one
261        // behind either.
262        pool.remove(&key);
263    }
264    Some(key.saturating_mul(size_of::<T>()))
265}
266
267#[derive(Clone, Copy)]
268enum TypedPoolKind {
269    F64,
270    F32,
271    I32,
272    I64,
273    Bool,
274    C64,
275    C32,
276}
277
278fn smallest_pool_candidate<T>(
279    pool: &BTreeMap<usize, Vec<Vec<T>>>,
280    kind: TypedPoolKind,
281) -> Option<(usize, TypedPoolKind)> {
282    // Bins are dropped when they empty (see `take_best_fit`), so the first
283    // entry is the smallest retained capacity; the emptiness check stays as a
284    // guard for a bin a future path could leave behind.
285    pool.iter()
286        .find(|(_, vecs)| !vecs.is_empty())
287        .map(|(&capacity, _)| (capacity.saturating_mul(size_of::<T>()), kind))
288}
289
290macro_rules! impl_pool_scalar {
291    ($ty:ty, $field:ident, $zero:expr) => {
292        // INVARIANT: this implementation is reachable only through the sealed
293        // crate-private helper and cannot be named by sibling crates.
294        #[allow(private_interfaces)]
295        impl private::Sealed for $ty {
296            fn pool_acquire_uninit_tracked(
297                pool: &mut BufferPool,
298                len: usize,
299            ) -> crate::Result<(Vec<MaybeUninit<Self>>, UninitCheckoutToken)> {
300                let mut state = lock_pool(&pool.state);
301                let pool = &mut *state;
302                match take_best_fit(&mut pool.$field, len) {
303                    Some(buf) => {
304                        let cap = buf.capacity();
305                        let bytes = cap.checked_mul(size_of::<Self>()).ok_or_else(|| {
306                            crate::Error::invalid_argument(
307                                "pooled_uninit_output",
308                                "length",
309                                "pool capacity byte length overflow",
310                            )
311                        })?;
312                        pool.retained_capacity_bytes -= bytes;
313                        let mut buf = ManuallyDrop::new(buf);
314                        // SAFETY: MaybeUninit<Self> has the same layout as Self and len <= cap.
315                        let buf = unsafe { Vec::from_raw_parts(buf.as_mut_ptr().cast(), len, cap) };
316                        Ok((
317                            buf,
318                            UninitCheckoutToken::Reused {
319                                actual_capacity: cap,
320                            },
321                        ))
322                    }
323                    None => {
324                        let mut buf = Vec::new();
325                        buf.try_reserve_exact(len).map_err(|err| {
326                            crate::Error::backend_source("pooled_uninit_output", err)
327                        })?;
328                        // SAFETY: every bit pattern is valid for MaybeUninit.
329                        unsafe { buf.set_len(len) };
330                        let actual_capacity = buf.capacity();
331                        Ok((buf, UninitCheckoutToken::Fresh { actual_capacity }))
332                    }
333                }
334            }
335
336            fn pool_finish_recycled(
337                pool: &mut BufferPool,
338                checkout: UninitCheckoutToken,
339            ) -> Weak<dyn HostBufferRecycler<Self>> {
340                let _ = checkout;
341                let owner: Arc<dyn HostBufferRecycler<Self>> = pool.state.clone();
342                Arc::downgrade(&owner)
343            }
344        }
345
346        impl PoolScalar for $ty {
347            fn pool_zero() -> Self {
348                $zero
349            }
350
351            fn pool_acquire_zeroed(pool: &mut BufferPool, len: usize) -> Vec<Self> {
352                let mut state = lock_pool(&pool.state);
353                let pool = &mut *state;
354                match take_best_fit(&mut pool.$field, len) {
355                    Some(mut buf) => {
356                        pool.retained_capacity_bytes = pool
357                            .retained_capacity_bytes
358                            .saturating_sub(buf.capacity().saturating_mul(size_of::<Self>()));
359                        buf.resize(len, Self::pool_zero());
360                        buf.fill(Self::pool_zero());
361                        buf
362                    }
363                    None => vec![Self::pool_zero(); len],
364                }
365            }
366
367            fn pool_release(pool: &mut BufferPool, buf: Vec<Self>) {
368                pool.state.recycle(buf);
369            }
370        }
371
372        impl HostBufferRecycler<$ty> for SharedPool {
373            fn recycle(&self, buf: Vec<$ty>) {
374                let mut state = lock_pool(self);
375                let pool = &mut *state;
376                let cap = buf.capacity();
377                if cap > 0 {
378                    pool.retained_capacity_bytes = pool
379                        .retained_capacity_bytes
380                        .saturating_add(cap.saturating_mul(size_of::<$ty>()));
381                    pool.$field.entry(cap).or_default().push(buf);
382                    pool.enforce_retention_limit();
383                }
384            }
385        }
386    };
387}
388
389impl_pool_scalar!(f64, f64_pool, 0.0);
390impl_pool_scalar!(f32, f32_pool, 0.0);
391impl_pool_scalar!(i32, i32_pool, 0);
392impl_pool_scalar!(i64, i64_pool, 0);
393impl_pool_scalar!(bool, bool_pool, false);
394impl_pool_scalar!(Complex64, c64_pool, Complex64::new(0.0, 0.0));
395impl_pool_scalar!(Complex32, c32_pool, Complex32::new(0.0, 0.0));
396
397impl BufferPool {
398    pub(crate) fn checkout_handle(&self) -> Self {
399        Self {
400            state: Arc::clone(&self.state),
401        }
402    }
403
404    /// Number of retained capacity bins, one per capacity the pool still holds.
405    #[cfg(test)]
406    pub(crate) fn bin_count(&self) -> usize {
407        let state = lock_pool(&self.state);
408        state.f64_pool.len()
409            + state.f32_pool.len()
410            + state.i32_pool.len()
411            + state.i64_pool.len()
412            + state.bool_pool.len()
413            + state.c64_pool.len()
414            + state.c32_pool.len()
415    }
416
417    /// Number of retained capacity bins that hold no buffer.
418    #[cfg(test)]
419    pub(crate) fn empty_bin_count(&self) -> usize {
420        let state = lock_pool(&self.state);
421        empty_bins(&state.f64_pool)
422            + empty_bins(&state.f32_pool)
423            + empty_bins(&state.i32_pool)
424            + empty_bins(&state.i64_pool)
425            + empty_bins(&state.bool_pool)
426            + empty_bins(&state.c64_pool)
427            + empty_bins(&state.c32_pool)
428    }
429
430    /// Create an empty typed buffer pool.
431    ///
432    /// # Examples
433    ///
434    /// ```rust
435    /// use tenferro_cpu_basic::buffer_pool::BufferPool;
436    ///
437    /// let pool = BufferPool::new();
438    /// assert!(pool.is_empty());
439    /// ```
440    pub fn new() -> Self {
441        Self::with_max_retained_capacity_bytes(default_max_retained_capacity_bytes())
442    }
443
444    /// Create an empty typed buffer pool with a specific retention cap.
445    ///
446    /// A cap of zero disables retention. Use [`BufferPool::unbounded`] only for
447    /// diagnostics or workloads that are externally memory-limited.
448    ///
449    /// # Examples
450    ///
451    /// ```rust
452    /// use tenferro_cpu_basic::buffer_pool::BufferPool;
453    ///
454    /// let pool = BufferPool::with_max_retained_capacity_bytes(1024);
455    /// assert_eq!(pool.max_retained_capacity_bytes(), 1024);
456    /// ```
457    pub fn with_max_retained_capacity_bytes(max_retained_capacity_bytes: usize) -> Self {
458        Self {
459            state: Arc::new(SharedPool(Mutex::new(PoolState {
460                f64_pool: BTreeMap::new(),
461                f32_pool: BTreeMap::new(),
462                i32_pool: BTreeMap::new(),
463                i64_pool: BTreeMap::new(),
464                bool_pool: BTreeMap::new(),
465                c64_pool: BTreeMap::new(),
466                c32_pool: BTreeMap::new(),
467                retained_capacity_bytes: 0,
468                max_retained_capacity_bytes,
469            }))),
470        }
471    }
472
473    /// Create an empty typed buffer pool without a retention cap.
474    ///
475    /// This preserves the historical behavior and is mainly useful for
476    /// diagnostics or controlled benchmarks.
477    ///
478    /// # Examples
479    ///
480    /// ```rust
481    /// use tenferro_cpu_basic::buffer_pool::BufferPool;
482    ///
483    /// let pool = BufferPool::unbounded();
484    /// assert_eq!(pool.max_retained_capacity_bytes(), usize::MAX);
485    /// ```
486    pub fn unbounded() -> Self {
487        Self::with_max_retained_capacity_bytes(usize::MAX)
488    }
489
490    /// Maximum retained typed host-buffer capacity in bytes.
491    ///
492    /// # Examples
493    ///
494    /// ```rust
495    /// use tenferro_cpu_basic::buffer_pool::BufferPool;
496    ///
497    /// let pool = BufferPool::with_max_retained_capacity_bytes(4096);
498    /// assert_eq!(pool.max_retained_capacity_bytes(), 4096);
499    /// ```
500    pub fn max_retained_capacity_bytes(&self) -> usize {
501        lock_pool(&self.state).max_retained_capacity_bytes
502    }
503
504    /// Update the maximum retained typed host-buffer capacity in bytes.
505    ///
506    /// Shrinking below the currently retained capacity immediately evicts
507    /// retained buffers until the new cap is satisfied. A cap of zero disables
508    /// retention.
509    ///
510    /// # Examples
511    ///
512    /// ```
513    /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
514    ///
515    /// let mut pool = BufferPool::with_max_retained_capacity_bytes(1024);
516    /// <f64 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(128));
517    /// pool.set_max_retained_capacity_bytes(0);
518    /// assert_eq!(pool.max_retained_capacity_bytes(), 0);
519    /// assert!(pool.is_empty());
520    /// ```
521    pub fn set_max_retained_capacity_bytes(&mut self, max_retained_capacity_bytes: usize) {
522        let mut state = lock_pool(&self.state);
523        state.max_retained_capacity_bytes = max_retained_capacity_bytes;
524        state.enforce_retention_limit();
525    }
526
527    /// Number of retained buffers across all typed pools.
528    ///
529    /// # Examples
530    ///
531    /// ```rust
532    /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
533    ///
534    /// let mut pool = BufferPool::new();
535    /// <f64 as PoolScalar>::pool_release(&mut pool, vec![0.0; 2]);
536    /// assert_eq!(pool.len(), 1);
537    /// ```
538    pub fn len(&self) -> usize {
539        self.stats().buffers
540    }
541
542    /// Total retained typed host-buffer capacity in bytes.
543    ///
544    /// This counts capacity that is still live in the pool. The operating
545    /// system RSS may remain high after clearing the pool because the process
546    /// allocator can keep freed pages for future allocations.
547    ///
548    /// # Examples
549    ///
550    /// ```rust
551    /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
552    ///
553    /// let mut pool = BufferPool::new();
554    /// <f64 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(2));
555    /// assert_eq!(pool.retained_capacity_bytes(), 16);
556    /// ```
557    pub fn retained_capacity_bytes(&self) -> usize {
558        self.stats().capacity_bytes
559    }
560
561    /// Snapshot retained-buffer count and capacity.
562    ///
563    /// # Examples
564    ///
565    /// ```rust
566    /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
567    ///
568    /// let mut pool = BufferPool::new();
569    /// <f32 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(4));
570    /// let stats = pool.stats();
571    /// assert_eq!(stats.buffers, 1);
572    /// assert_eq!(stats.capacity_bytes, 16);
573    /// ```
574    pub fn stats(&self) -> BufferPoolStats {
575        let state = lock_pool(&self.state);
576        BufferPoolStats {
577            buffers: pool_len(&state.f64_pool)
578                + pool_len(&state.f32_pool)
579                + pool_len(&state.i32_pool)
580                + pool_len(&state.i64_pool)
581                + pool_len(&state.bool_pool)
582                + pool_len(&state.c64_pool)
583                + pool_len(&state.c32_pool),
584            capacity_bytes: state.retained_capacity_bytes,
585        }
586    }
587
588    /// Return cache-style stats for the buffers retained by this pool.
589    ///
590    /// `entries` is the number of retained buffers, and `retained_bytes` is the
591    /// total retained vector capacity in bytes.
592    ///
593    /// # Examples
594    ///
595    /// ```
596    /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
597    ///
598    /// let mut pool = BufferPool::new();
599    /// <f32 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(4));
600    /// let stats = pool.cache_stats();
601    /// assert_eq!(stats.entries, 1);
602    /// assert_eq!(stats.retained_bytes, 16);
603    /// ```
604    pub fn cache_stats(&self) -> CacheStats {
605        let stats = self.stats();
606        CacheStats {
607            entries: stats.buffers,
608            retained_bytes: stats.capacity_bytes,
609            hits: 0,
610            misses: 0,
611            evictions: 0,
612            clears: 0,
613        }
614    }
615
616    /// Acquire a typed vector with length 0 and at least `cap` capacity.
617    ///
618    /// Returned buffers come from the typed pool when possible and are ready
619    /// for push-based population.
620    ///
621    /// # Examples
622    ///
623    /// ```rust
624    /// use tenferro_cpu_basic::buffer_pool::BufferPool;
625    ///
626    /// let mut pool = BufferPool::new();
627    /// let mut buf = pool.acquire_with_capacity::<f64>(4);
628    /// buf.extend_from_slice(&[1.0, 2.0]);
629    /// assert_eq!(buf.len(), 2);
630    /// assert!(buf.capacity() >= 4);
631    /// ```
632    pub fn acquire_with_capacity<T: PoolScalar>(&mut self, cap: usize) -> Vec<T> {
633        if cap == 0 {
634            return Vec::new();
635        }
636
637        let (data, _checkout) = <T as private::Sealed>::pool_acquire_uninit_tracked(self, cap)
638            .expect("validated typed pool capacity must be acquirable");
639        let mut data = ManuallyDrop::new(data);
640        let ptr = data.as_mut_ptr().cast::<T>();
641        let capacity = data.capacity();
642        // SAFETY: `MaybeUninit<T>` and `T` have identical layouts, and the
643        // returned vector has length zero, so no element is read before push.
644        unsafe { Vec::from_raw_parts(ptr, 0, capacity) }
645    }
646
647    /// Acquire a typed vector with length `len` initialized to zero.
648    ///
649    /// Use this only when the caller may read elements before overwriting the
650    /// entire buffer. Full-overwrite kernels should use
651    /// [`crate::PooledUninitOutput`] to avoid the initialization cost.
652    ///
653    /// # Examples
654    ///
655    /// ```rust
656    /// use tenferro_cpu_basic::buffer_pool::BufferPool;
657    ///
658    /// let mut pool = BufferPool::new();
659    /// let buf = pool.acquire_zeroed::<f32>(3);
660    /// assert_eq!(buf, vec![0.0, 0.0, 0.0]);
661    /// ```
662    pub fn acquire_zeroed<T: PoolScalar>(&mut self, len: usize) -> Vec<T> {
663        T::pool_acquire_zeroed(self, len)
664    }
665
666    /// Whether all typed pools are empty.
667    ///
668    /// # Examples
669    ///
670    /// ```rust
671    /// use tenferro_cpu_basic::buffer_pool::BufferPool;
672    ///
673    /// let pool = BufferPool::new();
674    /// assert!(pool.is_empty());
675    /// ```
676    pub fn is_empty(&self) -> bool {
677        self.len() == 0
678    }
679
680    /// Drop all retained buffers from the pool.
681    ///
682    /// This releases the vectors owned by the pool. The process allocator may
683    /// still keep freed pages mapped for reuse, so operating-system RSS is not
684    /// guaranteed to fall immediately.
685    ///
686    /// # Examples
687    ///
688    /// ```rust
689    /// use tenferro_cpu_basic::buffer_pool::{BufferPool, PoolScalar};
690    ///
691    /// let mut pool = BufferPool::new();
692    /// <f64 as PoolScalar>::pool_release(&mut pool, Vec::with_capacity(8));
693    /// pool.clear();
694    /// assert!(pool.is_empty());
695    /// ```
696    pub fn clear(&mut self) {
697        let mut state = lock_pool(&self.state);
698        state.clear();
699    }
700}
701
702// INVARIANT: no user callback or tensor destructor runs under this lock; it
703// protects only scalar Vec bins and their accounting, never execution admission.
704fn lock_pool(state: &SharedPool) -> MutexGuard<'_, PoolState> {
705    state
706        .0
707        .lock()
708        .unwrap_or_else(std::sync::PoisonError::into_inner)
709}
710
711impl PoolState {
712    fn clear(&mut self) {
713        self.f64_pool.clear();
714        self.f32_pool.clear();
715        self.i32_pool.clear();
716        self.i64_pool.clear();
717        self.bool_pool.clear();
718        self.c64_pool.clear();
719        self.c32_pool.clear();
720        self.retained_capacity_bytes = 0;
721    }
722
723    fn enforce_retention_limit(&mut self) {
724        while self.retained_capacity_bytes > self.max_retained_capacity_bytes {
725            let Some(evicted_bytes) = self.evict_smallest_retained_buffer() else {
726                self.retained_capacity_bytes = 0;
727                return;
728            };
729            if evicted_bytes == 0 {
730                if self.retained_capacity_bytes == 0 {
731                    self.retained_capacity_bytes = 0;
732                    return;
733                }
734                continue;
735            }
736            self.retained_capacity_bytes =
737                self.retained_capacity_bytes.saturating_sub(evicted_bytes);
738        }
739    }
740
741    fn evict_smallest_retained_buffer(&mut self) -> Option<usize> {
742        let candidates = [
743            smallest_pool_candidate(&self.f64_pool, TypedPoolKind::F64),
744            smallest_pool_candidate(&self.f32_pool, TypedPoolKind::F32),
745            smallest_pool_candidate(&self.i32_pool, TypedPoolKind::I32),
746            smallest_pool_candidate(&self.i64_pool, TypedPoolKind::I64),
747            smallest_pool_candidate(&self.bool_pool, TypedPoolKind::Bool),
748            smallest_pool_candidate(&self.c64_pool, TypedPoolKind::C64),
749            smallest_pool_candidate(&self.c32_pool, TypedPoolKind::C32),
750        ];
751        let (_, kind) = candidates
752            .into_iter()
753            .flatten()
754            .min_by_key(|(bytes, _)| *bytes)?;
755        match kind {
756            TypedPoolKind::F64 => evict_one_from_pool(&mut self.f64_pool),
757            TypedPoolKind::F32 => evict_one_from_pool(&mut self.f32_pool),
758            TypedPoolKind::I32 => evict_one_from_pool(&mut self.i32_pool),
759            TypedPoolKind::I64 => evict_one_from_pool(&mut self.i64_pool),
760            TypedPoolKind::Bool => evict_one_from_pool(&mut self.bool_pool),
761            TypedPoolKind::C64 => evict_one_from_pool(&mut self.c64_pool),
762            TypedPoolKind::C32 => evict_one_from_pool(&mut self.c32_pool),
763        }
764    }
765}
766
767fn default_max_retained_capacity_bytes() -> usize {
768    *DEFAULT_MAX_RETAINED_CAPACITY_FROM_ENV.get_or_init(|| {
769        parse_default_max_retained_capacity_bytes(env::var_os(BUFFER_POOL_MAX_RETAINED_BYTES_ENV))
770    })
771}
772
773fn parse_default_max_retained_capacity_bytes(value: Option<OsString>) -> usize {
774    value
775        .and_then(|value| value.into_string().ok())
776        .and_then(|value| value.parse().ok())
777        .unwrap_or(DEFAULT_MAX_RETAINED_CAPACITY_BYTES)
778}
779
780impl Default for BufferPool {
781    fn default() -> Self {
782        Self::new()
783    }
784}
785
786#[cfg(test)]
787mod tests;