Skip to main content

tenferro_ad/
extension.rs

1//! Eager AD support for out-of-tree extension primitives.
2
3use std::sync::Arc;
4
5use computegraph::GraphOperation;
6use tenferro_ops::std_tensor_op::StdTensorOp;
7use tenferro_runtime::{
8    Error, ErrorPhase, ExtensionModule, InputSignature, PrepareCapability, PrepareError,
9    PreparedOperationExecutorHandle, Result, Runtime, RuntimeConfigError,
10};
11use tenferro_tensor::{Tensor, TensorRead, TensorValue};
12
13use crate::eager::{
14    eager_capture_active, eager_grad_recording_enabled, execute_prepared_extension_on,
15    EagerRuntime, EagerSession, EagerTensor,
16};
17use tenferro_runtime::ExecutionEngine;
18
19pub use tenferro_runtime::extension::{
20    apply, ExtensionCacheKey, ExtensionCacheLimits, ExtensionCacheSelector, ExtensionCacheStore,
21    ExtensionExecutionContext, ExtensionFamilyId, ExtensionOp,
22};
23
24/// Closed backend kind selected by the eager runtime owner for an extension.
25///
26/// # Examples
27///
28/// ```rust
29/// use tenferro_ad::extension::{EagerExtensionBackendKind, EagerExtensionTarget};
30/// use tenferro_runtime::EngineId;
31///
32/// let target = EagerExtensionTarget {
33///     engine_id: EngineId::new("example.engine")?,
34///     backend_kind: EagerExtensionBackendKind::Cpu,
35/// };
36/// assert!(matches!(
37///     target.backend_kind,
38///     EagerExtensionBackendKind::Cpu
39/// ));
40/// assert_eq!(target.engine_id.as_str(), "example.engine");
41/// # Ok::<(), Box<dyn std::error::Error>>(())
42/// ```
43#[doc(hidden)]
44#[derive(Clone, Copy, Debug, Eq, PartialEq)]
45pub enum EagerExtensionBackendKind {
46    /// The eager runtime owns a CPU backend.
47    Cpu,
48    /// The eager runtime owns a CUDA backend.
49    #[cfg(feature = "cuda")]
50    Cuda,
51}
52
53/// Exact engine target selected by the eager runtime owner.
54///
55/// # Examples
56///
57/// ```rust
58/// use tenferro_ad::extension::{EagerExtensionBackendKind, EagerExtensionTarget};
59/// use tenferro_runtime::EngineId;
60///
61/// let target = EagerExtensionTarget {
62///     engine_id: EngineId::new("example.engine")?,
63///     backend_kind: EagerExtensionBackendKind::Cpu,
64/// };
65/// assert_eq!(target.backend_kind, EagerExtensionBackendKind::Cpu);
66/// # Ok::<(), Box<dyn std::error::Error>>(())
67/// ```
68#[doc(hidden)]
69#[derive(Clone, Debug, Eq, PartialEq)]
70pub struct EagerExtensionTarget {
71    /// Exact runtime engine selected for this eager context.
72    pub engine_id: tenferro_runtime::EngineId,
73    /// Closed backend kind selected for this eager context.
74    pub backend_kind: EagerExtensionBackendKind,
75}
76
77#[cfg(test)]
78mod tests;
79
80/// Validate an untracked eager input for in-place execution on `engine` and
81/// install the extension module the in-place kernel needs.
82///
83/// # Errors
84///
85/// Returns a runtime-state error for a tracked input or inside trace capture,
86/// and the runtime's typed error when the engine cannot accept the input or
87/// the module cannot be installed.
88#[doc(hidden)]
89pub fn prepare_eager_in_place_input(
90    engine: &mut dyn ExecutionEngine,
91    input: &EagerTensor,
92    family_id: &'static str,
93    module_factory: impl FnOnce(EagerExtensionTarget) -> Result<Arc<dyn ExtensionModule>>,
94) -> Result<()> {
95    if input.requires_grad || input.trace.is_some() || eager_capture_active() {
96        return Err(Error::runtime_state(
97            "eager in-place",
98            ErrorPhase::Execution,
99            "in-place execution requires an untracked value outside trace capture",
100        ));
101    }
102    let target = input.ctx.eager_extension_target(engine)?;
103    validate_eager_extension_input_signature(&input.ctx, &target, &[input.tensor_read()])?;
104    let module = module_factory(target.clone())?;
105    input
106        .ctx
107        .ensure_extension_module_for_engine(module, family_id, &target.engine_id)?;
108    Ok(())
109}
110
111/// Adopt a runtime value produced by an extension kernel as an untracked eager
112/// tensor of `ctx`.
113///
114/// # Examples
115///
116/// ```
117/// use tenferro_ad::{EagerRuntime, Tensor};
118/// use tenferro_tensor::TensorValue;
119///
120/// let ctx = EagerRuntime::new()?;
121/// let value = TensorValue::from_tensor(Tensor::from_vec_col_major(vec![1], vec![2.0_f64])?);
122/// let tensor = tenferro_ad::extension::adopt_untracked_eager_value(ctx, value)?;
123/// assert!(!tensor.tracks_grad());
124/// # Ok::<(), Box<dyn std::error::Error>>(())
125/// ```
126///
127/// # Errors
128///
129/// Returns [`Error::RuntimeState`] when the [`TensorValue`] cannot be
130/// transferred into its allocation group.
131#[must_use = "the adopted eager tensor carries the runtime value"]
132pub fn adopt_untracked_eager_value(
133    ctx: Arc<EagerRuntime>,
134    value: TensorValue,
135) -> Result<EagerTensor> {
136    EagerTensor::new_untracked_value_result(ctx, value)
137}
138
139/// Apply an extension op to eager tensors on `engine`, recording it for AD.
140///
141/// # Errors
142///
143/// Returns an input-count or context-mismatch error, a runtime error when no
144/// installed extension module serves the op on this engine, or the extension
145/// kernel's own typed error.
146pub fn apply_eager(
147    engine: &mut dyn ExecutionEngine,
148    op: Arc<dyn ExtensionOp>,
149    inputs: &[&EagerTensor],
150) -> Result<Vec<EagerTensor>> {
151    let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
152    ctx.with_eager_session(engine, |session| {
153        apply_eager_in_session(session, op, inputs)
154    })
155}
156
157/// Apply an extension op inside an eager session, preferring a prepared
158/// session executor and falling back to a compiled one-op program.
159fn apply_eager_in_session(
160    session: &mut EagerSession<'_>,
161    op: Arc<dyn ExtensionOp>,
162    inputs: &[&EagerTensor],
163) -> Result<Vec<EagerTensor>> {
164    let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
165    ensure_session_runtime(session, &ctx)?;
166    let std_op = StdTensorOp::Extension(Arc::clone(&op));
167    let input_reads: Vec<_> = inputs.iter().map(|tensor| tensor.tensor_read()).collect();
168    let outputs = match try_prepared_eager_extension(session, &std_op, &input_reads)? {
169        Some(outputs) => outputs,
170        None => ctx.exec_extension_outputs_read(session.engine(), &op, &input_reads)?,
171    };
172    finish_eager_extension_outputs(ctx, std_op, inputs, outputs, session)
173}
174
175/// Refuse `input` when it belongs to a different eager runtime than `session`.
176///
177/// Bridge for session entries implemented outside this crate, such as the
178/// consuming in-place FFT (owner: `tenferro-fft`, exception E01), which must
179/// return [`Error::ContextMismatch`] like every other session operation.
180///
181/// # Errors
182///
183/// Returns [`Error::ContextMismatch`] when `input` was created in another
184/// runtime.
185#[doc(hidden)]
186pub fn ensure_session_owns(session: &EagerSession<'_>, input: &EagerTensor) -> Result<()> {
187    ensure_session_runtime(session, &input.ctx)
188}
189
190fn ensure_session_runtime(session: &EagerSession<'_>, ctx: &Arc<EagerRuntime>) -> Result<()> {
191    if !Arc::ptr_eq(session.runtime(), ctx) {
192        return Err(Error::ContextMismatch {
193            lhs: session.runtime().id(),
194            rhs: ctx.id(),
195        });
196    }
197    Ok(())
198}
199
200/// Run a prepared extension executor on `engine`: through the session-capable
201/// entry when it has one, otherwise through the typed engine escape.
202fn execute_prepared_executor(
203    engine: &mut dyn ExecutionEngine,
204    executor: &dyn tenferro_runtime::PreparedOperationExecutor,
205    input_reads: &[TensorRead<'_>],
206) -> Result<Vec<Tensor>> {
207    if executor.supports_session() {
208        return execute_prepared_extension_on(engine, executor, input_reads);
209    }
210    let identity = engine.target().context_identity();
211    crate::eager::with_lent_extension_caches(engine, |engine, caches| {
212        let mut erased =
213            tenferro_runtime::ErasedExecutionContext::from_dyn(identity, engine.as_any_mut());
214        executor.execute(&mut erased, caches, input_reads)
215    })
216}
217
218fn try_prepared_eager_extension(
219    session: &mut EagerSession<'_>,
220    op: &StdTensorOp,
221    input_reads: &[TensorRead<'_>],
222) -> Result<Option<Vec<Tensor>>> {
223    let ctx = Arc::clone(session.runtime());
224    // An engine kind without an eager extension target falls back to the
225    // compiled one-op program.
226    let Ok(target) = ctx.eager_extension_target(session.engine()) else {
227        return Ok(None);
228    };
229    let Some(executor) = prepared_eager_extension_executor(&ctx, &target, op, input_reads)? else {
230        return Ok(None);
231    };
232    Ok(Some(execute_prepared_executor(
233        session.engine(),
234        executor.as_ref(),
235        input_reads,
236    )?))
237}
238
239fn prepared_eager_extension_executor(
240    ctx: &EagerRuntime,
241    target: &EagerExtensionTarget,
242    op: &StdTensorOp,
243    input_reads: &[TensorRead<'_>],
244) -> Result<Option<PreparedOperationExecutorHandle>> {
245    let StdTensorOp::Extension(ext) = op else {
246        return Ok(None);
247    };
248    let signature = InputSignature::from_reads(input_reads).map_err(|source| {
249        Error::runtime_state_source("extension::apply_eager", ErrorPhase::Execution, source)
250    })?;
251    let PrepareCapability::Prepared(plan) =
252        ctx.runtime()
253            .prepare_extension_immediate(&target.engine_id, ext.as_ref(), &signature)?
254    else {
255        return Ok(None);
256    };
257    Ok(plan.executor().cloned())
258}
259
260/// Install `module` and apply `op` on `engine`.
261///
262/// # Errors
263///
264/// As [`apply_eager`], plus the runtime's error when the module cannot be
265/// installed.
266#[doc(hidden)]
267pub fn apply_eager_with_extension_session(
268    engine: &mut dyn ExecutionEngine,
269    op: Arc<dyn ExtensionOp>,
270    inputs: &[&EagerTensor],
271    module: Arc<dyn ExtensionModule>,
272) -> Result<Vec<EagerTensor>> {
273    let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
274    ctx.install_extension_module(module)?;
275    apply_eager(engine, op, inputs)
276}
277
278/// Install the module `module_factory` builds for `engine`'s target and apply
279/// `op` on `engine`.
280///
281/// # Errors
282///
283/// As [`apply_eager`], plus input-ingress and module-installation errors.
284#[doc(hidden)]
285pub fn apply_eager_with_targeted_extension_session(
286    engine: &mut dyn ExecutionEngine,
287    op: Arc<dyn ExtensionOp>,
288    inputs: &[&EagerTensor],
289    module_factory: impl FnOnce(
290        EagerExtensionTarget,
291    ) -> tenferro_runtime::Result<Arc<dyn ExtensionModule>>,
292) -> Result<Vec<EagerTensor>> {
293    let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
294    let target = ctx.eager_extension_target(engine)?;
295    let input_reads: Vec<_> = inputs.iter().map(|tensor| tensor.tensor_read()).collect();
296    validate_eager_extension_input_signature(&ctx, &target, &input_reads)?;
297    let module = module_factory(target.clone())?;
298    ctx.ensure_extension_module_for_engine(module, op.family_id(), &target.engine_id)?;
299    apply_eager(engine, op, inputs)
300}
301
302/// [`apply_eager_with_targeted_extension_session`] inside an eager session.
303///
304/// # Errors
305///
306/// As [`apply_eager_with_targeted_extension_session`], plus a context mismatch
307/// when the inputs belong to another runtime.
308#[doc(hidden)]
309pub fn apply_eager_with_targeted_extension_in_session(
310    session: &mut EagerSession<'_>,
311    op: Arc<dyn ExtensionOp>,
312    inputs: &[&EagerTensor],
313    module_factory: impl FnOnce(
314        EagerExtensionTarget,
315    ) -> tenferro_runtime::Result<Arc<dyn ExtensionModule>>,
316) -> Result<Vec<EagerTensor>> {
317    let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
318    ensure_session_runtime(session, &ctx)?;
319    let target = ctx.eager_extension_target(session.engine())?;
320    let input_reads: Vec<_> = inputs.iter().map(|tensor| tensor.tensor_read()).collect();
321    validate_eager_extension_input_signature(&ctx, &target, &input_reads)?;
322    let module = module_factory(target.clone())?;
323    ctx.ensure_extension_module_for_engine(module, op.family_id(), &target.engine_id)?;
324    apply_eager_in_session(session, op, inputs)
325}
326
327pub(crate) fn validate_eager_extension_target(
328    runtime: &Runtime,
329    target: &EagerExtensionTarget,
330) -> Result<()> {
331    let snapshot = runtime.snapshot().map_err(|source| {
332        Error::runtime_state_source(
333            "extension::apply_eager_with_extension_session",
334            ErrorPhase::Execution,
335            source,
336        )
337    })?;
338    if snapshot.engine(&target.engine_id).is_none() {
339        return Err(Error::runtime_state_source(
340            "extension::apply_eager_with_extension_session",
341            ErrorPhase::Execution,
342            RuntimeConfigError::MissingEngine {
343                engine_id: target.engine_id.clone(),
344            },
345        ));
346    }
347    Ok(())
348}
349
350fn validate_eager_extension_input_signature(
351    ctx: &EagerRuntime,
352    target: &EagerExtensionTarget,
353    input_reads: &[TensorRead<'_>],
354) -> Result<()> {
355    let signature = InputSignature::from_reads(input_reads).map_err(|source| {
356        Error::runtime_state_source(
357            "extension::apply_eager_with_extension_session",
358            ErrorPhase::Execution,
359            source,
360        )
361    })?;
362    let snapshot = ctx.runtime().snapshot().map_err(|source| {
363        Error::runtime_state_source(
364            "extension::apply_eager_with_extension_session",
365            ErrorPhase::Execution,
366            source,
367        )
368    })?;
369    let engine = snapshot.engine(&target.engine_id).ok_or_else(|| {
370        Error::runtime_state_source(
371            "extension::apply_eager_with_extension_session",
372            ErrorPhase::Execution,
373            RuntimeConfigError::MissingEngine {
374                engine_id: target.engine_id.clone(),
375            },
376        )
377    })?;
378    for (input_index, entry) in signature.entries().iter().enumerate() {
379        if !engine.accepts_input_signature(entry) {
380            return Err(Error::runtime_state_source(
381                "extension::apply_eager_with_extension_session",
382                ErrorPhase::Execution,
383                PrepareError::NoInputIngress {
384                    input_index,
385                    placement: entry.placement().clone(),
386                },
387            ));
388        }
389    }
390    Ok(())
391}
392
393fn validate_eager_extension_inputs(
394    op: &dyn ExtensionOp,
395    inputs: &[&EagerTensor],
396) -> Result<Arc<EagerRuntime>> {
397    let Some(first) = inputs.first() else {
398        return Err(Error::invalid_argument(
399            "extension::apply_eager",
400            ErrorPhase::Execution,
401            "inputs",
402            "at least one input tensor is required",
403        ));
404    };
405    if inputs.len() != op.input_count() {
406        return Err(Error::invalid_argument(
407            "extension::apply_eager",
408            ErrorPhase::Execution,
409            "inputs",
410            format!(
411                "op family {:?} expects {} inputs, got {}",
412                op.family_id(),
413                op.input_count(),
414                inputs.len()
415            ),
416        ));
417    }
418
419    let ctx = Arc::clone(&first.ctx);
420    for tensor in inputs.iter().skip(1) {
421        if !first.same_context(tensor) {
422            return Err(Error::ContextMismatch {
423                lhs: first.ctx_id(),
424                rhs: tensor.ctx_id(),
425            });
426        }
427    }
428    Ok(ctx)
429}
430
431fn finish_eager_extension_outputs(
432    ctx: Arc<EagerRuntime>,
433    op: StdTensorOp,
434    inputs: &[&EagerTensor],
435    outputs: Vec<Tensor>,
436    session: &mut EagerSession<'_>,
437) -> Result<Vec<EagerTensor>> {
438    if outputs.len() != op.output_count() {
439        return Err(Error::Internal(format!(
440            "expected {} eager outputs for {:?}, got {}",
441            op.output_count(),
442            op,
443            outputs.len()
444        )));
445    }
446
447    if !eager_grad_recording_enabled()
448        || (!eager_capture_active() && !inputs.iter().any(|input| input.requires_grad))
449    {
450        return outputs
451            .into_iter()
452            .map(|output| EagerTensor::new_untracked_result(Arc::clone(&ctx), output))
453            .collect();
454    }
455
456    let output_refs: Vec<&Tensor> = outputs.iter().collect();
457    let recorded = session.record_outputs(&op, &output_refs, inputs)?;
458    if recorded.traces.len() != outputs.len() {
459        return Err(Error::Internal(format!(
460            "expected {} eager traces for {:?}, got {}",
461            outputs.len(),
462            op,
463            recorded.traces.len()
464        )));
465    }
466    let results = recorded
467        .traces
468        .into_iter()
469        .zip(recorded.semantic_values)
470        .zip(outputs)
471        .map(|((trace, semantic), output)| {
472            if trace.requires_grad {
473                EagerTensor::new_result_with_semantic(
474                    Arc::clone(&ctx),
475                    trace.key,
476                    output,
477                    trace.requires_grad,
478                    trace.trace,
479                    semantic,
480                )
481            } else {
482                EagerTensor::new_unregistered_result_with_semantic(
483                    Arc::clone(&ctx),
484                    trace.key,
485                    output,
486                    trace.requires_grad,
487                    trace.trace,
488                    semantic,
489                )
490            }
491        })
492        .collect::<Result<Vec<_>>>()?;
493    crate::eager::finish_residuals(&op, inputs, &results.iter().collect::<Vec<_>>())?;
494    Ok(results)
495}