1use std::sync::Arc;
4
5use computegraph::GraphOperation;
6use tenferro_ops::std_tensor_op::StdTensorOp;
7use tenferro_runtime::{
8 Error, ErrorPhase, ExtensionModule, InputSignature, PrepareCapability, PrepareError,
9 PreparedOperationExecutorHandle, Result, Runtime, RuntimeConfigError,
10};
11use tenferro_tensor::{Tensor, TensorRead, TensorValue};
12
13use crate::eager::{
14 eager_capture_active, eager_grad_recording_enabled, execute_prepared_extension_on,
15 EagerRuntime, EagerSession, EagerTensor,
16};
17use tenferro_runtime::ExecutionEngine;
18
19pub use tenferro_runtime::extension::{
20 apply, ExtensionCacheKey, ExtensionCacheLimits, ExtensionCacheSelector, ExtensionCacheStore,
21 ExtensionExecutionContext, ExtensionFamilyId, ExtensionOp,
22};
23
24#[doc(hidden)]
44#[derive(Clone, Copy, Debug, Eq, PartialEq)]
45pub enum EagerExtensionBackendKind {
46 Cpu,
48 #[cfg(feature = "cuda")]
50 Cuda,
51}
52
53#[doc(hidden)]
69#[derive(Clone, Debug, Eq, PartialEq)]
70pub struct EagerExtensionTarget {
71 pub engine_id: tenferro_runtime::EngineId,
73 pub backend_kind: EagerExtensionBackendKind,
75}
76
77#[cfg(test)]
78mod tests;
79
80#[doc(hidden)]
89pub fn prepare_eager_in_place_input(
90 engine: &mut dyn ExecutionEngine,
91 input: &EagerTensor,
92 family_id: &'static str,
93 module_factory: impl FnOnce(EagerExtensionTarget) -> Result<Arc<dyn ExtensionModule>>,
94) -> Result<()> {
95 if input.requires_grad || input.trace.is_some() || eager_capture_active() {
96 return Err(Error::runtime_state(
97 "eager in-place",
98 ErrorPhase::Execution,
99 "in-place execution requires an untracked value outside trace capture",
100 ));
101 }
102 let target = input.ctx.eager_extension_target(engine)?;
103 validate_eager_extension_input_signature(&input.ctx, &target, &[input.tensor_read()])?;
104 let module = module_factory(target.clone())?;
105 input
106 .ctx
107 .ensure_extension_module_for_engine(module, family_id, &target.engine_id)?;
108 Ok(())
109}
110
111#[must_use = "the adopted eager tensor carries the runtime value"]
132pub fn adopt_untracked_eager_value(
133 ctx: Arc<EagerRuntime>,
134 value: TensorValue,
135) -> Result<EagerTensor> {
136 EagerTensor::new_untracked_value_result(ctx, value)
137}
138
139pub fn apply_eager(
147 engine: &mut dyn ExecutionEngine,
148 op: Arc<dyn ExtensionOp>,
149 inputs: &[&EagerTensor],
150) -> Result<Vec<EagerTensor>> {
151 let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
152 ctx.with_eager_session(engine, |session| {
153 apply_eager_in_session(session, op, inputs)
154 })
155}
156
157fn apply_eager_in_session(
160 session: &mut EagerSession<'_>,
161 op: Arc<dyn ExtensionOp>,
162 inputs: &[&EagerTensor],
163) -> Result<Vec<EagerTensor>> {
164 let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
165 ensure_session_runtime(session, &ctx)?;
166 let std_op = StdTensorOp::Extension(Arc::clone(&op));
167 let input_reads: Vec<_> = inputs.iter().map(|tensor| tensor.tensor_read()).collect();
168 let outputs = match try_prepared_eager_extension(session, &std_op, &input_reads)? {
169 Some(outputs) => outputs,
170 None => ctx.exec_extension_outputs_read(session.engine(), &op, &input_reads)?,
171 };
172 finish_eager_extension_outputs(ctx, std_op, inputs, outputs, session)
173}
174
175#[doc(hidden)]
186pub fn ensure_session_owns(session: &EagerSession<'_>, input: &EagerTensor) -> Result<()> {
187 ensure_session_runtime(session, &input.ctx)
188}
189
190fn ensure_session_runtime(session: &EagerSession<'_>, ctx: &Arc<EagerRuntime>) -> Result<()> {
191 if !Arc::ptr_eq(session.runtime(), ctx) {
192 return Err(Error::ContextMismatch {
193 lhs: session.runtime().id(),
194 rhs: ctx.id(),
195 });
196 }
197 Ok(())
198}
199
200fn execute_prepared_executor(
203 engine: &mut dyn ExecutionEngine,
204 executor: &dyn tenferro_runtime::PreparedOperationExecutor,
205 input_reads: &[TensorRead<'_>],
206) -> Result<Vec<Tensor>> {
207 if executor.supports_session() {
208 return execute_prepared_extension_on(engine, executor, input_reads);
209 }
210 let identity = engine.target().context_identity();
211 crate::eager::with_lent_extension_caches(engine, |engine, caches| {
212 let mut erased =
213 tenferro_runtime::ErasedExecutionContext::from_dyn(identity, engine.as_any_mut());
214 executor.execute(&mut erased, caches, input_reads)
215 })
216}
217
218fn try_prepared_eager_extension(
219 session: &mut EagerSession<'_>,
220 op: &StdTensorOp,
221 input_reads: &[TensorRead<'_>],
222) -> Result<Option<Vec<Tensor>>> {
223 let ctx = Arc::clone(session.runtime());
224 let Ok(target) = ctx.eager_extension_target(session.engine()) else {
227 return Ok(None);
228 };
229 let Some(executor) = prepared_eager_extension_executor(&ctx, &target, op, input_reads)? else {
230 return Ok(None);
231 };
232 Ok(Some(execute_prepared_executor(
233 session.engine(),
234 executor.as_ref(),
235 input_reads,
236 )?))
237}
238
239fn prepared_eager_extension_executor(
240 ctx: &EagerRuntime,
241 target: &EagerExtensionTarget,
242 op: &StdTensorOp,
243 input_reads: &[TensorRead<'_>],
244) -> Result<Option<PreparedOperationExecutorHandle>> {
245 let StdTensorOp::Extension(ext) = op else {
246 return Ok(None);
247 };
248 let signature = InputSignature::from_reads(input_reads).map_err(|source| {
249 Error::runtime_state_source("extension::apply_eager", ErrorPhase::Execution, source)
250 })?;
251 let PrepareCapability::Prepared(plan) =
252 ctx.runtime()
253 .prepare_extension_immediate(&target.engine_id, ext.as_ref(), &signature)?
254 else {
255 return Ok(None);
256 };
257 Ok(plan.executor().cloned())
258}
259
260#[doc(hidden)]
267pub fn apply_eager_with_extension_session(
268 engine: &mut dyn ExecutionEngine,
269 op: Arc<dyn ExtensionOp>,
270 inputs: &[&EagerTensor],
271 module: Arc<dyn ExtensionModule>,
272) -> Result<Vec<EagerTensor>> {
273 let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
274 ctx.install_extension_module(module)?;
275 apply_eager(engine, op, inputs)
276}
277
278#[doc(hidden)]
285pub fn apply_eager_with_targeted_extension_session(
286 engine: &mut dyn ExecutionEngine,
287 op: Arc<dyn ExtensionOp>,
288 inputs: &[&EagerTensor],
289 module_factory: impl FnOnce(
290 EagerExtensionTarget,
291 ) -> tenferro_runtime::Result<Arc<dyn ExtensionModule>>,
292) -> Result<Vec<EagerTensor>> {
293 let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
294 let target = ctx.eager_extension_target(engine)?;
295 let input_reads: Vec<_> = inputs.iter().map(|tensor| tensor.tensor_read()).collect();
296 validate_eager_extension_input_signature(&ctx, &target, &input_reads)?;
297 let module = module_factory(target.clone())?;
298 ctx.ensure_extension_module_for_engine(module, op.family_id(), &target.engine_id)?;
299 apply_eager(engine, op, inputs)
300}
301
302#[doc(hidden)]
309pub fn apply_eager_with_targeted_extension_in_session(
310 session: &mut EagerSession<'_>,
311 op: Arc<dyn ExtensionOp>,
312 inputs: &[&EagerTensor],
313 module_factory: impl FnOnce(
314 EagerExtensionTarget,
315 ) -> tenferro_runtime::Result<Arc<dyn ExtensionModule>>,
316) -> Result<Vec<EagerTensor>> {
317 let ctx = validate_eager_extension_inputs(op.as_ref(), inputs)?;
318 ensure_session_runtime(session, &ctx)?;
319 let target = ctx.eager_extension_target(session.engine())?;
320 let input_reads: Vec<_> = inputs.iter().map(|tensor| tensor.tensor_read()).collect();
321 validate_eager_extension_input_signature(&ctx, &target, &input_reads)?;
322 let module = module_factory(target.clone())?;
323 ctx.ensure_extension_module_for_engine(module, op.family_id(), &target.engine_id)?;
324 apply_eager_in_session(session, op, inputs)
325}
326
327pub(crate) fn validate_eager_extension_target(
328 runtime: &Runtime,
329 target: &EagerExtensionTarget,
330) -> Result<()> {
331 let snapshot = runtime.snapshot().map_err(|source| {
332 Error::runtime_state_source(
333 "extension::apply_eager_with_extension_session",
334 ErrorPhase::Execution,
335 source,
336 )
337 })?;
338 if snapshot.engine(&target.engine_id).is_none() {
339 return Err(Error::runtime_state_source(
340 "extension::apply_eager_with_extension_session",
341 ErrorPhase::Execution,
342 RuntimeConfigError::MissingEngine {
343 engine_id: target.engine_id.clone(),
344 },
345 ));
346 }
347 Ok(())
348}
349
350fn validate_eager_extension_input_signature(
351 ctx: &EagerRuntime,
352 target: &EagerExtensionTarget,
353 input_reads: &[TensorRead<'_>],
354) -> Result<()> {
355 let signature = InputSignature::from_reads(input_reads).map_err(|source| {
356 Error::runtime_state_source(
357 "extension::apply_eager_with_extension_session",
358 ErrorPhase::Execution,
359 source,
360 )
361 })?;
362 let snapshot = ctx.runtime().snapshot().map_err(|source| {
363 Error::runtime_state_source(
364 "extension::apply_eager_with_extension_session",
365 ErrorPhase::Execution,
366 source,
367 )
368 })?;
369 let engine = snapshot.engine(&target.engine_id).ok_or_else(|| {
370 Error::runtime_state_source(
371 "extension::apply_eager_with_extension_session",
372 ErrorPhase::Execution,
373 RuntimeConfigError::MissingEngine {
374 engine_id: target.engine_id.clone(),
375 },
376 )
377 })?;
378 for (input_index, entry) in signature.entries().iter().enumerate() {
379 if !engine.accepts_input_signature(entry) {
380 return Err(Error::runtime_state_source(
381 "extension::apply_eager_with_extension_session",
382 ErrorPhase::Execution,
383 PrepareError::NoInputIngress {
384 input_index,
385 placement: entry.placement().clone(),
386 },
387 ));
388 }
389 }
390 Ok(())
391}
392
393fn validate_eager_extension_inputs(
394 op: &dyn ExtensionOp,
395 inputs: &[&EagerTensor],
396) -> Result<Arc<EagerRuntime>> {
397 let Some(first) = inputs.first() else {
398 return Err(Error::invalid_argument(
399 "extension::apply_eager",
400 ErrorPhase::Execution,
401 "inputs",
402 "at least one input tensor is required",
403 ));
404 };
405 if inputs.len() != op.input_count() {
406 return Err(Error::invalid_argument(
407 "extension::apply_eager",
408 ErrorPhase::Execution,
409 "inputs",
410 format!(
411 "op family {:?} expects {} inputs, got {}",
412 op.family_id(),
413 op.input_count(),
414 inputs.len()
415 ),
416 ));
417 }
418
419 let ctx = Arc::clone(&first.ctx);
420 for tensor in inputs.iter().skip(1) {
421 if !first.same_context(tensor) {
422 return Err(Error::ContextMismatch {
423 lhs: first.ctx_id(),
424 rhs: tensor.ctx_id(),
425 });
426 }
427 }
428 Ok(ctx)
429}
430
431fn finish_eager_extension_outputs(
432 ctx: Arc<EagerRuntime>,
433 op: StdTensorOp,
434 inputs: &[&EagerTensor],
435 outputs: Vec<Tensor>,
436 session: &mut EagerSession<'_>,
437) -> Result<Vec<EagerTensor>> {
438 if outputs.len() != op.output_count() {
439 return Err(Error::Internal(format!(
440 "expected {} eager outputs for {:?}, got {}",
441 op.output_count(),
442 op,
443 outputs.len()
444 )));
445 }
446
447 if !eager_grad_recording_enabled()
448 || (!eager_capture_active() && !inputs.iter().any(|input| input.requires_grad))
449 {
450 return outputs
451 .into_iter()
452 .map(|output| EagerTensor::new_untracked_result(Arc::clone(&ctx), output))
453 .collect();
454 }
455
456 let output_refs: Vec<&Tensor> = outputs.iter().collect();
457 let recorded = session.record_outputs(&op, &output_refs, inputs)?;
458 if recorded.traces.len() != outputs.len() {
459 return Err(Error::Internal(format!(
460 "expected {} eager traces for {:?}, got {}",
461 outputs.len(),
462 op,
463 recorded.traces.len()
464 )));
465 }
466 let results = recorded
467 .traces
468 .into_iter()
469 .zip(recorded.semantic_values)
470 .zip(outputs)
471 .map(|((trace, semantic), output)| {
472 if trace.requires_grad {
473 EagerTensor::new_result_with_semantic(
474 Arc::clone(&ctx),
475 trace.key,
476 output,
477 trace.requires_grad,
478 trace.trace,
479 semantic,
480 )
481 } else {
482 EagerTensor::new_unregistered_result_with_semantic(
483 Arc::clone(&ctx),
484 trace.key,
485 output,
486 trace.requires_grad,
487 trace.trace,
488 semantic,
489 )
490 }
491 })
492 .collect::<Result<Vec<_>>>()?;
493 crate::eager::finish_residuals(&op, inputs, &results.iter().collect::<Vec<_>>())?;
494 Ok(results)
495}