Skip to main content

tenferro_runtime/runtime/
execution.rs

1//! Runtime-owned compiled-graph execution boundary.
2//!
3//! [`crate::Runtime::prepare`] turns a compiled graph into a [`Prepared`]
4//! program for a set of engine targets, and [`crate::Runtime::run`] runs it on
5//! the engines the caller passes in. The runtime holds no engine and no
6//! execution state between calls.
7
8use num_complex::{Complex32, Complex64};
9use std::collections::{HashMap, HashSet};
10use std::error::Error as StdError;
11use std::fmt;
12use std::panic::AssertUnwindSafe;
13use std::sync::Arc;
14
15use tenferro_ops::shape_extent::ShapeExtent;
16use tenferro_tensor::{DType, Engine, Tensor, TensorRead};
17use tinyvec::TinyVec;
18
19use crate::error::ErrorPhase;
20use crate::exec::{ExecInstruction, ExecProgram, ExecSlot, ExtensionExecutionDispatch};
21use crate::extension_cache::ExtensionCacheStore;
22use crate::graph::CompiledGraph;
23use crate::runtime::schedule::{
24    EventDependency, ExecutionLocation, ScheduledGraph, ScheduledNode, ScheduledNodeKind,
25    ScheduledTransfer, UnsupportedScheduledNodeError,
26};
27use crate::runtime::{
28    EngineId, EngineTarget, EventDomainDriver, EventDomainError, EventDomainOperation,
29    EventDomainRun, EventToken, ExecutionEngine, InputSignature, Output, PrepareError,
30    PrepareOptions, PreparedOperationPlan, ProgramPlacementConstraint, Runtime, TransferError,
31    TransferProviderContractError, TransferRequest,
32};
33use crate::{Error, Result};
34
35// `TensorRead` has no `Default`, so `tinyvec::Array` cannot hold it inline.
36// The run's input list is a `Vec` that is moved, not copied, into the
37// execution slots (`RunSlots::initialize` maps it to `ExecSlot::Read` without
38// collecting), so it is the only per-run input-list allocation.
39type RuntimeInputReads<'a> = Vec<TensorRead<'a>>;
40type RuntimeInputShapes<'a> = TinyVec<[&'a [usize]; 8]>;
41type RuntimeShapeScratch = TinyVec<[usize; 8]>;
42
43/// A compiled graph prepared for a set of engine targets.
44///
45/// A prepared program holds the execution plan and the requirements each
46/// target slot puts on its engine; it holds no engine, stream or event state,
47/// so any engines that report the same targets can run it, and several callers
48/// can run it at the same time on their own engines. It is tied to the runtime
49/// epoch that created it and becomes stale after runtime reconfiguration.
50///
51/// # Examples
52///
53/// ```
54/// use tenferro_runtime::Prepared;
55///
56/// fn target_count(prepared: &Prepared) -> usize {
57///     prepared.targets().len()
58/// }
59/// ```
60#[derive(Clone)]
61pub struct Prepared {
62    runtime_id: super::RuntimeId,
63    epoch: super::RuntimeEpoch,
64    program: CompiledGraph,
65    prepared: Arc<super::preparation::PreparedProgram>,
66    /// Whether this handle's preparation ran the metadata check on the graph's
67    /// *retained* inputs, i.e. on the values an empty-input run would reuse.
68    ///
69    /// Preparation that replaced every input validated the replacements only,
70    /// so its prepared handle must re-check the defaults at execution time.
71    defaults_validated: bool,
72    /// The engine targets, in slot order, that a run must bind.
73    targets: Arc<[EngineTarget]>,
74    /// The input signature this handle was prepared for.
75    ///
76    /// A run whose inputs produce the same signature is exactly the run the
77    /// preparation validated: the projection, the arity, and the shape guards
78    /// all passed on this signature then, so re-deriving and re-comparing the
79    /// specialization per run is redundant (#2186).
80    signature: Arc<super::InputSignature>,
81}
82
83impl fmt::Debug for Prepared {
84    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
85        formatter
86            .debug_struct("Prepared")
87            .field("runtime_id", &self.runtime_id)
88            .field("epoch", &self.epoch)
89            .field("targets", &self.targets)
90            .finish_non_exhaustive()
91    }
92}
93
94impl Prepared {
95    /// The engine targets this program was prepared for, in slot order.
96    ///
97    /// [`crate::Runtime::run`] binds slot `i` to the `i`-th engine it receives.
98    ///
99    /// # Examples
100    ///
101    /// ```
102    /// use tenferro_runtime::Prepared;
103    ///
104    /// fn first_engine(prepared: &Prepared) -> Option<&str> {
105    ///     prepared.targets().first().map(|target| target.engine_id().as_str())
106    /// }
107    /// ```
108    #[must_use]
109    pub fn targets(&self) -> &[EngineTarget] {
110        &self.targets
111    }
112
113    /// How executions must treat this handle's retained inputs.
114    ///
115    /// Skipping the check is sound only when preparation itself ran it on those
116    /// values; preparation with a complete replacement input list did not.
117    fn bound_input_check(&self) -> BoundInputCheck {
118        if self.defaults_validated {
119            BoundInputCheck::AlreadyValidatedAtPrepare
120        } else {
121            BoundInputCheck::Validate
122        }
123    }
124
125    /// Number of planned elementwise regions and the instructions they cover,
126    /// for execution-path parity tests.
127    #[doc(hidden)]
128    #[must_use]
129    pub fn elementwise_region_summary(&self) -> (usize, usize) {
130        super::region::region_summary(self.prepared.root().regions())
131    }
132
133    /// Number of planned elementwise regions executed as one fused command, and
134    /// the number that fell back to per-instruction dispatch.
135    #[doc(hidden)]
136    #[must_use]
137    pub fn elementwise_region_execution_counts(&self) -> (usize, usize) {
138        let counters = self.prepared.root().region_counters();
139        (counters.fused(), counters.fallbacks())
140    }
141
142    /// Number of successful scheduled event-domain submissions observed by this
143    /// prepared program's production executor.
144    ///
145    /// This is runtime submission evidence, rather than a plan-derived census.
146    #[doc(hidden)]
147    #[must_use]
148    pub fn execution_submission_count(&self) -> usize {
149        self.prepared.root().region_counters().submissions()
150    }
151
152    /// Plan-derived command count for the production execution path, for
153    /// execution-path tests: scheduled operations that no region covers, plus
154    /// one command per planned region.
155    ///
156    /// This is a plan census, not a runtime submission count.
157    #[doc(hidden)]
158    #[must_use]
159    pub fn execution_command_count(&self) -> usize {
160        let root = self.prepared.root();
161        super::region::command_count(root.schedule(), root.regions())
162    }
163}
164
165/// Run an extension callback, marking the engine broken if it unwinds.
166fn run_extension_callback<R>(
167    engine: &mut dyn ExecutionEngine,
168    callback: impl FnOnce(&mut dyn ExecutionEngine) -> Result<R>,
169) -> Result<R> {
170    match std::panic::catch_unwind(AssertUnwindSafe(|| callback(engine))) {
171        Ok(result) => result,
172        Err(payload) => {
173            engine.mark_broken();
174            std::panic::resume_unwind(payload)
175        }
176    }
177}
178
179/// Run one scheduled instruction on `exec`.
180///
181/// An extension instruction is the only one that borrows the engine's
182/// extension caches; it takes the owner fallback when its executor cannot run
183/// in the engine's session. Every other instruction executes through the
184/// engine's typed operations and never touches the caches, so a program
185/// without extension commands lends none (#2226).
186fn execute_slot_instruction_on_engine<'input>(
187    engine: &mut dyn ExecutionEngine,
188    extension_caches: Option<&mut ExtensionCacheStore>,
189    operations: &[PreparedOperationPlan],
190    instruction_index: usize,
191    instruction: &ExecInstruction,
192    slots: &mut [Option<ExecSlot<'input>>],
193) -> Result<()> {
194    if matches!(&instruction.op, crate::exec::ExecOp::Extension(_)) {
195        let caches = extension_caches.ok_or_else(|| {
196            Error::runtime_state(
197                "Runtime::run",
198                ErrorPhase::Execution,
199                "engine extension caches were not taken before an extension command",
200            )
201        })?;
202        let mut extension_dispatch = ExtensionExecutionDispatch { operations, caches };
203        if crate::exec::needs_owner_extension_fallback(instruction, Some(&extension_dispatch)) {
204            // A prepared operation that executes against the concrete engine
205            // type instead of the engine operations receives the engine itself.
206            let identity = engine.target().context_identity();
207            run_extension_callback(engine, |engine| {
208                crate::exec::execute_owner_extension_fallback(
209                    engine.as_any_mut(),
210                    identity,
211                    slots,
212                    instruction,
213                    Some(&mut extension_dispatch),
214                )
215            })?;
216        } else {
217            run_extension_callback(engine, |engine| {
218                crate::exec::execute_ffi_instruction_exec(
219                    engine,
220                    slots,
221                    instruction,
222                    Some(instruction_index),
223                    Some(&mut extension_dispatch),
224                )
225            })?;
226        }
227        crate::exec::reclaim_last_use_inputs_exec(slots, instruction, engine);
228        return Ok(());
229    }
230    if crate::exec::is_host_instruction(instruction) {
231        crate::exec::execute_host_instruction_exec(engine, slots, instruction)?;
232    } else if crate::exec::is_ffi_instruction(instruction) {
233        crate::exec::execute_ffi_instruction_exec(
234            engine,
235            slots,
236            instruction,
237            Some(instruction_index),
238            None,
239        )?;
240    } else if let Some(view) = crate::exec::execute_backend_view_op(slots, instruction) {
241        slots[instruction.output_slots[0]] = Some(view);
242    } else {
243        let result = crate::exec::execute_backend_op(engine, slots, instruction)?;
244        slots[instruction.output_slots[0]] = Some(ExecSlot::Owned(result));
245    }
246    crate::exec::reclaim_last_use_inputs_exec(slots, instruction, engine);
247    Ok(())
248}
249
250/// Run one planned elementwise region on `exec`.
251fn execute_elementwise_fusion_slots_on_engine<'input>(
252    exec: &mut dyn Engine,
253    input_slots: &[usize],
254    instruction_count: usize,
255    plan: &tenferro_tensor::backend::ElementwiseFusionPlan,
256    slots: &mut [Option<ExecSlot<'input>>],
257    output_slots: &[usize],
258) -> Result<bool> {
259    // The fused entry point takes owned tensors. A borrowed view input is
260    // materialized once when the region is long enough to amortize that
261    // copy; otherwise the per-instruction path, which resolves views itself,
262    // stays in charge.
263    let mut views: Vec<usize> = Vec::new();
264    for &slot in input_slots {
265        let value = slots
266            .get(slot)
267            .and_then(Option::as_ref)
268            .ok_or_else(|| crate::Error::from(tenferro_tensor::Error::MissingValue { slot }))?;
269        if value.as_tensor("elementwise_region").is_err() {
270            views.push(slot);
271        }
272    }
273    if !views.is_empty() {
274        if instruction_count < super::region::VIEW_COPY_MIN_INSTRUCTIONS {
275            return Ok(false);
276        }
277        for &slot in &views {
278            let tensor = {
279                let read = slots[slot].as_ref().map(ExecSlot::as_read).ok_or_else(|| {
280                    crate::Error::from(tenferro_tensor::Error::MissingValue { slot })
281                })?;
282                exec.to_contiguous_read(read)?
283            };
284            slots[slot] = Some(ExecSlot::Owned(tensor));
285        }
286    }
287
288    let mut inputs: Vec<&Tensor> = Vec::with_capacity(input_slots.len());
289    for &slot in input_slots {
290        let value = slots
291            .get(slot)
292            .and_then(Option::as_ref)
293            .ok_or_else(|| crate::Error::from(tenferro_tensor::Error::MissingValue { slot }))?;
294        inputs.push(value.as_tensor("elementwise_region")?);
295    }
296
297    if !crate::segment::plan_matches_input_dtypes(plan, &inputs) {
298        return Ok(false);
299    }
300
301    let outputs = exec.execute_elementwise_fusion(&inputs, plan)?;
302    let Some(outputs) = outputs else {
303        return Ok(false);
304    };
305    if outputs.len() != output_slots.len() {
306        return Err(crate::Error::Internal(format!(
307            "fused elementwise region produced {} outputs for {} slots",
308            outputs.len(),
309            output_slots.len()
310        )));
311    }
312    for (&slot, tensor) in output_slots.iter().zip(outputs) {
313        slots[slot] = Some(ExecSlot::Owned(tensor));
314    }
315    Ok(true)
316}
317
318/// Prepare `program` for the engines described by `targets`.
319pub(super) fn prepare(
320    runtime: &Runtime,
321    program: &CompiledGraph,
322    inputs: &[TensorRead<'_>],
323    targets: &[EngineTarget],
324) -> Result<Prepared> {
325    const CALLER: &str = "Runtime::prepare";
326    if targets.is_empty() {
327        return Err(Error::invalid_argument(
328            CALLER,
329            ErrorPhase::Execution,
330            "targets",
331            "prepare needs at least one engine target",
332        ));
333    }
334    let snapshot = runtime
335        .snapshot()
336        .map_err(|source| Error::runtime_state_source(CALLER, ErrorPhase::Execution, source))?;
337    let mut engine_ids = Vec::with_capacity(targets.len());
338    for (index, target) in targets.iter().enumerate() {
339        if targets[..index]
340            .iter()
341            .any(|earlier| earlier.engine_id() == target.engine_id())
342        {
343            return Err(Error::invalid_argument(
344                CALLER,
345                ErrorPhase::Execution,
346                "targets",
347                format!(
348                    "engine {:?} is named by more than one target",
349                    target.engine_id().as_str()
350                ),
351            ));
352        }
353        let view = snapshot.engine(target.engine_id()).ok_or_else(|| {
354            Error::runtime_state(
355                CALLER,
356                ErrorPhase::Execution,
357                format!(
358                    "no engine {:?} is registered with this runtime",
359                    target.engine_id().as_str()
360                ),
361            )
362        })?;
363        if view.context_identity() != target.context_identity() {
364            return Err(Error::runtime_state(
365                CALLER,
366                ErrorPhase::Execution,
367                format!(
368                    "engine {:?} is registered for context {:?}, but the target reports {:?}",
369                    target.engine_id().as_str(),
370                    view.context_identity(),
371                    target.context_identity()
372                ),
373            ));
374        }
375        engine_ids.push(target.engine_id().clone());
376    }
377    let defaults_validated = retained_inputs_validated(program, inputs.len());
378    let inputs = resolve_input_reads(program, inputs, BoundInputCheck::Validate)?;
379    let signature = input_signature_reads(&inputs)?;
380    // An extension whose output extent was symbolic at compile time is re-inferred
381    // against the concrete signature here, so its own typed validation surfaces at
382    // preparation instead of at execution (#2434).
383    super::preparation::validate_resolved_extension_extents(program.frozen(), &signature)?;
384    let placement = ProgramPlacementConstraint::new(engine_ids, None)
385        .map_err(|source| Error::runtime_state_source(CALLER, ErrorPhase::Execution, source))?;
386    let options = PrepareOptions::new().with_placement(placement);
387    let prepared = runtime
388        .prepare_compiled_for(program, &signature, &options)
389        .map_err(prepare_error)?;
390    validate_prepared_epoch(runtime, prepared.root().epoch(), CALLER)?;
391    Ok(Prepared {
392        runtime_id: runtime.id(),
393        epoch: prepared.root().epoch(),
394        program: program.clone(),
395        prepared,
396        defaults_validated,
397        targets: targets.into(),
398        signature: Arc::new(signature),
399    })
400}
401
402/// Whether resolving `input_count` inputs makes [`resolve_input_reads`] check
403/// the graph's retained inputs.
404///
405/// Empty inputs resolve to the retained values, and a partial list still
406/// resolves every bound input, so both validate them. A list that covers every
407/// input replaces all of them, which leaves the retained values unchecked.
408fn retained_inputs_validated(program: &CompiledGraph, input_count: usize) -> bool {
409    let expected = program.program().inputs().len();
410    input_count != expected || expected == 0
411}
412
413/// Run `prepared` on the engines the caller passes in.
414pub(super) fn run<'a>(
415    runtime: &Runtime,
416    engines: &mut [&mut dyn ExecutionEngine],
417    prepared: &'a Prepared,
418    inputs: &[TensorRead<'a>],
419) -> Result<Vec<Output<'a>>> {
420    const CALLER: &str = "Runtime::run";
421    validate_prepared_runtime(runtime, prepared, CALLER)?;
422    let mut engines = EngineSet::bind(engines, &prepared.targets, CALLER)?;
423    let inputs = resolve_input_reads(&prepared.program, inputs, prepared.bound_input_check())?;
424    let signature = input_signature_reads(&inputs)?;
425    if *prepared.signature != signature {
426        super::preparation::validate_execution_signature(
427            prepared.prepared.as_ref(),
428            prepared.program.frozen(),
429            &signature,
430        )
431        .map_err(prepare_error)?;
432    }
433    let root = prepared.prepared.root();
434    engines.take_caches(root.needs_extension_caches());
435    let result = execute_scheduled(root, prepared.prepared.operations(), inputs, &mut engines);
436    engines.restore_caches();
437    result
438}
439
440fn validate_prepared_runtime(
441    runtime: &Runtime,
442    prepared: &Prepared,
443    caller: &'static str,
444) -> Result<()> {
445    if runtime.id() != prepared.runtime_id {
446        return Err(Error::runtime_state(
447            caller,
448            ErrorPhase::Execution,
449            "prepared program belongs to a different runtime",
450        ));
451    }
452    validate_prepared_epoch(runtime, prepared.epoch, caller)
453}
454
455fn validate_prepared_epoch(
456    runtime: &Runtime,
457    prepared_epoch: super::RuntimeEpoch,
458    caller: &'static str,
459) -> Result<()> {
460    let epoch = runtime
461        .epoch()
462        .map_err(|source| Error::runtime_state_source(caller, ErrorPhase::Execution, source))?;
463    if epoch != prepared_epoch {
464        return Err(Error::runtime_state(
465            caller,
466            ErrorPhase::Execution,
467            format!(
468                "prepared epoch {:?} does not match current epoch {:?}",
469                prepared_epoch, epoch
470            ),
471        ));
472    }
473    Ok(())
474}
475
476/// The engines of one run, bound to the prepared targets.
477///
478/// Slot `i` of the prepared program runs on `engines[i]`; [`Self::bind`]
479/// checks once, before any launch, that every engine reports the target it is
480/// bound to. Extension caches are moved out of the engines for the duration of
481/// the run so an instruction can borrow an engine and its caches together.
482struct EngineSet<'e, 'r> {
483    engines: &'e mut [&'r mut dyn ExecutionEngine],
484    targets: &'e [EngineTarget],
485    caches: Vec<ExtensionCacheStore>,
486}
487
488impl<'e, 'r> EngineSet<'e, 'r> {
489    fn bind(
490        engines: &'e mut [&'r mut dyn ExecutionEngine],
491        targets: &'e [EngineTarget],
492        caller: &'static str,
493    ) -> Result<Self> {
494        if engines.len() != targets.len() {
495            return Err(Error::invalid_argument(
496                caller,
497                ErrorPhase::Execution,
498                "engines",
499                format!(
500                    "the program was prepared for {} engines, but {} were passed",
501                    targets.len(),
502                    engines.len()
503                ),
504            ));
505        }
506        for (index, (engine, target)) in engines.iter().zip(targets).enumerate() {
507            let actual = engine.target();
508            if &actual != target {
509                return Err(Error::invalid_argument(
510                    caller,
511                    ErrorPhase::Execution,
512                    "engines",
513                    format!(
514                        "engine {index} reports target {actual:?}, but the program was prepared for {target:?}"
515                    ),
516                ));
517            }
518        }
519        Ok(Self {
520            engines,
521            targets,
522            caches: Vec::new(),
523        })
524    }
525
526    /// Lend every engine's extension caches to this run.
527    ///
528    /// Only a program with an extension command reads them; a run of any other
529    /// program leaves the caches in place and skips the exchange.
530    fn take_caches(&mut self, needed: bool) {
531        if needed {
532            self.caches = self
533                .engines
534                .iter_mut()
535                .map(|engine| engine.take_extension_caches())
536                .collect();
537        }
538    }
539
540    fn restore_caches(&mut self) {
541        for (engine, caches) in self.engines.iter_mut().zip(self.caches.drain(..)) {
542            engine.restore_extension_caches(caches);
543        }
544    }
545
546    fn index_of(&self, engine_id: &EngineId) -> Result<usize> {
547        self.targets
548            .iter()
549            .position(|target| target.engine_id() == engine_id)
550            .ok_or_else(|| {
551                Error::runtime_state(
552                    "Runtime::run",
553                    ErrorPhase::Execution,
554                    format!(
555                        "the prepared schedule uses engine {:?}, which is not among the run's engines",
556                        engine_id.as_str()
557                    ),
558                )
559            })
560    }
561
562    fn engine(&mut self, engine_id: &EngineId) -> Result<&mut dyn ExecutionEngine> {
563        let index = self.index_of(engine_id)?;
564        Ok(&mut *self.engines[index])
565    }
566
567    fn driver(&self, engine_id: &EngineId) -> Result<Arc<dyn EventDomainDriver>> {
568        let index = self.index_of(engine_id)?;
569        Ok(self.engines[index].event_domain_driver())
570    }
571
572    fn execute_slot_instruction<'input>(
573        &mut self,
574        location: &ExecutionLocation,
575        instruction_index: usize,
576        instruction: &ExecInstruction,
577        operations: &[PreparedOperationPlan],
578        slots: &mut [Option<ExecSlot<'input>>],
579    ) -> Result<()> {
580        let index = self.index_of(location.engine_id())?;
581        let engine = &mut *self.engines[index];
582        execute_slot_instruction_on_engine(
583            engine,
584            self.caches.get_mut(index),
585            operations,
586            instruction_index,
587            instruction,
588            slots,
589        )
590    }
591
592    fn execute_elementwise_fusion_slots<'input>(
593        &mut self,
594        location: &ExecutionLocation,
595        input_slots: &[usize],
596        instruction_count: usize,
597        plan: &tenferro_tensor::backend::ElementwiseFusionPlan,
598        slots: &mut [Option<ExecSlot<'input>>],
599        output_slots: &[usize],
600    ) -> Result<bool> {
601        let index = self.index_of(location.engine_id())?;
602        execute_elementwise_fusion_slots_on_engine(
603            &mut *self.engines[index],
604            input_slots,
605            instruction_count,
606            plan,
607            slots,
608            output_slots,
609        )
610    }
611
612    /// Hand every event-domain run to the engine that owns its domain.
613    fn retire(&mut self, event_domains: ScheduledEventDomains) -> Result<()> {
614        let mut failures = Vec::new();
615        for (engine_id, run) in event_domains.into_pending_runs() {
616            let result = match self.index_of(&engine_id) {
617                Ok(index) => self.engines[index].retire_run(run),
618                Err(error) => {
619                    drop_event_domain_run(run);
620                    Err(error)
621                }
622            };
623            if let Err(error) = result {
624                failures.push(error);
625            }
626        }
627        let mut failures = failures.into_iter();
628        let Some(mut error) = failures.next() else {
629            return Ok(());
630        };
631        for failure in failures {
632            error = Error::with_suppressed(error, failure);
633        }
634        Err(error)
635    }
636}
637
638impl Drop for EngineSet<'_, '_> {
639    fn drop(&mut self) {
640        self.restore_caches();
641    }
642}
643
644fn execute_scheduled<'input>(
645    root: &super::preparation::PreparedProgramRoot,
646    operations: &[PreparedOperationPlan],
647    inputs: RuntimeInputReads<'input>,
648    engines: &mut EngineSet<'_, '_>,
649) -> Result<Vec<Output<'input>>> {
650    let program = root.staging();
651    let schedule = root.schedule();
652    // The immutable program was validated when the root was prepared; only
653    // the run's own input list is checked here.
654    validate_exec_input_count(program, inputs.len())?;
655    let mut event_domains = Some(ScheduledEventDomains::new(schedule, engines)?);
656    let flat = schedule.flat_location();
657    let executed = execute_scheduled_slots_into(
658        program,
659        schedule,
660        flat,
661        root.regions(),
662        root.node_regions(),
663        root.region_counters(),
664        operations,
665        inputs,
666        engines,
667        &mut event_domains,
668    );
669    let Some(event_domains) = event_domains.take() else {
670        return executed.and_then(|_| {
671            Err(Error::runtime_state(
672                "Runtime::run",
673                ErrorPhase::Execution,
674                "scheduled execution consumed its event domains before collecting outputs",
675            ))
676        });
677    };
678    let collected = executed
679        .and_then(|slots| collect_outputs(program, slots, flat, root.output_duplicates(), engines));
680    match collected {
681        Ok(outputs) => {
682            engines.retire(event_domains)?;
683            Ok(outputs)
684        }
685        Err(primary) => {
686            // On failure, nothing the run produced is handed out; retire the
687            // run now so its device work cannot outlive the error.
688            let mut event_domains = event_domains;
689            match event_domains.drain() {
690                Ok(()) => Err(primary),
691                Err(cleanup) => Err(scheduled_execution_cleanup_error(primary, cleanup)),
692            }
693        }
694    }
695}
696
697/// Turn the run's output slots into [`Output`]s, in program output order.
698///
699/// A slot that still reads one of the run's inputs (the input itself or a view
700/// of it) is returned borrowed, without a copy. Owned results, which are
701/// always compact, are returned as they are. A program may name one output slot
702/// more than once (`duplicates`, computed at prepare): the first position takes
703/// the value, a repeated borrowed read is cloned shallowly, and a repeated
704/// owned result is an explicit engine copy on the value's location.
705fn collect_outputs<'input>(
706    program: &ExecProgram,
707    slots: RunSlots<'input>,
708    flat: Option<&ExecutionLocation>,
709    duplicates: &[Option<usize>],
710    engines: &mut EngineSet<'_, '_>,
711) -> Result<Vec<Output<'input>>> {
712    let RunSlots {
713        mut staged,
714        mut located,
715    } = slots;
716    let mut collected: Vec<Output<'input>> = Vec::with_capacity(program.output_slots.len());
717    // The location of each collected owned output, for a repeated position's
718    // engine copy; the flat run has one location for every value.
719    let mut locations: Vec<Option<ExecutionLocation>> = Vec::new();
720    for (position, &slot) in program.output_slots.iter().enumerate() {
721        if let Some(first) = duplicates.get(position).copied().flatten() {
722            let output = match collected.get(first) {
723                Some(Output::Borrowed(read)) => Output::Borrowed(read.clone()),
724                Some(Output::Owned(tensor)) => {
725                    let location = match flat {
726                        Some(location) => location,
727                        None => locations
728                            .get(first)
729                            .and_then(Option::as_ref)
730                            .ok_or(tenferro_tensor::Error::MissingValue { slot })?,
731                    };
732                    Output::Owned(duplicate_owned_output(engines, location, tensor)?)
733                }
734                None => return Err(tenferro_tensor::Error::MissingValue { slot }.into()),
735            };
736            collected.push(output);
737            if flat.is_none() {
738                locations.push(None);
739            }
740            continue;
741        }
742        let value = match (flat, &mut located) {
743            (Some(_), _) => staged
744                .get_mut(slot)
745                .and_then(Option::take)
746                .ok_or(tenferro_tensor::Error::MissingValue { slot })?,
747            (None, Some(located)) => {
748                let values = located
749                    .get_mut(slot)
750                    .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
751                let value = values
752                    .pop()
753                    .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
754                values.clear();
755                locations.push(Some(value.location));
756                value.value
757            }
758            (None, None) => {
759                return Err(Error::runtime_state(
760                    "Runtime::run",
761                    ErrorPhase::Execution,
762                    "a multi-location run lost its value lists".to_string(),
763                ))
764            }
765        };
766        collected.push(match value {
767            ExecSlot::Read(read) => Output::Borrowed(read),
768            ExecSlot::Owned(tensor) => Output::Owned(tensor),
769        });
770    }
771    Ok(collected)
772}
773
774fn duplicate_owned_output(
775    engines: &mut EngineSet<'_, '_>,
776    location: &ExecutionLocation,
777    tensor: &Tensor,
778) -> Result<Tensor> {
779    let engine = engines.engine(location.engine_id())?;
780    engine
781        .to_contiguous_read(TensorRead::from_tensor(tensor))
782        .map_err(Error::from)
783}
784
785/// One run's value store.
786///
787/// A run has either one location or several. With one (`flat`) a slot holds at
788/// most one value and always at that location, so `staged` is the slot array
789/// itself: a command neither stages its inputs nor republishes its results.
790/// With several, `located` keeps one value list per slot per location and a
791/// command stages its inputs into `staged` and publishes its results back.
792struct RunSlots<'input> {
793    /// The array commands execute against.
794    staged: Vec<Option<ExecSlot<'input>>>,
795    /// `Some` exactly when the run visits more than one location.
796    located: Option<Vec<Vec<LocatedExecSlot<'input>>>>,
797}
798
799impl<'input> RunSlots<'input> {
800    fn new(n_slots: usize, flat: bool) -> Self {
801        Self {
802            staged: Vec::new(),
803            located: (!flat).then(|| (0..n_slots).map(|_| Vec::new()).collect()),
804        }
805    }
806
807    /// Place the run's inputs and check they are readable at their location.
808    fn initialize(
809        &mut self,
810        program: &ExecProgram,
811        inputs: RuntimeInputReads<'input>,
812        schedule: &ScheduledGraph,
813    ) -> Result<()> {
814        crate::exec::initialize_exec_slots_in(
815            program,
816            inputs.into_iter().map(ExecSlot::Read),
817            &mut self.staged,
818        )?;
819        if schedule.input_locations().len() != program.input_slots.len() {
820            return Err(Error::runtime_state(
821                "Runtime::run",
822                ErrorPhase::Execution,
823                format!(
824                    "prepared schedule has {} input locations for {} inputs",
825                    schedule.input_locations().len(),
826                    program.input_slots.len()
827                ),
828            ));
829        }
830        for (&slot, location) in program.input_slots.iter().zip(schedule.input_locations()) {
831            match &mut self.located {
832                Some(located) => {
833                    let value = self
834                        .staged
835                        .get_mut(slot)
836                        .and_then(Option::take)
837                        .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
838                    validate_runtime_input_ingress(location, &value.as_read(), slot)?;
839                    located[slot].push(LocatedExecSlot {
840                        location: location.clone(),
841                        value,
842                    });
843                }
844                None => {
845                    let value = self
846                        .staged
847                        .get(slot)
848                        .and_then(Option::as_ref)
849                        .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
850                    validate_runtime_input_ingress(location, &value.as_read(), slot)?;
851                }
852            }
853        }
854        Ok(())
855    }
856
857    fn execution(&mut self) -> &mut [Option<ExecSlot<'input>>] {
858        &mut self.staged
859    }
860
861    fn execution_ref(&self) -> &[Option<ExecSlot<'input>>] {
862        &self.staged
863    }
864
865    /// Move one command's inputs into the execution array.
866    fn stage_command(
867        &mut self,
868        instruction: &ExecInstruction,
869        location: &ExecutionLocation,
870    ) -> Result<()> {
871        match &mut self.located {
872            Some(located) => {
873                stage_instruction_inputs(instruction, location, located, &mut self.staged)
874            }
875            None => Ok(()),
876        }
877    }
878
879    /// Publish one command's results, releasing the inputs it consumed last.
880    fn publish_command(
881        &mut self,
882        instruction: &ExecInstruction,
883        location: &ExecutionLocation,
884    ) -> Result<()> {
885        match &mut self.located {
886            Some(located) => {
887                retain_instruction_results(instruction, location, located, &mut self.staged)
888            }
889            None => Ok(()),
890        }
891    }
892
893    /// Move a region's inputs into the execution array.
894    fn stage_region(
895        &mut self,
896        region: &super::region::ElementwiseRegion,
897        location: &ExecutionLocation,
898    ) -> Result<()> {
899        let Some(located) = &mut self.located else {
900            return Ok(());
901        };
902        for &slot in &region.input_slots {
903            stage_slot_input(slot, location, located, &mut self.staged)?;
904        }
905        Ok(())
906    }
907
908    /// Publish a fused region's live-outs, releasing the inputs whose last use
909    /// is inside the region.
910    fn publish_region(
911        &mut self,
912        region: &super::region::ElementwiseRegion,
913        location: &ExecutionLocation,
914    ) -> Result<()> {
915        match &mut self.located {
916            Some(located) => retain_region_results(region, location, located, &mut self.staged),
917            None => {
918                for (&slot, &last_use) in
919                    region.input_slots.iter().zip(region.input_last_use.iter())
920                {
921                    if last_use {
922                        self.staged[slot] = None;
923                    }
924                }
925                Ok(())
926            }
927        }
928    }
929
930    /// The per-location value lists a transfer node needs.
931    fn transfers(&mut self) -> Result<&mut [Vec<LocatedExecSlot<'input>>]> {
932        self.located.as_deref_mut().ok_or_else(|| {
933            Error::runtime_state(
934                "Runtime::run",
935                ErrorPhase::Execution,
936                "a single-location run must not contain a transfer node".to_string(),
937            )
938        })
939    }
940}
941
942/// The location `instruction` executes at, proven equal to its node's.
943///
944/// A flat schedule proved at prepare that every operation sits at its one
945/// location (`ScheduledGraph::flat_location`), so the per-command comparison
946/// only runs for a multi-location schedule.
947fn command_location<'a>(
948    schedule: &'a ScheduledGraph,
949    flat: Option<&'a ExecutionLocation>,
950    instruction: &ExecInstruction,
951    node_location: &ExecutionLocation,
952) -> Result<&'a ExecutionLocation> {
953    if let Some(location) = flat {
954        return Ok(location);
955    }
956    let location = instruction_execution(schedule, instruction)?.location();
957    if location != node_location {
958        return Err(Error::runtime_state(
959            "Runtime::run",
960            ErrorPhase::Execution,
961            "a prepared command's location does not match its scheduled node".to_string(),
962        ));
963    }
964    Ok(location)
965}
966
967#[allow(clippy::too_many_arguments)]
968fn execute_scheduled_slots_into<'input>(
969    program: &ExecProgram,
970    schedule: &ScheduledGraph,
971    flat: Option<&ExecutionLocation>,
972    regions: &[super::region::ElementwiseRegion],
973    node_regions: &[super::region::NodeRegion],
974    region_counters: &super::region::RegionExecutionCounters,
975    operations: &[PreparedOperationPlan],
976    inputs: RuntimeInputReads<'input>,
977    engines: &mut EngineSet<'_, '_>,
978    event_domains: &mut Option<ScheduledEventDomains>,
979) -> Result<RunSlots<'input>> {
980    let domains = event_domains.as_mut().ok_or_else(|| {
981        Error::runtime_state(
982            "Runtime::run",
983            ErrorPhase::Execution,
984            "scheduled execution has no event-domain run to submit into",
985        )
986    })?;
987    let mut slots = RunSlots::new(program.n_slots, flat.is_some());
988    // INVARIANT: the immutable schedule was validated during preparation;
989    // driver preflight completes before input ingress or any operation launch.
990    // A provider panic is caught after this body and drops the event-domain
991    // run before this frame's staged tensor storage drops.
992    // Planned elementwise regions (`node_regions`, computed at prepare): the
993    // start node executes the whole region as one fused command and the
994    // covered nodes are skipped.
995    let result = std::panic::catch_unwind(AssertUnwindSafe(|| -> Result<RunSlots<'input>> {
996        (|| {
997            slots.initialize(program, inputs, schedule)?;
998            for (node_index, node) in schedule.nodes().iter().enumerate() {
999                match node {
1000                    ScheduledNode::Operation(operation_node) => {
1001                        let role = node_regions
1002                            .get(node_index)
1003                            .copied()
1004                            .unwrap_or(super::region::NodeRegion::Command);
1005                        if let super::region::NodeRegion::Start(region_index) = role {
1006                            let region = &regions[region_index];
1007                            let first_instruction = program
1008                                    .instructions
1009                                    .get(region.instruction_range.start)
1010                                    .ok_or_else(|| {
1011                                        Error::runtime_state(
1012                                            "Runtime::run",
1013                                            ErrorPhase::Execution,
1014                                            format!(
1015                                                "elementwise region references instruction {}, but the execution program has {} instructions",
1016                                                region.instruction_range.start,
1017                                                program.instructions.len()
1018                                            ),
1019                                        )
1020                                    })?;
1021                            let location = command_location(
1022                                schedule,
1023                                flat,
1024                                first_instruction,
1025                                operation_node.location(),
1026                            )?;
1027                            let mut launch = || {
1028                                slots.stage_region(region, location)?;
1029                                let applied = engines.execute_elementwise_fusion_slots(
1030                                    location,
1031                                    &region.input_slots,
1032                                    region.instructions.len(),
1033                                    &region.plan,
1034                                    slots.execution(),
1035                                    &region.output_slots,
1036                                )?;
1037                                if applied {
1038                                    region_counters.record_fused();
1039                                    validate_region_outputs(
1040                                        region,
1041                                        location,
1042                                        slots.execution_ref(),
1043                                    )?;
1044                                    return slots.publish_region(region, location);
1045                                }
1046                                // The backend rejected the fusion: execute the
1047                                // region's instructions one by one.
1048                                region_counters.record_fallback();
1049                                for (offset, instruction) in region.instructions.iter().enumerate()
1050                                {
1051                                    let instruction_index = region.instruction_range.start + offset;
1052                                    let member_location = command_location(
1053                                        schedule,
1054                                        flat,
1055                                        instruction,
1056                                        operation_node.location(),
1057                                    )?;
1058                                    slots.stage_command(instruction, member_location)?;
1059                                    engines.execute_slot_instruction(
1060                                        member_location,
1061                                        instruction_index,
1062                                        instruction,
1063                                        operations,
1064                                        slots.execution(),
1065                                    )?;
1066                                    validate_instruction_outputs(
1067                                        instruction_index,
1068                                        instruction,
1069                                        member_location,
1070                                        slots.execution_ref(),
1071                                    )?;
1072                                    slots.publish_command(instruction, member_location)?;
1073                                }
1074                                Ok(())
1075                            };
1076                            domains.enqueue(node_index, node, &mut launch)?;
1077                            region_counters.record_submission();
1078                            // Every covered node's completion resolves to the region's
1079                            // completion token, which was recorded after the region's
1080                            // last command.
1081                            let completion = EventDependency::from_completion(node.completion());
1082                            if let Some(token) = domains.completions.get(&completion).cloned() {
1083                                for &covered in &region.node_indices {
1084                                    if covered == node_index {
1085                                        continue;
1086                                    }
1087                                    let covered_node = schedule.nodes().get(covered).ok_or_else(|| {
1088                                            Error::runtime_state(
1089                                                "Runtime::run",
1090                                                ErrorPhase::Execution,
1091                                                format!(
1092                                                    "elementwise region references schedule node {covered}, but the schedule has {} nodes",
1093                                                    schedule.nodes().len()
1094                                                ),
1095                                            )
1096                                        })?;
1097                                    domains.completions.insert(
1098                                        EventDependency::from_completion(covered_node.completion()),
1099                                        Arc::clone(&token),
1100                                    );
1101                                }
1102                            }
1103                            continue;
1104                        }
1105                        if role == super::region::NodeRegion::Covered {
1106                            // Executed as part of its region.
1107                            continue;
1108                        }
1109                        let mut launch = || {
1110                            let instruction_index = operation_node.instruction_index();
1111                            let instruction = program
1112                                    .instructions
1113                                    .get(instruction_index)
1114                                    .ok_or_else(|| {
1115                                        Error::runtime_state(
1116                                            "Runtime::run",
1117                                            ErrorPhase::Execution,
1118                                            format!(
1119                                                "scheduled operation references instruction {instruction_index}, but the execution program has {} instructions",
1120                                                program.instructions.len()
1121                                            ),
1122                                        )
1123                                    })?;
1124                            let location = command_location(
1125                                schedule,
1126                                flat,
1127                                instruction,
1128                                operation_node.location(),
1129                            )?;
1130                            slots.stage_command(instruction, location)?;
1131                            engines.execute_slot_instruction(
1132                                location,
1133                                instruction_index,
1134                                instruction,
1135                                operations,
1136                                slots.execution(),
1137                            )?;
1138                            validate_instruction_outputs(
1139                                instruction_index,
1140                                instruction,
1141                                location,
1142                                slots.execution_ref(),
1143                            )?;
1144                            slots.publish_command(instruction, location)
1145                        };
1146                        domains.enqueue(node_index, node, &mut launch)?;
1147                        region_counters.record_submission();
1148                    }
1149                    ScheduledNode::Transfer(transfer) => {
1150                        let mut launch =
1151                            || execute_scheduled_transfer(transfer, slots.transfers()?);
1152                        domains.enqueue(node_index, node, &mut launch)?;
1153                        region_counters.record_submission();
1154                    }
1155                    ScheduledNode::Collective(_) => {
1156                        return Err(Error::runtime_state_source(
1157                            "Runtime::run",
1158                            ErrorPhase::Execution,
1159                            UnsupportedScheduledNodeError {
1160                                node_index,
1161                                node_kind: ScheduledNodeKind::Collective,
1162                            },
1163                        ));
1164                    }
1165                    ScheduledNode::Barrier(_) => {
1166                        let mut launch = || Ok(());
1167                        domains.enqueue(node_index, node, &mut launch)?;
1168                        region_counters.record_submission();
1169                    }
1170                }
1171            }
1172            Ok(())
1173        })()?;
1174        Ok(slots)
1175    }));
1176    match result {
1177        Ok(slots) => slots,
1178        Err(payload) => {
1179            // A provider launch may unwind through the scheduler. Drop the
1180            // event-domain run before this frame's staged tensor storage drops,
1181            // preserving the same cleanup order as the synchronous wrapper.
1182            event_domains.take();
1183            std::panic::resume_unwind(payload)
1184        }
1185    }
1186}
1187
1188#[derive(Debug)]
1189pub(crate) struct ScheduledEventDomains {
1190    runs: Vec<RuntimeOwnedEventDomainRun>,
1191    completions: HashMap<EventDependency, Arc<dyn EventToken>>,
1192    /// Whether the single run reported that submission order alone orders its
1193    /// nodes. The scheduler then submits each launch without resolving
1194    /// dependencies or recording a completion token for it.
1195    coalesced: bool,
1196}
1197
1198#[derive(Debug, thiserror::Error)]
1199#[error(
1200    "scheduled node {node_index} depends on completion {dependency:?}, but no completion token was recorded"
1201)]
1202pub(crate) struct MissingScheduledDependencyCompletionError {
1203    pub(crate) dependency: EventDependency,
1204    pub(crate) node_index: usize,
1205}
1206
1207#[derive(Debug, thiserror::Error)]
1208#[error("scheduled transfer destination {destination:?} already contains value slot {value_slot}")]
1209pub(crate) struct DuplicateTransferDestinationError {
1210    pub(crate) value_slot: usize,
1211    pub(crate) destination: ExecutionLocation,
1212}
1213
1214impl ScheduledEventDomains {
1215    /// Number of completion tokens recorded so far.
1216    #[cfg(test)]
1217    pub(crate) fn completion_count_for_test(&self) -> usize {
1218        self.completions.len()
1219    }
1220
1221    fn new(schedule: &ScheduledGraph, engines: &EngineSet<'_, '_>) -> Result<Self> {
1222        Self::build(schedule, |engine_id| engines.driver(engine_id))
1223    }
1224
1225    /// Build the runs a schedule needs with an explicit driver lookup.
1226    ///
1227    /// Test-only: production always takes each run's driver from the engine the
1228    /// schedule placed the work on.
1229    #[cfg(test)]
1230    pub(crate) fn new_for_test(
1231        schedule: &ScheduledGraph,
1232        drivers: &[(EngineId, Arc<dyn super::EventDomainDriver>)],
1233    ) -> Result<Self> {
1234        Self::build(schedule, |engine_id| {
1235            drivers
1236                .iter()
1237                .find(|(id, _)| id == engine_id)
1238                .map(|(_, driver)| Arc::clone(driver))
1239                .ok_or_else(|| {
1240                    Error::runtime_state(
1241                        "ScheduledEventDomains::new_for_test",
1242                        ErrorPhase::Execution,
1243                        format!("no test driver for engine {:?}", engine_id.as_str()),
1244                    )
1245                })
1246        })
1247    }
1248
1249    fn build(
1250        schedule: &ScheduledGraph,
1251        driver_for: impl Fn(&EngineId) -> Result<Arc<dyn super::EventDomainDriver>>,
1252    ) -> Result<Self> {
1253        schedule.preflight().map_err(|source| {
1254            Error::runtime_state_source("Runtime::run", ErrorPhase::Execution, source)
1255        })?;
1256        // A coalescing-eligible schedule names exactly one event domain: every
1257        // location, completion, and dependency carries the root's, and no
1258        // transfer node exists. Seed that single run directly instead of
1259        // scanning every location into a driver table only to find one entry.
1260        if schedule.coalescing_eligible() {
1261            let domain = schedule.root_location().event_domain_id();
1262            let engine_id = schedule.root_location().engine_id().clone();
1263            let mut run = RuntimeOwnedEventDomainRun::new(
1264                domain,
1265                engine_id.clone(),
1266                driver_for(&engine_id)?.begin_run(domain)?,
1267            );
1268            let actual = run.domain(EventDomainOperation::BeginRun)?;
1269            if actual != domain {
1270                return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1271                    operation: EventDomainOperation::BeginRun,
1272                    node_index: None,
1273                    expected: domain,
1274                    actual,
1275                }));
1276            }
1277            let coalesced = run.enable_single_domain_coalescing();
1278            return Ok(Self {
1279                runs: vec![run],
1280                completions: HashMap::new(),
1281                coalesced,
1282            });
1283        }
1284        fn add_domain(
1285            drivers: &mut Vec<(super::EventDomainId, EngineId)>,
1286            seen_domains: &mut HashSet<super::EventDomainId>,
1287            domain: super::EventDomainId,
1288            engine_id: &EngineId,
1289        ) {
1290            if seen_domains.insert(domain) {
1291                drivers.push((domain, engine_id.clone()));
1292            }
1293        }
1294        let mut drivers = Vec::new();
1295        let mut seen_domains = HashSet::new();
1296        // Seed a run for every scheduled location, not only for scheduled nodes.
1297        // Output materialization launches work on the output location's stream
1298        // after the nodes have run, and a zero-instruction graph (an identity
1299        // graph) has no node to seed a run at all; without the location runs
1300        // there would be nothing to retire that launch.
1301        add_domain(
1302            &mut drivers,
1303            &mut seen_domains,
1304            schedule.root_location().event_domain_id(),
1305            schedule.root_location().engine_id(),
1306        );
1307        for location in schedule.input_locations() {
1308            add_domain(
1309                &mut drivers,
1310                &mut seen_domains,
1311                location.event_domain_id(),
1312                location.engine_id(),
1313            );
1314        }
1315        for location in schedule.operation_locations() {
1316            add_domain(
1317                &mut drivers,
1318                &mut seen_domains,
1319                location.event_domain_id(),
1320                location.engine_id(),
1321            );
1322        }
1323        for node in schedule.nodes() {
1324            add_domain(
1325                &mut drivers,
1326                &mut seen_domains,
1327                node.completion().domain(),
1328                node.event_domain_witness().engine_id(),
1329            );
1330        }
1331        let mut runs = Vec::with_capacity(drivers.len());
1332        for (domain, engine_id) in drivers {
1333            let driver = driver_for(&engine_id)?;
1334            let run = RuntimeOwnedEventDomainRun::new(domain, engine_id, driver.begin_run(domain)?);
1335            let actual = run.domain(EventDomainOperation::BeginRun)?;
1336            if actual != domain {
1337                return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1338                    operation: EventDomainOperation::BeginRun,
1339                    node_index: None,
1340                    expected: domain,
1341                    actual,
1342                }));
1343            }
1344            runs.push(run);
1345        }
1346        Ok(Self {
1347            runs,
1348            completions: HashMap::new(),
1349            coalesced: false,
1350        })
1351    }
1352
1353    #[cfg(test)]
1354    pub(crate) fn for_test(
1355        drivers: Vec<(super::EventDomainId, Arc<dyn super::EventDomainDriver>)>,
1356    ) -> Result<Self> {
1357        let mut runs = Vec::with_capacity(drivers.len());
1358        for (domain, driver) in drivers {
1359            let engine_id =
1360                EngineId::new("tenferro-test.event-domain-engine").expect("test engine id");
1361            let run = RuntimeOwnedEventDomainRun::new(domain, engine_id, driver.begin_run(domain)?);
1362            let actual = run.domain(EventDomainOperation::BeginRun)?;
1363            if actual != domain {
1364                return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1365                    operation: EventDomainOperation::BeginRun,
1366                    node_index: None,
1367                    expected: domain,
1368                    actual,
1369                }));
1370            }
1371            runs.push(run);
1372        }
1373        Ok(Self {
1374            runs,
1375            completions: HashMap::new(),
1376            coalesced: false,
1377        })
1378    }
1379
1380    pub(crate) fn enqueue(
1381        &mut self,
1382        node_index: usize,
1383        node: &ScheduledNode,
1384        launch: &mut dyn FnMut() -> Result<()>,
1385    ) -> Result<()> {
1386        let completion = node.completion();
1387        if self.coalesced {
1388            // The run reported that submission order alone orders its nodes
1389            // (#2186): no dependency token exists that a later node could read,
1390            // so submit the launch through the run's own submission context
1391            // without resolving dependencies or recording a completion.
1392            debug_assert_eq!(self.runs.len(), 1);
1393            self.runs[0].enqueue(&[], launch)?;
1394            return Ok(());
1395        }
1396        let destination = completion.domain();
1397        let run_index = self.run_index(completion.domain())?;
1398        let actual_preflight_domain = self.runs[run_index].domain(EventDomainOperation::Enqueue)?;
1399        if actual_preflight_domain != destination {
1400            return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1401                operation: EventDomainOperation::Enqueue,
1402                node_index: Some(node_index),
1403                expected: destination,
1404                actual: actual_preflight_domain,
1405            }));
1406        }
1407        let dependencies = self.classify_dependencies(node_index, node, destination)?;
1408        let actual_run_domain = self.runs[run_index].domain(EventDomainOperation::Enqueue)?;
1409        if actual_run_domain != destination {
1410            return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1411                operation: EventDomainOperation::Enqueue,
1412                node_index: Some(node_index),
1413                expected: destination,
1414                actual: actual_run_domain,
1415            }));
1416        }
1417        let completion_event = self.runs[run_index].enqueue(&dependencies, launch)?;
1418        let actual = completion_event.origin();
1419        if actual != completion.domain() {
1420            return Err(event_domain_error(
1421                EventDomainError::CompletionTokenDomainMismatch {
1422                    operation: EventDomainOperation::ValidateCompletion,
1423                    node_index: Some(node_index),
1424                    expected: completion.domain(),
1425                    actual,
1426                },
1427            ));
1428        }
1429        self.completions.insert(
1430            EventDependency::from_completion(completion),
1431            completion_event,
1432        );
1433        Ok(())
1434    }
1435
1436    fn classify_dependencies(
1437        &self,
1438        node_index: usize,
1439        node: &ScheduledNode,
1440        destination: super::EventDomainId,
1441    ) -> Result<Vec<Arc<dyn EventToken>>> {
1442        // `Arc<dyn EventToken>` has neither `Default` nor `Copy`, so it fits no
1443        // inline array. This path is cold: a single-domain run coalesces and
1444        // returns from `enqueue` before reaching here (#2186), so only
1445        // multi-domain runs classify dependencies, and each such node already
1446        // allocates its completion token and records a device event. An empty
1447        // dependency list does not allocate.
1448        let mut admitted = Vec::with_capacity(node.dependencies().len());
1449        for dependency in node.dependencies() {
1450            let dependency_completion =
1451                self.completions.get(dependency).cloned().ok_or_else(|| {
1452                    Error::runtime_state_source(
1453                        "Runtime::run",
1454                        ErrorPhase::Execution,
1455                        MissingScheduledDependencyCompletionError {
1456                            dependency: *dependency,
1457                            node_index,
1458                        },
1459                    )
1460                })?;
1461            let actual = dependency_completion.origin();
1462            if actual != dependency.domain() {
1463                return Err(event_domain_error(
1464                    EventDomainError::DependencyDomainMismatch {
1465                        operation: match node {
1466                            ScheduledNode::Transfer(_) => EventDomainOperation::TransferBridge,
1467                            ScheduledNode::Operation(_)
1468                            | ScheduledNode::Collective(_)
1469                            | ScheduledNode::Barrier(_) => EventDomainOperation::Enqueue,
1470                        },
1471                        node_index: Some(node_index),
1472                        expected: dependency.domain(),
1473                        actual,
1474                    },
1475                ));
1476            }
1477            match node {
1478                ScheduledNode::Transfer(transfer) => {
1479                    let source = transfer.source_event_domain();
1480                    if actual == destination {
1481                        admitted.push(dependency_completion);
1482                    } else if actual == source {
1483                        dependency_completion.wait().map_err(|source_error| {
1484                            event_domain_error(EventDomainError::DependencyWaitFailed {
1485                                operation: EventDomainOperation::TransferBridge,
1486                                node_index: Some(node_index),
1487                                expected: destination,
1488                                actual,
1489                                source: Box::new(source_error),
1490                            })
1491                        })?;
1492                    } else {
1493                        return Err(event_domain_error(
1494                            EventDomainError::DependencyDomainMismatch {
1495                                operation: EventDomainOperation::TransferBridge,
1496                                node_index: Some(node_index),
1497                                expected: source,
1498                                actual,
1499                            },
1500                        ));
1501                    }
1502                }
1503                ScheduledNode::Operation(_)
1504                | ScheduledNode::Collective(_)
1505                | ScheduledNode::Barrier(_) => {
1506                    if actual != destination {
1507                        return Err(event_domain_error(
1508                            EventDomainError::DependencyDomainMismatch {
1509                                operation: EventDomainOperation::Enqueue,
1510                                node_index: Some(node_index),
1511                                expected: destination,
1512                                actual,
1513                            },
1514                        ));
1515                    }
1516                    admitted.push(dependency_completion);
1517                }
1518            }
1519        }
1520        Ok(admitted)
1521    }
1522
1523    fn run_index(&self, domain: super::EventDomainId) -> Result<usize> {
1524        if let Some(index) = self
1525            .runs
1526            .iter()
1527            .position(|run| run.requested_domain() == domain)
1528        {
1529            return Ok(index);
1530        }
1531        Err(missing_event_domain_driver(domain))
1532    }
1533
1534    /// Release every pending run, each with the engine that owns it.
1535    fn into_pending_runs(self) -> impl Iterator<Item = (EngineId, Box<dyn EventDomainRun>)> {
1536        self.runs
1537            .into_iter()
1538            .filter_map(RuntimeOwnedEventDomainRun::into_pending)
1539    }
1540
1541    pub(crate) fn drain(&mut self) -> Result<()> {
1542        let mut failures = Vec::new();
1543        for run in &mut self.runs {
1544            if let Err(error) = run.drain() {
1545                failures.push(error);
1546            }
1547        }
1548        let mut failures = failures.into_iter();
1549        let Some(mut error) = failures.next() else {
1550            return Ok(());
1551        };
1552        for failure in failures {
1553            error = Error::with_suppressed(error, failure);
1554        }
1555        Err(error)
1556    }
1557}
1558
1559#[derive(Debug)]
1560enum RuntimeOwnedEventDomainRunState {
1561    Pending(Box<dyn EventDomainRun>),
1562    Retired,
1563    Failed,
1564}
1565
1566#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1567enum EventDomainRunTerminalState {
1568    Retired,
1569    Failed,
1570}
1571
1572impl fmt::Display for EventDomainRunTerminalState {
1573    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1574        match self {
1575            Self::Retired => formatter.write_str("retired"),
1576            Self::Failed => formatter.write_str("failed"),
1577        }
1578    }
1579}
1580
1581#[derive(Debug)]
1582struct RuntimeOwnedEventDomainRun {
1583    requested_domain: super::EventDomainId,
1584    /// The engine whose driver opened this run; it takes the run over once the
1585    /// outputs are collected.
1586    engine_id: EngineId,
1587    state: RuntimeOwnedEventDomainRunState,
1588}
1589
1590impl RuntimeOwnedEventDomainRun {
1591    fn new(
1592        requested_domain: super::EventDomainId,
1593        engine_id: EngineId,
1594        inner: Box<dyn EventDomainRun>,
1595    ) -> Self {
1596        Self {
1597            requested_domain,
1598            engine_id,
1599            state: RuntimeOwnedEventDomainRunState::Pending(inner),
1600        }
1601    }
1602
1603    /// Release the pending run to its engine, or `None` once it is terminal.
1604    fn into_pending(mut self) -> Option<(EngineId, Box<dyn EventDomainRun>)> {
1605        match std::mem::replace(&mut self.state, RuntimeOwnedEventDomainRunState::Retired) {
1606            RuntimeOwnedEventDomainRunState::Pending(run) => Some((self.engine_id.clone(), run)),
1607            RuntimeOwnedEventDomainRunState::Retired | RuntimeOwnedEventDomainRunState::Failed => {
1608                None
1609            }
1610        }
1611    }
1612
1613    fn requested_domain(&self) -> super::EventDomainId {
1614        self.requested_domain
1615    }
1616
1617    /// Ask the provider run to accept coalesced single-domain submission.
1618    ///
1619    /// A run that keeps the default per-node contract returns `false`, and a
1620    /// retired or failed run is never asked.
1621    fn enable_single_domain_coalescing(&mut self) -> bool {
1622        match &mut self.state {
1623            RuntimeOwnedEventDomainRunState::Pending(run) => run.enable_single_domain_coalescing(),
1624            RuntimeOwnedEventDomainRunState::Retired | RuntimeOwnedEventDomainRunState::Failed => {
1625                false
1626            }
1627        }
1628    }
1629
1630    fn domain(&self, operation: EventDomainOperation) -> Result<super::EventDomainId> {
1631        match &self.state {
1632            RuntimeOwnedEventDomainRunState::Pending(run) => Ok(run.domain()),
1633            RuntimeOwnedEventDomainRunState::Retired => Err(event_domain_run_state_error(
1634                operation,
1635                self.requested_domain,
1636                EventDomainRunTerminalState::Retired,
1637            )),
1638            RuntimeOwnedEventDomainRunState::Failed => Err(event_domain_run_state_error(
1639                operation,
1640                self.requested_domain,
1641                EventDomainRunTerminalState::Failed,
1642            )),
1643        }
1644    }
1645
1646    fn enqueue(
1647        &mut self,
1648        dependencies: &[Arc<dyn EventToken>],
1649        launch: &mut dyn FnMut() -> Result<()>,
1650    ) -> Result<Arc<dyn EventToken>> {
1651        match &mut self.state {
1652            RuntimeOwnedEventDomainRunState::Pending(run) => run.enqueue(dependencies, launch),
1653            RuntimeOwnedEventDomainRunState::Retired => Err(event_domain_run_state_error(
1654                EventDomainOperation::Enqueue,
1655                self.requested_domain,
1656                EventDomainRunTerminalState::Retired,
1657            )),
1658            RuntimeOwnedEventDomainRunState::Failed => Err(event_domain_run_state_error(
1659                EventDomainOperation::Enqueue,
1660                self.requested_domain,
1661                EventDomainRunTerminalState::Failed,
1662            )),
1663        }
1664    }
1665
1666    fn drain(&mut self) -> Result<()> {
1667        let run = match std::mem::replace(&mut self.state, RuntimeOwnedEventDomainRunState::Failed)
1668        {
1669            RuntimeOwnedEventDomainRunState::Pending(run) => run,
1670            RuntimeOwnedEventDomainRunState::Retired => {
1671                self.state = RuntimeOwnedEventDomainRunState::Retired;
1672                return Err(event_domain_run_state_error(
1673                    EventDomainOperation::Drain,
1674                    self.requested_domain,
1675                    EventDomainRunTerminalState::Retired,
1676                ));
1677            }
1678            RuntimeOwnedEventDomainRunState::Failed => {
1679                self.state = RuntimeOwnedEventDomainRunState::Failed;
1680                return Err(event_domain_run_state_error(
1681                    EventDomainOperation::Drain,
1682                    self.requested_domain,
1683                    EventDomainRunTerminalState::Failed,
1684                ));
1685            }
1686        };
1687        let domain = self.requested_domain;
1688        let mut run = run;
1689        let drain_result = std::panic::catch_unwind(AssertUnwindSafe(|| run.drain()));
1690        drop_event_domain_run(run);
1691        match drain_result {
1692            Ok(Ok(())) => {
1693                self.state = RuntimeOwnedEventDomainRunState::Retired;
1694                Ok(())
1695            }
1696            Ok(Err(error)) => {
1697                self.state = RuntimeOwnedEventDomainRunState::Failed;
1698                Err(error)
1699            }
1700            Err(payload) => {
1701                self.state = RuntimeOwnedEventDomainRunState::Failed;
1702                Err(event_domain_error(EventDomainError::DrainPanicked {
1703                    operation: EventDomainOperation::Drain,
1704                    domain,
1705                    message: safe_event_domain_panic_message(payload),
1706                }))
1707            }
1708        }
1709    }
1710}
1711
1712/// Internal error returned when a runtime-owned event run is used after
1713/// explicit retirement or failure.
1714#[derive(Debug, thiserror::Error)]
1715#[error("{operation} used event-domain run {domain:?} after it reached terminal state {state}")]
1716pub(crate) struct EventDomainRunLifecycleError {
1717    operation: EventDomainOperation,
1718    domain: super::EventDomainId,
1719    state: EventDomainRunTerminalState,
1720}
1721
1722fn event_domain_run_state_error(
1723    operation: EventDomainOperation,
1724    domain: super::EventDomainId,
1725    state: EventDomainRunTerminalState,
1726) -> Error {
1727    Error::runtime_state_source(
1728        "Runtime::run",
1729        ErrorPhase::Execution,
1730        EventDomainRunLifecycleError {
1731            operation,
1732            domain,
1733            state,
1734        },
1735    )
1736}
1737
1738fn drop_event_domain_run(run: Box<dyn EventDomainRun>) {
1739    if std::panic::catch_unwind(AssertUnwindSafe(|| drop(run))).is_err() {
1740        // The provider run has been consumed; its cleanup panic is contained.
1741    }
1742}
1743
1744impl Drop for RuntimeOwnedEventDomainRun {
1745    fn drop(&mut self) {
1746        let state = std::mem::replace(&mut self.state, RuntimeOwnedEventDomainRunState::Failed);
1747        if let RuntimeOwnedEventDomainRunState::Pending(run) = state {
1748            drop_event_domain_run(run);
1749        }
1750    }
1751}
1752
1753fn missing_event_domain_driver(domain: super::EventDomainId) -> Error {
1754    Error::from(EventDomainError::MissingDriver { domain })
1755}
1756
1757fn event_domain_error(source: EventDomainError) -> Error {
1758    Error::from(source)
1759}
1760
1761fn safe_event_domain_panic_message(payload: Box<dyn std::any::Any + Send + 'static>) -> String {
1762    match payload.downcast::<&'static str>() {
1763        Ok(message) => (*message).to_owned(),
1764        Err(payload) => match payload.downcast::<String>() {
1765            Ok(message) => *message,
1766            Err(_) => "non-string panic payload".to_owned(),
1767        },
1768    }
1769}
1770
1771fn scheduled_execution_cleanup_error(primary: Error, cleanup: Error) -> Error {
1772    Error::with_suppressed(primary, cleanup)
1773}
1774
1775fn instruction_execution<'a>(
1776    schedule: &'a ScheduledGraph,
1777    instruction: &ExecInstruction,
1778) -> Result<InstructionExecution<'a>> {
1779    let Some(operation_index) = instruction.semantic_operation_index else {
1780        let location = schedule.root_location();
1781        return Ok(InstructionExecution { location });
1782    };
1783    let location = schedule
1784        .operation_locations()
1785        .get(operation_index)
1786        .ok_or_else(|| {
1787            Error::runtime_state(
1788                "Runtime::run",
1789                ErrorPhase::Execution,
1790                format!(
1791                    "instruction references semantic operation {operation_index}, but prepared schedule has {} operations",
1792                    schedule.operation_locations().len()
1793                ),
1794            )
1795        })?;
1796    Ok(InstructionExecution { location })
1797}
1798
1799struct InstructionExecution<'a> {
1800    location: &'a ExecutionLocation,
1801}
1802
1803impl<'a> InstructionExecution<'a> {
1804    fn location(&self) -> &'a ExecutionLocation {
1805        self.location
1806    }
1807}
1808
1809pub(crate) struct LocatedExecSlot<'input> {
1810    pub(crate) location: ExecutionLocation,
1811    pub(crate) value: ExecSlot<'input>,
1812}
1813
1814fn stage_instruction_inputs<'input>(
1815    instruction: &ExecInstruction,
1816    location: &ExecutionLocation,
1817    located: &mut [Vec<LocatedExecSlot<'input>>],
1818    staged: &mut [Option<ExecSlot<'input>>],
1819) -> Result<()> {
1820    for &slot in &instruction.input_slots {
1821        stage_slot_input(slot, location, located, staged)?;
1822    }
1823    Ok(())
1824}
1825
1826/// Move one slot's value from `located` into `staged`.
1827fn stage_slot_input<'input>(
1828    slot: usize,
1829    location: &ExecutionLocation,
1830    located: &mut [Vec<LocatedExecSlot<'input>>],
1831    staged: &mut [Option<ExecSlot<'input>>],
1832) -> Result<()> {
1833    if staged
1834        .get(slot)
1835        .ok_or(tenferro_tensor::Error::MissingValue { slot })?
1836        .is_some()
1837    {
1838        return Ok(());
1839    }
1840    let values = located
1841        .get_mut(slot)
1842        .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1843    let value_index = values
1844        .iter()
1845        .position(|value| &value.location == location)
1846        .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1847    staged[slot] = Some(values.swap_remove(value_index).value);
1848    Ok(())
1849}
1850
1851/// Validate every live-out of a fused elementwise region: present and resident
1852/// on the region's location.
1853fn validate_region_outputs(
1854    region: &super::region::ElementwiseRegion,
1855    location: &ExecutionLocation,
1856    staged: &[Option<ExecSlot<'_>>],
1857) -> Result<()> {
1858    for &output_slot in &region.output_slots {
1859        let output = staged
1860            .get(output_slot)
1861            .and_then(Option::as_ref)
1862            .ok_or(tenferro_tensor::Error::MissingValue { slot: output_slot })?;
1863        let output = output.as_read();
1864        if !location
1865            .witness()
1866            .owns_resident_tensor(&output, location.storage_class())
1867        {
1868            return Err(Error::runtime_state_source(
1869                "Runtime::run",
1870                ErrorPhase::Execution,
1871                super::EngineExecutionContractError::OutputResidencyMismatch {
1872                    instruction_index: region.instruction_range.start,
1873                    output_slot,
1874                    engine_id: location.engine_id().clone(),
1875                    storage_class: location.storage_class().clone(),
1876                    backend_family: output.backend_family(),
1877                    allocation_domain: output.allocation_domain(),
1878                },
1879            ));
1880        }
1881    }
1882    Ok(())
1883}
1884
1885/// Publish a fused region's results: release region inputs whose last use is
1886/// inside the region and move every live-out into `located`.
1887fn retain_region_results<'input>(
1888    region: &super::region::ElementwiseRegion,
1889    location: &ExecutionLocation,
1890    located: &mut [Vec<LocatedExecSlot<'input>>],
1891    staged: &mut [Option<ExecSlot<'input>>],
1892) -> Result<()> {
1893    for (&slot, &last_use) in region.input_slots.iter().zip(region.input_last_use.iter()) {
1894        if last_use {
1895            located[slot].clear();
1896            staged[slot].take();
1897        }
1898    }
1899    for &slot in &region.output_slots {
1900        let value = staged
1901            .get_mut(slot)
1902            .and_then(Option::take)
1903            .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1904        let values = located
1905            .get_mut(slot)
1906            .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1907        values.clear();
1908        values.push(LocatedExecSlot {
1909            location: location.clone(),
1910            value,
1911        });
1912    }
1913    Ok(())
1914}
1915
1916fn validate_instruction_outputs(
1917    instruction_index: usize,
1918    instruction: &ExecInstruction,
1919    location: &ExecutionLocation,
1920    staged: &[Option<ExecSlot<'_>>],
1921) -> Result<()> {
1922    for &output_slot in &instruction.output_slots {
1923        let output = staged
1924            .get(output_slot)
1925            .and_then(Option::as_ref)
1926            .ok_or(tenferro_tensor::Error::MissingValue { slot: output_slot })?;
1927        let output = output.as_read();
1928        if !location
1929            .witness()
1930            .owns_resident_tensor(&output, location.storage_class())
1931        {
1932            return Err(Error::runtime_state_source(
1933                "Runtime::run",
1934                ErrorPhase::Execution,
1935                super::EngineExecutionContractError::OutputResidencyMismatch {
1936                    instruction_index,
1937                    output_slot,
1938                    engine_id: location.engine_id().clone(),
1939                    storage_class: location.storage_class().clone(),
1940                    backend_family: output.backend_family(),
1941                    allocation_domain: output.allocation_domain(),
1942                },
1943            ));
1944        }
1945    }
1946    Ok(())
1947}
1948
1949pub(crate) fn retain_instruction_results<'input>(
1950    instruction: &ExecInstruction,
1951    location: &ExecutionLocation,
1952    located: &mut [Vec<LocatedExecSlot<'input>>],
1953    staged: &mut [Option<ExecSlot<'input>>],
1954) -> Result<()> {
1955    for &slot in &instruction.input_slots {
1956        let is_output = instruction.output_slots.contains(&slot);
1957        let is_last_use = instruction
1958            .input_slots
1959            .iter()
1960            .enumerate()
1961            .any(|(index, &candidate)| {
1962                candidate == slot && instruction.last_use.get(index).copied().unwrap_or(false)
1963            });
1964        if is_last_use {
1965            located[slot].clear();
1966            if !is_output {
1967                staged[slot].take();
1968            }
1969        } else if !is_output && let Some(value) = staged[slot].take() {
1970            located[slot].push(LocatedExecSlot {
1971                location: location.clone(),
1972                value,
1973            });
1974        }
1975    }
1976
1977    for &slot in &instruction.output_slots {
1978        let value = staged
1979            .get_mut(slot)
1980            .and_then(Option::take)
1981            .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1982        let values = located
1983            .get_mut(slot)
1984            .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1985        values.clear();
1986        values.push(LocatedExecSlot {
1987            location: location.clone(),
1988            value,
1989        });
1990    }
1991    Ok(())
1992}
1993
1994fn validate_runtime_input_ingress(
1995    location: &ExecutionLocation,
1996    input: &TensorRead<'_>,
1997    slot: usize,
1998) -> Result<()> {
1999    let accepted = location
2000        .witness()
2001        .accepts_runtime_input(input, location.storage_class());
2002    if accepted {
2003        return Ok(());
2004    }
2005    Err(Error::runtime_state_source(
2006        "Runtime::run",
2007        ErrorPhase::Execution,
2008        super::InputIngressContractError::ResidencyMismatch {
2009            input_slot: slot,
2010            ingress_engine_id: location.engine_id().clone(),
2011            ingress_storage_class: location.storage_class().clone(),
2012            placement: input.placement().clone(),
2013            backend_family: input.backend_family(),
2014            allocation_domain: input.allocation_domain(),
2015        },
2016    ))
2017}
2018
2019fn execute_scheduled_transfer<'input>(
2020    transfer: &ScheduledTransfer,
2021    located: &mut [Vec<LocatedExecSlot<'input>>],
2022) -> Result<()> {
2023    let source = transfer.source_location();
2024    let destination = transfer.destination_location();
2025    let provider = transfer.provider();
2026    let values =
2027        located
2028            .get_mut(transfer.value_slot())
2029            .ok_or(tenferro_tensor::Error::MissingValue {
2030                slot: transfer.value_slot(),
2031            })?;
2032    if values.iter().any(|value| &value.location == destination) {
2033        return Err(Error::runtime_state_source(
2034            "Runtime::run",
2035            ErrorPhase::Execution,
2036            DuplicateTransferDestinationError {
2037                value_slot: transfer.value_slot(),
2038                destination: destination.clone(),
2039            },
2040        ));
2041    }
2042    let transferred = {
2043        let source_value = values
2044            .iter()
2045            .find(|value| &value.location == source)
2046            .ok_or(tenferro_tensor::Error::MissingValue {
2047                slot: transfer.value_slot(),
2048            })?;
2049        let source_read = source_value.value.as_read();
2050        let expected_dtype = source_read.dtype();
2051        let expected_shape = source_read.shape().to_vec();
2052        let transferred =
2053            provider.transfer_blocking(TransferRequest::new(source, destination, source_read))?;
2054        validate_transfer_output(destination, expected_dtype, &expected_shape, &transferred)?;
2055        transferred
2056    };
2057    values.push(LocatedExecSlot {
2058        location: destination.clone(),
2059        value: ExecSlot::Owned(transferred),
2060    });
2061    Ok(())
2062}
2063
2064fn validate_transfer_output(
2065    destination: &ExecutionLocation,
2066    expected_dtype: tenferro_tensor::DType,
2067    expected_shape: &[usize],
2068    output: &Tensor,
2069) -> Result<()> {
2070    let expected_elements = checked_transfer_element_count(expected_shape).map_err(|source| {
2071        Error::runtime_state_source(
2072            "Runtime::run",
2073            ErrorPhase::Execution,
2074            TransferError::ProviderContract { source },
2075        )
2076    })?;
2077    let contract_error = if output.dtype() != expected_dtype {
2078        Some(TransferProviderContractError::DTypeMismatch {
2079            expected: expected_dtype,
2080            actual: output.dtype(),
2081        })
2082    } else if output.shape() != expected_shape {
2083        Some(TransferProviderContractError::ShapeMismatch {
2084            expected: expected_shape.to_vec(),
2085            actual: output.shape().to_vec(),
2086        })
2087    } else if tensor_buffer_len(output) != expected_elements {
2088        Some(TransferProviderContractError::InvalidBufferLength {
2089            expected: expected_elements,
2090            actual: tensor_buffer_len(output),
2091        })
2092    } else if !destination
2093        .witness()
2094        .accepts_input_placement(output.placement(), destination.storage_class())
2095    {
2096        Some(
2097            TransferProviderContractError::DestinationPlacementMismatch {
2098                destination_engine_id: destination.engine_id().clone(),
2099                destination_storage_class: destination.storage_class().clone(),
2100                actual: output.placement().clone(),
2101            },
2102        )
2103    } else if !destination.witness().owns_resident_tensor(
2104        &TensorRead::from_tensor(output),
2105        destination.storage_class(),
2106    ) {
2107        Some(
2108            TransferProviderContractError::DestinationResidencyMismatch {
2109                destination_engine_id: destination.engine_id().clone(),
2110                destination_storage_class: destination.storage_class().clone(),
2111                actual_backend_family: TensorRead::from_tensor(output).backend_family(),
2112                actual_allocation_domain: TensorRead::from_tensor(output).allocation_domain(),
2113            },
2114        )
2115    } else {
2116        None
2117    };
2118    match contract_error {
2119        None => Ok(()),
2120        Some(source) => Err(Error::runtime_state_source(
2121            "Runtime::run",
2122            ErrorPhase::Execution,
2123            TransferError::ProviderContract { source },
2124        )),
2125    }
2126}
2127
2128fn checked_transfer_element_count(
2129    shape: &[usize],
2130) -> std::result::Result<usize, TransferProviderContractError> {
2131    tenferro_tensor::validate::checked_shape_product("Runtime::run", "transfer source shape", shape)
2132        .map_err(|source| TransferProviderContractError::LogicalElementCount { source })
2133}
2134
2135fn tensor_buffer_len(tensor: &Tensor) -> usize {
2136    match tensor.dtype() {
2137        DType::F32 => tensor.as_typed::<f32>().map_or(0, |t| t.buffer().len()),
2138        DType::F64 => tensor.as_typed::<f64>().map_or(0, |t| t.buffer().len()),
2139        DType::I32 => tensor.as_typed::<i32>().map_or(0, |t| t.buffer().len()),
2140        DType::I64 => tensor.as_typed::<i64>().map_or(0, |t| t.buffer().len()),
2141        DType::Bool => tensor.as_typed::<bool>().map_or(0, |t| t.buffer().len()),
2142        DType::C32 => tensor
2143            .as_typed::<Complex32>()
2144            .map_or(0, |t| t.buffer().len()),
2145        DType::C64 => tensor
2146            .as_typed::<Complex64>()
2147            .map_or(0, |t| t.buffer().len()),
2148        // A caller-owned payload has no runtime buffer length to report, which is
2149        // also what the accessor falls back to when the tag table and the runtime
2150        // dtype disagree.
2151        DType::External(_) => 0,
2152    }
2153}
2154
2155#[cfg(test)]
2156mod transfer_validation_tests {
2157    use std::error::Error as _;
2158
2159    use super::checked_transfer_element_count;
2160    use crate::TransferProviderContractError;
2161
2162    #[test]
2163    fn transfer_element_count_overflow_is_typed_and_preserves_source() {
2164        let error = checked_transfer_element_count(&[usize::MAX, 2]).unwrap_err();
2165
2166        assert!(matches!(
2167            error,
2168            TransferProviderContractError::LogicalElementCount { .. }
2169        ));
2170        assert!(error.source().is_some());
2171    }
2172}
2173
2174fn input_signature_reads(inputs: &[TensorRead<'_>]) -> Result<InputSignature> {
2175    InputSignature::from_reads(inputs).map_err(|source| prepare_error(Arc::new(source)))
2176}
2177
2178/// Whether [`resolve_input_reads`] must re-validate the graph's retained inputs.
2179///
2180/// A retained input's value is the program's own tensor and cannot change
2181/// between calls, so a check that already ran on it need not run again. That is
2182/// a fact about the resolution, not about preparedness:
2183/// `Prepared::defaults_validated` records whether preparation
2184/// itself checked these values, and only then may an execution skip the check.
2185#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2186enum BoundInputCheck {
2187    Validate,
2188    AlreadyValidatedAtPrepare,
2189}
2190
2191fn resolve_input_reads<'a>(
2192    program: &'a CompiledGraph,
2193    inputs: &[TensorRead<'a>],
2194    bound_inputs: BoundInputCheck,
2195) -> Result<RuntimeInputReads<'a>> {
2196    let expected = program.program().inputs().len();
2197    if inputs.len() > expected {
2198        return Err(Error::GraphInputCountMismatch {
2199            expected,
2200            actual: inputs.len(),
2201        });
2202    }
2203    if inputs.is_empty() {
2204        let resolved = semantic_default_inputs(program)?;
2205        if matches!(bound_inputs, BoundInputCheck::AlreadyValidatedAtPrepare) {
2206            return Ok(resolved);
2207        }
2208        validate_ordered_input_metadata_reads(program, &resolved)?;
2209        return Ok(resolved);
2210    }
2211    let resolved = if inputs.len() == expected {
2212        inputs.to_vec()
2213    } else {
2214        let mut explicit = inputs.iter();
2215        let mut resolved = RuntimeInputReads::new();
2216        for value in program.program().inputs() {
2217            if let Some(retained) = program.bindings().tensor_ref_for_input(*value) {
2218                resolved.push(retained.tensor_read()?);
2219            } else {
2220                let read = explicit.next().ok_or_else(|| {
2221                    Error::invalid_argument(
2222                        "Runtime::run",
2223                        ErrorPhase::Execution,
2224                        "inputs",
2225                        format!(
2226                            "expected {expected} inputs, or one explicit input for each unbound placeholder"
2227                        ),
2228                    )
2229                })?;
2230                resolved.push(read.clone());
2231            }
2232        }
2233        if explicit.next().is_some() {
2234            return Err(Error::invalid_argument(
2235                "Runtime::run",
2236                ErrorPhase::Execution,
2237                "inputs",
2238                format!("expected {expected} inputs, received {}", inputs.len()),
2239            ));
2240        }
2241        resolved
2242    };
2243    validate_ordered_input_metadata_reads(program, &resolved)?;
2244    Ok(resolved)
2245}
2246
2247fn semantic_default_inputs(program: &CompiledGraph) -> Result<RuntimeInputReads<'_>> {
2248    program
2249        .program()
2250        .inputs()
2251        .iter()
2252        .enumerate()
2253        .map(|(input_index, value)| {
2254            let retained = program
2255                .bindings()
2256                .tensor_ref_for_input(*value)
2257                .ok_or_else(|| Error::UnboundPlaceholder {
2258                    input_key: format!("semantic input {input_index}"),
2259                })?;
2260            retained.tensor_read()
2261        })
2262        .collect()
2263}
2264
2265fn validate_ordered_input_metadata_reads(
2266    program: &CompiledGraph,
2267    inputs: &[TensorRead<'_>],
2268) -> Result<()> {
2269    let actuals = inputs
2270        .iter()
2271        .map(|input| (input.dtype(), input.shape().to_vec()))
2272        .collect::<Vec<_>>();
2273    validate_ordered_input_metadata_values(program, &actuals, "Runtime::submit")
2274}
2275
2276fn validate_ordered_input_metadata_values(
2277    program: &CompiledGraph,
2278    actuals: &[(tenferro_tensor::DType, Vec<usize>)],
2279    caller: &'static str,
2280) -> Result<()> {
2281    let expected = program.input_count();
2282    if actuals.len() != expected {
2283        return Err(Error::GraphInputCountMismatch {
2284            expected,
2285            actual: actuals.len(),
2286        });
2287    }
2288    let input_shapes: RuntimeInputShapes<'_> =
2289        actuals.iter().map(|(_, shape)| shape.as_slice()).collect();
2290    for (input_value, (actual_dtype, actual_shape)) in
2291        program.program().inputs().iter().zip(actuals)
2292    {
2293        let metadata = program
2294            .program()
2295            .value_metadata(*input_value)
2296            .map_err(|source| Error::runtime_state_source(caller, ErrorPhase::Execution, source))?;
2297        if metadata.dtype() != *actual_dtype {
2298            return Err(Error::PlaceholderDtypeMismatch {
2299                expected: metadata.dtype(),
2300                actual: *actual_dtype,
2301            });
2302        }
2303        if metadata.shape().len() != actual_shape.len() {
2304            return Err(Error::PlaceholderRankMismatch {
2305                expected: metadata.shape().len(),
2306                actual: actual_shape.len(),
2307            });
2308        }
2309        let mut expected_shape: RuntimeShapeScratch = actual_shape.iter().copied().collect();
2310        let mut exact_mismatch = false;
2311        for (axis, (extent, actual_size)) in metadata.shape().iter().zip(actual_shape).enumerate() {
2312            match extent {
2313                ShapeExtent::Exact(expression) => {
2314                    let expected = expression.eval(&input_shapes).map_err(|source| {
2315                        Error::runtime_state_source(caller, ErrorPhase::Execution, source)
2316                    })?;
2317                    expected_shape[axis] = expected;
2318                    exact_mismatch |= expected != *actual_size;
2319                }
2320                ShapeExtent::UpperBound(expression) => {
2321                    let bound = expression.eval(&input_shapes).map_err(|source| {
2322                        Error::runtime_state_source(caller, ErrorPhase::Execution, source)
2323                    })?;
2324                    if *actual_size > bound {
2325                        return Err(Error::PlaceholderShapeBoundExceeded {
2326                            axis,
2327                            bound,
2328                            actual: *actual_size,
2329                        });
2330                    }
2331                }
2332                ShapeExtent::Unknown => {}
2333            }
2334        }
2335        if exact_mismatch {
2336            return Err(Error::PlaceholderShapeMismatch {
2337                expected: expected_shape.into_vec(),
2338                actual: actual_shape.clone(),
2339            });
2340        }
2341    }
2342    Ok(())
2343}
2344
2345fn validate_exec_input_count(program: &ExecProgram, actual: usize) -> Result<()> {
2346    let expected = program.input_slots.len();
2347    if actual != expected {
2348        return Err(Error::runtime_state(
2349            "Runtime::run",
2350            ErrorPhase::Execution,
2351            format!("expected {expected} inputs for execution program, got {actual}"),
2352        ));
2353    }
2354    Ok(())
2355}
2356
2357fn prepare_error(source: Arc<PrepareError>) -> Error {
2358    Error::runtime_state_source(
2359        "Runtime::run",
2360        ErrorPhase::Execution,
2361        SharedPrepareError(source),
2362    )
2363}
2364
2365#[derive(Clone, Debug)]
2366struct SharedPrepareError(Arc<PrepareError>);
2367
2368impl fmt::Display for SharedPrepareError {
2369    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
2370        write!(formatter, "{}", self.0)
2371    }
2372}
2373
2374impl StdError for SharedPrepareError {
2375    fn source(&self) -> Option<&(dyn StdError + 'static)> {
2376        Some(self.0.as_ref())
2377    }
2378}