1use num_complex::{Complex32, Complex64};
9use std::collections::{HashMap, HashSet};
10use std::error::Error as StdError;
11use std::fmt;
12use std::panic::AssertUnwindSafe;
13use std::sync::Arc;
14
15use tenferro_ops::shape_extent::ShapeExtent;
16use tenferro_tensor::{DType, Engine, Tensor, TensorRead};
17use tinyvec::TinyVec;
18
19use crate::error::ErrorPhase;
20use crate::exec::{ExecInstruction, ExecProgram, ExecSlot, ExtensionExecutionDispatch};
21use crate::extension_cache::ExtensionCacheStore;
22use crate::graph::CompiledGraph;
23use crate::runtime::schedule::{
24 EventDependency, ExecutionLocation, ScheduledGraph, ScheduledNode, ScheduledNodeKind,
25 ScheduledTransfer, UnsupportedScheduledNodeError,
26};
27use crate::runtime::{
28 EngineId, EngineTarget, EventDomainDriver, EventDomainError, EventDomainOperation,
29 EventDomainRun, EventToken, ExecutionEngine, InputSignature, Output, PrepareError,
30 PrepareOptions, PreparedOperationPlan, ProgramPlacementConstraint, Runtime, TransferError,
31 TransferProviderContractError, TransferRequest,
32};
33use crate::{Error, Result};
34
35type RuntimeInputReads<'a> = Vec<TensorRead<'a>>;
40type RuntimeInputShapes<'a> = TinyVec<[&'a [usize]; 8]>;
41type RuntimeShapeScratch = TinyVec<[usize; 8]>;
42
43#[derive(Clone)]
61pub struct Prepared {
62 runtime_id: super::RuntimeId,
63 epoch: super::RuntimeEpoch,
64 program: CompiledGraph,
65 prepared: Arc<super::preparation::PreparedProgram>,
66 defaults_validated: bool,
72 targets: Arc<[EngineTarget]>,
74 signature: Arc<super::InputSignature>,
81}
82
83impl fmt::Debug for Prepared {
84 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
85 formatter
86 .debug_struct("Prepared")
87 .field("runtime_id", &self.runtime_id)
88 .field("epoch", &self.epoch)
89 .field("targets", &self.targets)
90 .finish_non_exhaustive()
91 }
92}
93
94impl Prepared {
95 #[must_use]
109 pub fn targets(&self) -> &[EngineTarget] {
110 &self.targets
111 }
112
113 fn bound_input_check(&self) -> BoundInputCheck {
118 if self.defaults_validated {
119 BoundInputCheck::AlreadyValidatedAtPrepare
120 } else {
121 BoundInputCheck::Validate
122 }
123 }
124
125 #[doc(hidden)]
128 #[must_use]
129 pub fn elementwise_region_summary(&self) -> (usize, usize) {
130 super::region::region_summary(self.prepared.root().regions())
131 }
132
133 #[doc(hidden)]
136 #[must_use]
137 pub fn elementwise_region_execution_counts(&self) -> (usize, usize) {
138 let counters = self.prepared.root().region_counters();
139 (counters.fused(), counters.fallbacks())
140 }
141
142 #[doc(hidden)]
147 #[must_use]
148 pub fn execution_submission_count(&self) -> usize {
149 self.prepared.root().region_counters().submissions()
150 }
151
152 #[doc(hidden)]
158 #[must_use]
159 pub fn execution_command_count(&self) -> usize {
160 let root = self.prepared.root();
161 super::region::command_count(root.schedule(), root.regions())
162 }
163}
164
165fn run_extension_callback<R>(
167 engine: &mut dyn ExecutionEngine,
168 callback: impl FnOnce(&mut dyn ExecutionEngine) -> Result<R>,
169) -> Result<R> {
170 match std::panic::catch_unwind(AssertUnwindSafe(|| callback(engine))) {
171 Ok(result) => result,
172 Err(payload) => {
173 engine.mark_broken();
174 std::panic::resume_unwind(payload)
175 }
176 }
177}
178
179fn execute_slot_instruction_on_engine<'input>(
187 engine: &mut dyn ExecutionEngine,
188 extension_caches: Option<&mut ExtensionCacheStore>,
189 operations: &[PreparedOperationPlan],
190 instruction_index: usize,
191 instruction: &ExecInstruction,
192 slots: &mut [Option<ExecSlot<'input>>],
193) -> Result<()> {
194 if matches!(&instruction.op, crate::exec::ExecOp::Extension(_)) {
195 let caches = extension_caches.ok_or_else(|| {
196 Error::runtime_state(
197 "Runtime::run",
198 ErrorPhase::Execution,
199 "engine extension caches were not taken before an extension command",
200 )
201 })?;
202 let mut extension_dispatch = ExtensionExecutionDispatch { operations, caches };
203 if crate::exec::needs_owner_extension_fallback(instruction, Some(&extension_dispatch)) {
204 let identity = engine.target().context_identity();
207 run_extension_callback(engine, |engine| {
208 crate::exec::execute_owner_extension_fallback(
209 engine.as_any_mut(),
210 identity,
211 slots,
212 instruction,
213 Some(&mut extension_dispatch),
214 )
215 })?;
216 } else {
217 run_extension_callback(engine, |engine| {
218 crate::exec::execute_ffi_instruction_exec(
219 engine,
220 slots,
221 instruction,
222 Some(instruction_index),
223 Some(&mut extension_dispatch),
224 )
225 })?;
226 }
227 crate::exec::reclaim_last_use_inputs_exec(slots, instruction, engine);
228 return Ok(());
229 }
230 if crate::exec::is_host_instruction(instruction) {
231 crate::exec::execute_host_instruction_exec(engine, slots, instruction)?;
232 } else if crate::exec::is_ffi_instruction(instruction) {
233 crate::exec::execute_ffi_instruction_exec(
234 engine,
235 slots,
236 instruction,
237 Some(instruction_index),
238 None,
239 )?;
240 } else if let Some(view) = crate::exec::execute_backend_view_op(slots, instruction) {
241 slots[instruction.output_slots[0]] = Some(view);
242 } else {
243 let result = crate::exec::execute_backend_op(engine, slots, instruction)?;
244 slots[instruction.output_slots[0]] = Some(ExecSlot::Owned(result));
245 }
246 crate::exec::reclaim_last_use_inputs_exec(slots, instruction, engine);
247 Ok(())
248}
249
250fn execute_elementwise_fusion_slots_on_engine<'input>(
252 exec: &mut dyn Engine,
253 input_slots: &[usize],
254 instruction_count: usize,
255 plan: &tenferro_tensor::backend::ElementwiseFusionPlan,
256 slots: &mut [Option<ExecSlot<'input>>],
257 output_slots: &[usize],
258) -> Result<bool> {
259 let mut views: Vec<usize> = Vec::new();
264 for &slot in input_slots {
265 let value = slots
266 .get(slot)
267 .and_then(Option::as_ref)
268 .ok_or_else(|| crate::Error::from(tenferro_tensor::Error::MissingValue { slot }))?;
269 if value.as_tensor("elementwise_region").is_err() {
270 views.push(slot);
271 }
272 }
273 if !views.is_empty() {
274 if instruction_count < super::region::VIEW_COPY_MIN_INSTRUCTIONS {
275 return Ok(false);
276 }
277 for &slot in &views {
278 let tensor = {
279 let read = slots[slot].as_ref().map(ExecSlot::as_read).ok_or_else(|| {
280 crate::Error::from(tenferro_tensor::Error::MissingValue { slot })
281 })?;
282 exec.to_contiguous_read(read)?
283 };
284 slots[slot] = Some(ExecSlot::Owned(tensor));
285 }
286 }
287
288 let mut inputs: Vec<&Tensor> = Vec::with_capacity(input_slots.len());
289 for &slot in input_slots {
290 let value = slots
291 .get(slot)
292 .and_then(Option::as_ref)
293 .ok_or_else(|| crate::Error::from(tenferro_tensor::Error::MissingValue { slot }))?;
294 inputs.push(value.as_tensor("elementwise_region")?);
295 }
296
297 if !crate::segment::plan_matches_input_dtypes(plan, &inputs) {
298 return Ok(false);
299 }
300
301 let outputs = exec.execute_elementwise_fusion(&inputs, plan)?;
302 let Some(outputs) = outputs else {
303 return Ok(false);
304 };
305 if outputs.len() != output_slots.len() {
306 return Err(crate::Error::Internal(format!(
307 "fused elementwise region produced {} outputs for {} slots",
308 outputs.len(),
309 output_slots.len()
310 )));
311 }
312 for (&slot, tensor) in output_slots.iter().zip(outputs) {
313 slots[slot] = Some(ExecSlot::Owned(tensor));
314 }
315 Ok(true)
316}
317
318pub(super) fn prepare(
320 runtime: &Runtime,
321 program: &CompiledGraph,
322 inputs: &[TensorRead<'_>],
323 targets: &[EngineTarget],
324) -> Result<Prepared> {
325 const CALLER: &str = "Runtime::prepare";
326 if targets.is_empty() {
327 return Err(Error::invalid_argument(
328 CALLER,
329 ErrorPhase::Execution,
330 "targets",
331 "prepare needs at least one engine target",
332 ));
333 }
334 let snapshot = runtime
335 .snapshot()
336 .map_err(|source| Error::runtime_state_source(CALLER, ErrorPhase::Execution, source))?;
337 let mut engine_ids = Vec::with_capacity(targets.len());
338 for (index, target) in targets.iter().enumerate() {
339 if targets[..index]
340 .iter()
341 .any(|earlier| earlier.engine_id() == target.engine_id())
342 {
343 return Err(Error::invalid_argument(
344 CALLER,
345 ErrorPhase::Execution,
346 "targets",
347 format!(
348 "engine {:?} is named by more than one target",
349 target.engine_id().as_str()
350 ),
351 ));
352 }
353 let view = snapshot.engine(target.engine_id()).ok_or_else(|| {
354 Error::runtime_state(
355 CALLER,
356 ErrorPhase::Execution,
357 format!(
358 "no engine {:?} is registered with this runtime",
359 target.engine_id().as_str()
360 ),
361 )
362 })?;
363 if view.context_identity() != target.context_identity() {
364 return Err(Error::runtime_state(
365 CALLER,
366 ErrorPhase::Execution,
367 format!(
368 "engine {:?} is registered for context {:?}, but the target reports {:?}",
369 target.engine_id().as_str(),
370 view.context_identity(),
371 target.context_identity()
372 ),
373 ));
374 }
375 engine_ids.push(target.engine_id().clone());
376 }
377 let defaults_validated = retained_inputs_validated(program, inputs.len());
378 let inputs = resolve_input_reads(program, inputs, BoundInputCheck::Validate)?;
379 let signature = input_signature_reads(&inputs)?;
380 super::preparation::validate_resolved_extension_extents(program.frozen(), &signature)?;
384 let placement = ProgramPlacementConstraint::new(engine_ids, None)
385 .map_err(|source| Error::runtime_state_source(CALLER, ErrorPhase::Execution, source))?;
386 let options = PrepareOptions::new().with_placement(placement);
387 let prepared = runtime
388 .prepare_compiled_for(program, &signature, &options)
389 .map_err(prepare_error)?;
390 validate_prepared_epoch(runtime, prepared.root().epoch(), CALLER)?;
391 Ok(Prepared {
392 runtime_id: runtime.id(),
393 epoch: prepared.root().epoch(),
394 program: program.clone(),
395 prepared,
396 defaults_validated,
397 targets: targets.into(),
398 signature: Arc::new(signature),
399 })
400}
401
402fn retained_inputs_validated(program: &CompiledGraph, input_count: usize) -> bool {
409 let expected = program.program().inputs().len();
410 input_count != expected || expected == 0
411}
412
413pub(super) fn run<'a>(
415 runtime: &Runtime,
416 engines: &mut [&mut dyn ExecutionEngine],
417 prepared: &'a Prepared,
418 inputs: &[TensorRead<'a>],
419) -> Result<Vec<Output<'a>>> {
420 const CALLER: &str = "Runtime::run";
421 validate_prepared_runtime(runtime, prepared, CALLER)?;
422 let mut engines = EngineSet::bind(engines, &prepared.targets, CALLER)?;
423 let inputs = resolve_input_reads(&prepared.program, inputs, prepared.bound_input_check())?;
424 let signature = input_signature_reads(&inputs)?;
425 if *prepared.signature != signature {
426 super::preparation::validate_execution_signature(
427 prepared.prepared.as_ref(),
428 prepared.program.frozen(),
429 &signature,
430 )
431 .map_err(prepare_error)?;
432 }
433 let root = prepared.prepared.root();
434 engines.take_caches(root.needs_extension_caches());
435 let result = execute_scheduled(root, prepared.prepared.operations(), inputs, &mut engines);
436 engines.restore_caches();
437 result
438}
439
440fn validate_prepared_runtime(
441 runtime: &Runtime,
442 prepared: &Prepared,
443 caller: &'static str,
444) -> Result<()> {
445 if runtime.id() != prepared.runtime_id {
446 return Err(Error::runtime_state(
447 caller,
448 ErrorPhase::Execution,
449 "prepared program belongs to a different runtime",
450 ));
451 }
452 validate_prepared_epoch(runtime, prepared.epoch, caller)
453}
454
455fn validate_prepared_epoch(
456 runtime: &Runtime,
457 prepared_epoch: super::RuntimeEpoch,
458 caller: &'static str,
459) -> Result<()> {
460 let epoch = runtime
461 .epoch()
462 .map_err(|source| Error::runtime_state_source(caller, ErrorPhase::Execution, source))?;
463 if epoch != prepared_epoch {
464 return Err(Error::runtime_state(
465 caller,
466 ErrorPhase::Execution,
467 format!(
468 "prepared epoch {:?} does not match current epoch {:?}",
469 prepared_epoch, epoch
470 ),
471 ));
472 }
473 Ok(())
474}
475
476struct EngineSet<'e, 'r> {
483 engines: &'e mut [&'r mut dyn ExecutionEngine],
484 targets: &'e [EngineTarget],
485 caches: Vec<ExtensionCacheStore>,
486}
487
488impl<'e, 'r> EngineSet<'e, 'r> {
489 fn bind(
490 engines: &'e mut [&'r mut dyn ExecutionEngine],
491 targets: &'e [EngineTarget],
492 caller: &'static str,
493 ) -> Result<Self> {
494 if engines.len() != targets.len() {
495 return Err(Error::invalid_argument(
496 caller,
497 ErrorPhase::Execution,
498 "engines",
499 format!(
500 "the program was prepared for {} engines, but {} were passed",
501 targets.len(),
502 engines.len()
503 ),
504 ));
505 }
506 for (index, (engine, target)) in engines.iter().zip(targets).enumerate() {
507 let actual = engine.target();
508 if &actual != target {
509 return Err(Error::invalid_argument(
510 caller,
511 ErrorPhase::Execution,
512 "engines",
513 format!(
514 "engine {index} reports target {actual:?}, but the program was prepared for {target:?}"
515 ),
516 ));
517 }
518 }
519 Ok(Self {
520 engines,
521 targets,
522 caches: Vec::new(),
523 })
524 }
525
526 fn take_caches(&mut self, needed: bool) {
531 if needed {
532 self.caches = self
533 .engines
534 .iter_mut()
535 .map(|engine| engine.take_extension_caches())
536 .collect();
537 }
538 }
539
540 fn restore_caches(&mut self) {
541 for (engine, caches) in self.engines.iter_mut().zip(self.caches.drain(..)) {
542 engine.restore_extension_caches(caches);
543 }
544 }
545
546 fn index_of(&self, engine_id: &EngineId) -> Result<usize> {
547 self.targets
548 .iter()
549 .position(|target| target.engine_id() == engine_id)
550 .ok_or_else(|| {
551 Error::runtime_state(
552 "Runtime::run",
553 ErrorPhase::Execution,
554 format!(
555 "the prepared schedule uses engine {:?}, which is not among the run's engines",
556 engine_id.as_str()
557 ),
558 )
559 })
560 }
561
562 fn engine(&mut self, engine_id: &EngineId) -> Result<&mut dyn ExecutionEngine> {
563 let index = self.index_of(engine_id)?;
564 Ok(&mut *self.engines[index])
565 }
566
567 fn driver(&self, engine_id: &EngineId) -> Result<Arc<dyn EventDomainDriver>> {
568 let index = self.index_of(engine_id)?;
569 Ok(self.engines[index].event_domain_driver())
570 }
571
572 fn execute_slot_instruction<'input>(
573 &mut self,
574 location: &ExecutionLocation,
575 instruction_index: usize,
576 instruction: &ExecInstruction,
577 operations: &[PreparedOperationPlan],
578 slots: &mut [Option<ExecSlot<'input>>],
579 ) -> Result<()> {
580 let index = self.index_of(location.engine_id())?;
581 let engine = &mut *self.engines[index];
582 execute_slot_instruction_on_engine(
583 engine,
584 self.caches.get_mut(index),
585 operations,
586 instruction_index,
587 instruction,
588 slots,
589 )
590 }
591
592 fn execute_elementwise_fusion_slots<'input>(
593 &mut self,
594 location: &ExecutionLocation,
595 input_slots: &[usize],
596 instruction_count: usize,
597 plan: &tenferro_tensor::backend::ElementwiseFusionPlan,
598 slots: &mut [Option<ExecSlot<'input>>],
599 output_slots: &[usize],
600 ) -> Result<bool> {
601 let index = self.index_of(location.engine_id())?;
602 execute_elementwise_fusion_slots_on_engine(
603 &mut *self.engines[index],
604 input_slots,
605 instruction_count,
606 plan,
607 slots,
608 output_slots,
609 )
610 }
611
612 fn retire(&mut self, event_domains: ScheduledEventDomains) -> Result<()> {
614 let mut failures = Vec::new();
615 for (engine_id, run) in event_domains.into_pending_runs() {
616 let result = match self.index_of(&engine_id) {
617 Ok(index) => self.engines[index].retire_run(run),
618 Err(error) => {
619 drop_event_domain_run(run);
620 Err(error)
621 }
622 };
623 if let Err(error) = result {
624 failures.push(error);
625 }
626 }
627 let mut failures = failures.into_iter();
628 let Some(mut error) = failures.next() else {
629 return Ok(());
630 };
631 for failure in failures {
632 error = Error::with_suppressed(error, failure);
633 }
634 Err(error)
635 }
636}
637
638impl Drop for EngineSet<'_, '_> {
639 fn drop(&mut self) {
640 self.restore_caches();
641 }
642}
643
644fn execute_scheduled<'input>(
645 root: &super::preparation::PreparedProgramRoot,
646 operations: &[PreparedOperationPlan],
647 inputs: RuntimeInputReads<'input>,
648 engines: &mut EngineSet<'_, '_>,
649) -> Result<Vec<Output<'input>>> {
650 let program = root.staging();
651 let schedule = root.schedule();
652 validate_exec_input_count(program, inputs.len())?;
655 let mut event_domains = Some(ScheduledEventDomains::new(schedule, engines)?);
656 let flat = schedule.flat_location();
657 let executed = execute_scheduled_slots_into(
658 program,
659 schedule,
660 flat,
661 root.regions(),
662 root.node_regions(),
663 root.region_counters(),
664 operations,
665 inputs,
666 engines,
667 &mut event_domains,
668 );
669 let Some(event_domains) = event_domains.take() else {
670 return executed.and_then(|_| {
671 Err(Error::runtime_state(
672 "Runtime::run",
673 ErrorPhase::Execution,
674 "scheduled execution consumed its event domains before collecting outputs",
675 ))
676 });
677 };
678 let collected = executed
679 .and_then(|slots| collect_outputs(program, slots, flat, root.output_duplicates(), engines));
680 match collected {
681 Ok(outputs) => {
682 engines.retire(event_domains)?;
683 Ok(outputs)
684 }
685 Err(primary) => {
686 let mut event_domains = event_domains;
689 match event_domains.drain() {
690 Ok(()) => Err(primary),
691 Err(cleanup) => Err(scheduled_execution_cleanup_error(primary, cleanup)),
692 }
693 }
694 }
695}
696
697fn collect_outputs<'input>(
706 program: &ExecProgram,
707 slots: RunSlots<'input>,
708 flat: Option<&ExecutionLocation>,
709 duplicates: &[Option<usize>],
710 engines: &mut EngineSet<'_, '_>,
711) -> Result<Vec<Output<'input>>> {
712 let RunSlots {
713 mut staged,
714 mut located,
715 } = slots;
716 let mut collected: Vec<Output<'input>> = Vec::with_capacity(program.output_slots.len());
717 let mut locations: Vec<Option<ExecutionLocation>> = Vec::new();
720 for (position, &slot) in program.output_slots.iter().enumerate() {
721 if let Some(first) = duplicates.get(position).copied().flatten() {
722 let output = match collected.get(first) {
723 Some(Output::Borrowed(read)) => Output::Borrowed(read.clone()),
724 Some(Output::Owned(tensor)) => {
725 let location = match flat {
726 Some(location) => location,
727 None => locations
728 .get(first)
729 .and_then(Option::as_ref)
730 .ok_or(tenferro_tensor::Error::MissingValue { slot })?,
731 };
732 Output::Owned(duplicate_owned_output(engines, location, tensor)?)
733 }
734 None => return Err(tenferro_tensor::Error::MissingValue { slot }.into()),
735 };
736 collected.push(output);
737 if flat.is_none() {
738 locations.push(None);
739 }
740 continue;
741 }
742 let value = match (flat, &mut located) {
743 (Some(_), _) => staged
744 .get_mut(slot)
745 .and_then(Option::take)
746 .ok_or(tenferro_tensor::Error::MissingValue { slot })?,
747 (None, Some(located)) => {
748 let values = located
749 .get_mut(slot)
750 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
751 let value = values
752 .pop()
753 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
754 values.clear();
755 locations.push(Some(value.location));
756 value.value
757 }
758 (None, None) => {
759 return Err(Error::runtime_state(
760 "Runtime::run",
761 ErrorPhase::Execution,
762 "a multi-location run lost its value lists".to_string(),
763 ))
764 }
765 };
766 collected.push(match value {
767 ExecSlot::Read(read) => Output::Borrowed(read),
768 ExecSlot::Owned(tensor) => Output::Owned(tensor),
769 });
770 }
771 Ok(collected)
772}
773
774fn duplicate_owned_output(
775 engines: &mut EngineSet<'_, '_>,
776 location: &ExecutionLocation,
777 tensor: &Tensor,
778) -> Result<Tensor> {
779 let engine = engines.engine(location.engine_id())?;
780 engine
781 .to_contiguous_read(TensorRead::from_tensor(tensor))
782 .map_err(Error::from)
783}
784
785struct RunSlots<'input> {
793 staged: Vec<Option<ExecSlot<'input>>>,
795 located: Option<Vec<Vec<LocatedExecSlot<'input>>>>,
797}
798
799impl<'input> RunSlots<'input> {
800 fn new(n_slots: usize, flat: bool) -> Self {
801 Self {
802 staged: Vec::new(),
803 located: (!flat).then(|| (0..n_slots).map(|_| Vec::new()).collect()),
804 }
805 }
806
807 fn initialize(
809 &mut self,
810 program: &ExecProgram,
811 inputs: RuntimeInputReads<'input>,
812 schedule: &ScheduledGraph,
813 ) -> Result<()> {
814 crate::exec::initialize_exec_slots_in(
815 program,
816 inputs.into_iter().map(ExecSlot::Read),
817 &mut self.staged,
818 )?;
819 if schedule.input_locations().len() != program.input_slots.len() {
820 return Err(Error::runtime_state(
821 "Runtime::run",
822 ErrorPhase::Execution,
823 format!(
824 "prepared schedule has {} input locations for {} inputs",
825 schedule.input_locations().len(),
826 program.input_slots.len()
827 ),
828 ));
829 }
830 for (&slot, location) in program.input_slots.iter().zip(schedule.input_locations()) {
831 match &mut self.located {
832 Some(located) => {
833 let value = self
834 .staged
835 .get_mut(slot)
836 .and_then(Option::take)
837 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
838 validate_runtime_input_ingress(location, &value.as_read(), slot)?;
839 located[slot].push(LocatedExecSlot {
840 location: location.clone(),
841 value,
842 });
843 }
844 None => {
845 let value = self
846 .staged
847 .get(slot)
848 .and_then(Option::as_ref)
849 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
850 validate_runtime_input_ingress(location, &value.as_read(), slot)?;
851 }
852 }
853 }
854 Ok(())
855 }
856
857 fn execution(&mut self) -> &mut [Option<ExecSlot<'input>>] {
858 &mut self.staged
859 }
860
861 fn execution_ref(&self) -> &[Option<ExecSlot<'input>>] {
862 &self.staged
863 }
864
865 fn stage_command(
867 &mut self,
868 instruction: &ExecInstruction,
869 location: &ExecutionLocation,
870 ) -> Result<()> {
871 match &mut self.located {
872 Some(located) => {
873 stage_instruction_inputs(instruction, location, located, &mut self.staged)
874 }
875 None => Ok(()),
876 }
877 }
878
879 fn publish_command(
881 &mut self,
882 instruction: &ExecInstruction,
883 location: &ExecutionLocation,
884 ) -> Result<()> {
885 match &mut self.located {
886 Some(located) => {
887 retain_instruction_results(instruction, location, located, &mut self.staged)
888 }
889 None => Ok(()),
890 }
891 }
892
893 fn stage_region(
895 &mut self,
896 region: &super::region::ElementwiseRegion,
897 location: &ExecutionLocation,
898 ) -> Result<()> {
899 let Some(located) = &mut self.located else {
900 return Ok(());
901 };
902 for &slot in ®ion.input_slots {
903 stage_slot_input(slot, location, located, &mut self.staged)?;
904 }
905 Ok(())
906 }
907
908 fn publish_region(
911 &mut self,
912 region: &super::region::ElementwiseRegion,
913 location: &ExecutionLocation,
914 ) -> Result<()> {
915 match &mut self.located {
916 Some(located) => retain_region_results(region, location, located, &mut self.staged),
917 None => {
918 for (&slot, &last_use) in
919 region.input_slots.iter().zip(region.input_last_use.iter())
920 {
921 if last_use {
922 self.staged[slot] = None;
923 }
924 }
925 Ok(())
926 }
927 }
928 }
929
930 fn transfers(&mut self) -> Result<&mut [Vec<LocatedExecSlot<'input>>]> {
932 self.located.as_deref_mut().ok_or_else(|| {
933 Error::runtime_state(
934 "Runtime::run",
935 ErrorPhase::Execution,
936 "a single-location run must not contain a transfer node".to_string(),
937 )
938 })
939 }
940}
941
942fn command_location<'a>(
948 schedule: &'a ScheduledGraph,
949 flat: Option<&'a ExecutionLocation>,
950 instruction: &ExecInstruction,
951 node_location: &ExecutionLocation,
952) -> Result<&'a ExecutionLocation> {
953 if let Some(location) = flat {
954 return Ok(location);
955 }
956 let location = instruction_execution(schedule, instruction)?.location();
957 if location != node_location {
958 return Err(Error::runtime_state(
959 "Runtime::run",
960 ErrorPhase::Execution,
961 "a prepared command's location does not match its scheduled node".to_string(),
962 ));
963 }
964 Ok(location)
965}
966
967#[allow(clippy::too_many_arguments)]
968fn execute_scheduled_slots_into<'input>(
969 program: &ExecProgram,
970 schedule: &ScheduledGraph,
971 flat: Option<&ExecutionLocation>,
972 regions: &[super::region::ElementwiseRegion],
973 node_regions: &[super::region::NodeRegion],
974 region_counters: &super::region::RegionExecutionCounters,
975 operations: &[PreparedOperationPlan],
976 inputs: RuntimeInputReads<'input>,
977 engines: &mut EngineSet<'_, '_>,
978 event_domains: &mut Option<ScheduledEventDomains>,
979) -> Result<RunSlots<'input>> {
980 let domains = event_domains.as_mut().ok_or_else(|| {
981 Error::runtime_state(
982 "Runtime::run",
983 ErrorPhase::Execution,
984 "scheduled execution has no event-domain run to submit into",
985 )
986 })?;
987 let mut slots = RunSlots::new(program.n_slots, flat.is_some());
988 let result = std::panic::catch_unwind(AssertUnwindSafe(|| -> Result<RunSlots<'input>> {
996 (|| {
997 slots.initialize(program, inputs, schedule)?;
998 for (node_index, node) in schedule.nodes().iter().enumerate() {
999 match node {
1000 ScheduledNode::Operation(operation_node) => {
1001 let role = node_regions
1002 .get(node_index)
1003 .copied()
1004 .unwrap_or(super::region::NodeRegion::Command);
1005 if let super::region::NodeRegion::Start(region_index) = role {
1006 let region = ®ions[region_index];
1007 let first_instruction = program
1008 .instructions
1009 .get(region.instruction_range.start)
1010 .ok_or_else(|| {
1011 Error::runtime_state(
1012 "Runtime::run",
1013 ErrorPhase::Execution,
1014 format!(
1015 "elementwise region references instruction {}, but the execution program has {} instructions",
1016 region.instruction_range.start,
1017 program.instructions.len()
1018 ),
1019 )
1020 })?;
1021 let location = command_location(
1022 schedule,
1023 flat,
1024 first_instruction,
1025 operation_node.location(),
1026 )?;
1027 let mut launch = || {
1028 slots.stage_region(region, location)?;
1029 let applied = engines.execute_elementwise_fusion_slots(
1030 location,
1031 ®ion.input_slots,
1032 region.instructions.len(),
1033 ®ion.plan,
1034 slots.execution(),
1035 ®ion.output_slots,
1036 )?;
1037 if applied {
1038 region_counters.record_fused();
1039 validate_region_outputs(
1040 region,
1041 location,
1042 slots.execution_ref(),
1043 )?;
1044 return slots.publish_region(region, location);
1045 }
1046 region_counters.record_fallback();
1049 for (offset, instruction) in region.instructions.iter().enumerate()
1050 {
1051 let instruction_index = region.instruction_range.start + offset;
1052 let member_location = command_location(
1053 schedule,
1054 flat,
1055 instruction,
1056 operation_node.location(),
1057 )?;
1058 slots.stage_command(instruction, member_location)?;
1059 engines.execute_slot_instruction(
1060 member_location,
1061 instruction_index,
1062 instruction,
1063 operations,
1064 slots.execution(),
1065 )?;
1066 validate_instruction_outputs(
1067 instruction_index,
1068 instruction,
1069 member_location,
1070 slots.execution_ref(),
1071 )?;
1072 slots.publish_command(instruction, member_location)?;
1073 }
1074 Ok(())
1075 };
1076 domains.enqueue(node_index, node, &mut launch)?;
1077 region_counters.record_submission();
1078 let completion = EventDependency::from_completion(node.completion());
1082 if let Some(token) = domains.completions.get(&completion).cloned() {
1083 for &covered in ®ion.node_indices {
1084 if covered == node_index {
1085 continue;
1086 }
1087 let covered_node = schedule.nodes().get(covered).ok_or_else(|| {
1088 Error::runtime_state(
1089 "Runtime::run",
1090 ErrorPhase::Execution,
1091 format!(
1092 "elementwise region references schedule node {covered}, but the schedule has {} nodes",
1093 schedule.nodes().len()
1094 ),
1095 )
1096 })?;
1097 domains.completions.insert(
1098 EventDependency::from_completion(covered_node.completion()),
1099 Arc::clone(&token),
1100 );
1101 }
1102 }
1103 continue;
1104 }
1105 if role == super::region::NodeRegion::Covered {
1106 continue;
1108 }
1109 let mut launch = || {
1110 let instruction_index = operation_node.instruction_index();
1111 let instruction = program
1112 .instructions
1113 .get(instruction_index)
1114 .ok_or_else(|| {
1115 Error::runtime_state(
1116 "Runtime::run",
1117 ErrorPhase::Execution,
1118 format!(
1119 "scheduled operation references instruction {instruction_index}, but the execution program has {} instructions",
1120 program.instructions.len()
1121 ),
1122 )
1123 })?;
1124 let location = command_location(
1125 schedule,
1126 flat,
1127 instruction,
1128 operation_node.location(),
1129 )?;
1130 slots.stage_command(instruction, location)?;
1131 engines.execute_slot_instruction(
1132 location,
1133 instruction_index,
1134 instruction,
1135 operations,
1136 slots.execution(),
1137 )?;
1138 validate_instruction_outputs(
1139 instruction_index,
1140 instruction,
1141 location,
1142 slots.execution_ref(),
1143 )?;
1144 slots.publish_command(instruction, location)
1145 };
1146 domains.enqueue(node_index, node, &mut launch)?;
1147 region_counters.record_submission();
1148 }
1149 ScheduledNode::Transfer(transfer) => {
1150 let mut launch =
1151 || execute_scheduled_transfer(transfer, slots.transfers()?);
1152 domains.enqueue(node_index, node, &mut launch)?;
1153 region_counters.record_submission();
1154 }
1155 ScheduledNode::Collective(_) => {
1156 return Err(Error::runtime_state_source(
1157 "Runtime::run",
1158 ErrorPhase::Execution,
1159 UnsupportedScheduledNodeError {
1160 node_index,
1161 node_kind: ScheduledNodeKind::Collective,
1162 },
1163 ));
1164 }
1165 ScheduledNode::Barrier(_) => {
1166 let mut launch = || Ok(());
1167 domains.enqueue(node_index, node, &mut launch)?;
1168 region_counters.record_submission();
1169 }
1170 }
1171 }
1172 Ok(())
1173 })()?;
1174 Ok(slots)
1175 }));
1176 match result {
1177 Ok(slots) => slots,
1178 Err(payload) => {
1179 event_domains.take();
1183 std::panic::resume_unwind(payload)
1184 }
1185 }
1186}
1187
1188#[derive(Debug)]
1189pub(crate) struct ScheduledEventDomains {
1190 runs: Vec<RuntimeOwnedEventDomainRun>,
1191 completions: HashMap<EventDependency, Arc<dyn EventToken>>,
1192 coalesced: bool,
1196}
1197
1198#[derive(Debug, thiserror::Error)]
1199#[error(
1200 "scheduled node {node_index} depends on completion {dependency:?}, but no completion token was recorded"
1201)]
1202pub(crate) struct MissingScheduledDependencyCompletionError {
1203 pub(crate) dependency: EventDependency,
1204 pub(crate) node_index: usize,
1205}
1206
1207#[derive(Debug, thiserror::Error)]
1208#[error("scheduled transfer destination {destination:?} already contains value slot {value_slot}")]
1209pub(crate) struct DuplicateTransferDestinationError {
1210 pub(crate) value_slot: usize,
1211 pub(crate) destination: ExecutionLocation,
1212}
1213
1214impl ScheduledEventDomains {
1215 #[cfg(test)]
1217 pub(crate) fn completion_count_for_test(&self) -> usize {
1218 self.completions.len()
1219 }
1220
1221 fn new(schedule: &ScheduledGraph, engines: &EngineSet<'_, '_>) -> Result<Self> {
1222 Self::build(schedule, |engine_id| engines.driver(engine_id))
1223 }
1224
1225 #[cfg(test)]
1230 pub(crate) fn new_for_test(
1231 schedule: &ScheduledGraph,
1232 drivers: &[(EngineId, Arc<dyn super::EventDomainDriver>)],
1233 ) -> Result<Self> {
1234 Self::build(schedule, |engine_id| {
1235 drivers
1236 .iter()
1237 .find(|(id, _)| id == engine_id)
1238 .map(|(_, driver)| Arc::clone(driver))
1239 .ok_or_else(|| {
1240 Error::runtime_state(
1241 "ScheduledEventDomains::new_for_test",
1242 ErrorPhase::Execution,
1243 format!("no test driver for engine {:?}", engine_id.as_str()),
1244 )
1245 })
1246 })
1247 }
1248
1249 fn build(
1250 schedule: &ScheduledGraph,
1251 driver_for: impl Fn(&EngineId) -> Result<Arc<dyn super::EventDomainDriver>>,
1252 ) -> Result<Self> {
1253 schedule.preflight().map_err(|source| {
1254 Error::runtime_state_source("Runtime::run", ErrorPhase::Execution, source)
1255 })?;
1256 if schedule.coalescing_eligible() {
1261 let domain = schedule.root_location().event_domain_id();
1262 let engine_id = schedule.root_location().engine_id().clone();
1263 let mut run = RuntimeOwnedEventDomainRun::new(
1264 domain,
1265 engine_id.clone(),
1266 driver_for(&engine_id)?.begin_run(domain)?,
1267 );
1268 let actual = run.domain(EventDomainOperation::BeginRun)?;
1269 if actual != domain {
1270 return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1271 operation: EventDomainOperation::BeginRun,
1272 node_index: None,
1273 expected: domain,
1274 actual,
1275 }));
1276 }
1277 let coalesced = run.enable_single_domain_coalescing();
1278 return Ok(Self {
1279 runs: vec![run],
1280 completions: HashMap::new(),
1281 coalesced,
1282 });
1283 }
1284 fn add_domain(
1285 drivers: &mut Vec<(super::EventDomainId, EngineId)>,
1286 seen_domains: &mut HashSet<super::EventDomainId>,
1287 domain: super::EventDomainId,
1288 engine_id: &EngineId,
1289 ) {
1290 if seen_domains.insert(domain) {
1291 drivers.push((domain, engine_id.clone()));
1292 }
1293 }
1294 let mut drivers = Vec::new();
1295 let mut seen_domains = HashSet::new();
1296 add_domain(
1302 &mut drivers,
1303 &mut seen_domains,
1304 schedule.root_location().event_domain_id(),
1305 schedule.root_location().engine_id(),
1306 );
1307 for location in schedule.input_locations() {
1308 add_domain(
1309 &mut drivers,
1310 &mut seen_domains,
1311 location.event_domain_id(),
1312 location.engine_id(),
1313 );
1314 }
1315 for location in schedule.operation_locations() {
1316 add_domain(
1317 &mut drivers,
1318 &mut seen_domains,
1319 location.event_domain_id(),
1320 location.engine_id(),
1321 );
1322 }
1323 for node in schedule.nodes() {
1324 add_domain(
1325 &mut drivers,
1326 &mut seen_domains,
1327 node.completion().domain(),
1328 node.event_domain_witness().engine_id(),
1329 );
1330 }
1331 let mut runs = Vec::with_capacity(drivers.len());
1332 for (domain, engine_id) in drivers {
1333 let driver = driver_for(&engine_id)?;
1334 let run = RuntimeOwnedEventDomainRun::new(domain, engine_id, driver.begin_run(domain)?);
1335 let actual = run.domain(EventDomainOperation::BeginRun)?;
1336 if actual != domain {
1337 return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1338 operation: EventDomainOperation::BeginRun,
1339 node_index: None,
1340 expected: domain,
1341 actual,
1342 }));
1343 }
1344 runs.push(run);
1345 }
1346 Ok(Self {
1347 runs,
1348 completions: HashMap::new(),
1349 coalesced: false,
1350 })
1351 }
1352
1353 #[cfg(test)]
1354 pub(crate) fn for_test(
1355 drivers: Vec<(super::EventDomainId, Arc<dyn super::EventDomainDriver>)>,
1356 ) -> Result<Self> {
1357 let mut runs = Vec::with_capacity(drivers.len());
1358 for (domain, driver) in drivers {
1359 let engine_id =
1360 EngineId::new("tenferro-test.event-domain-engine").expect("test engine id");
1361 let run = RuntimeOwnedEventDomainRun::new(domain, engine_id, driver.begin_run(domain)?);
1362 let actual = run.domain(EventDomainOperation::BeginRun)?;
1363 if actual != domain {
1364 return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1365 operation: EventDomainOperation::BeginRun,
1366 node_index: None,
1367 expected: domain,
1368 actual,
1369 }));
1370 }
1371 runs.push(run);
1372 }
1373 Ok(Self {
1374 runs,
1375 completions: HashMap::new(),
1376 coalesced: false,
1377 })
1378 }
1379
1380 pub(crate) fn enqueue(
1381 &mut self,
1382 node_index: usize,
1383 node: &ScheduledNode,
1384 launch: &mut dyn FnMut() -> Result<()>,
1385 ) -> Result<()> {
1386 let completion = node.completion();
1387 if self.coalesced {
1388 debug_assert_eq!(self.runs.len(), 1);
1393 self.runs[0].enqueue(&[], launch)?;
1394 return Ok(());
1395 }
1396 let destination = completion.domain();
1397 let run_index = self.run_index(completion.domain())?;
1398 let actual_preflight_domain = self.runs[run_index].domain(EventDomainOperation::Enqueue)?;
1399 if actual_preflight_domain != destination {
1400 return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1401 operation: EventDomainOperation::Enqueue,
1402 node_index: Some(node_index),
1403 expected: destination,
1404 actual: actual_preflight_domain,
1405 }));
1406 }
1407 let dependencies = self.classify_dependencies(node_index, node, destination)?;
1408 let actual_run_domain = self.runs[run_index].domain(EventDomainOperation::Enqueue)?;
1409 if actual_run_domain != destination {
1410 return Err(event_domain_error(EventDomainError::RunDomainMismatch {
1411 operation: EventDomainOperation::Enqueue,
1412 node_index: Some(node_index),
1413 expected: destination,
1414 actual: actual_run_domain,
1415 }));
1416 }
1417 let completion_event = self.runs[run_index].enqueue(&dependencies, launch)?;
1418 let actual = completion_event.origin();
1419 if actual != completion.domain() {
1420 return Err(event_domain_error(
1421 EventDomainError::CompletionTokenDomainMismatch {
1422 operation: EventDomainOperation::ValidateCompletion,
1423 node_index: Some(node_index),
1424 expected: completion.domain(),
1425 actual,
1426 },
1427 ));
1428 }
1429 self.completions.insert(
1430 EventDependency::from_completion(completion),
1431 completion_event,
1432 );
1433 Ok(())
1434 }
1435
1436 fn classify_dependencies(
1437 &self,
1438 node_index: usize,
1439 node: &ScheduledNode,
1440 destination: super::EventDomainId,
1441 ) -> Result<Vec<Arc<dyn EventToken>>> {
1442 let mut admitted = Vec::with_capacity(node.dependencies().len());
1449 for dependency in node.dependencies() {
1450 let dependency_completion =
1451 self.completions.get(dependency).cloned().ok_or_else(|| {
1452 Error::runtime_state_source(
1453 "Runtime::run",
1454 ErrorPhase::Execution,
1455 MissingScheduledDependencyCompletionError {
1456 dependency: *dependency,
1457 node_index,
1458 },
1459 )
1460 })?;
1461 let actual = dependency_completion.origin();
1462 if actual != dependency.domain() {
1463 return Err(event_domain_error(
1464 EventDomainError::DependencyDomainMismatch {
1465 operation: match node {
1466 ScheduledNode::Transfer(_) => EventDomainOperation::TransferBridge,
1467 ScheduledNode::Operation(_)
1468 | ScheduledNode::Collective(_)
1469 | ScheduledNode::Barrier(_) => EventDomainOperation::Enqueue,
1470 },
1471 node_index: Some(node_index),
1472 expected: dependency.domain(),
1473 actual,
1474 },
1475 ));
1476 }
1477 match node {
1478 ScheduledNode::Transfer(transfer) => {
1479 let source = transfer.source_event_domain();
1480 if actual == destination {
1481 admitted.push(dependency_completion);
1482 } else if actual == source {
1483 dependency_completion.wait().map_err(|source_error| {
1484 event_domain_error(EventDomainError::DependencyWaitFailed {
1485 operation: EventDomainOperation::TransferBridge,
1486 node_index: Some(node_index),
1487 expected: destination,
1488 actual,
1489 source: Box::new(source_error),
1490 })
1491 })?;
1492 } else {
1493 return Err(event_domain_error(
1494 EventDomainError::DependencyDomainMismatch {
1495 operation: EventDomainOperation::TransferBridge,
1496 node_index: Some(node_index),
1497 expected: source,
1498 actual,
1499 },
1500 ));
1501 }
1502 }
1503 ScheduledNode::Operation(_)
1504 | ScheduledNode::Collective(_)
1505 | ScheduledNode::Barrier(_) => {
1506 if actual != destination {
1507 return Err(event_domain_error(
1508 EventDomainError::DependencyDomainMismatch {
1509 operation: EventDomainOperation::Enqueue,
1510 node_index: Some(node_index),
1511 expected: destination,
1512 actual,
1513 },
1514 ));
1515 }
1516 admitted.push(dependency_completion);
1517 }
1518 }
1519 }
1520 Ok(admitted)
1521 }
1522
1523 fn run_index(&self, domain: super::EventDomainId) -> Result<usize> {
1524 if let Some(index) = self
1525 .runs
1526 .iter()
1527 .position(|run| run.requested_domain() == domain)
1528 {
1529 return Ok(index);
1530 }
1531 Err(missing_event_domain_driver(domain))
1532 }
1533
1534 fn into_pending_runs(self) -> impl Iterator<Item = (EngineId, Box<dyn EventDomainRun>)> {
1536 self.runs
1537 .into_iter()
1538 .filter_map(RuntimeOwnedEventDomainRun::into_pending)
1539 }
1540
1541 pub(crate) fn drain(&mut self) -> Result<()> {
1542 let mut failures = Vec::new();
1543 for run in &mut self.runs {
1544 if let Err(error) = run.drain() {
1545 failures.push(error);
1546 }
1547 }
1548 let mut failures = failures.into_iter();
1549 let Some(mut error) = failures.next() else {
1550 return Ok(());
1551 };
1552 for failure in failures {
1553 error = Error::with_suppressed(error, failure);
1554 }
1555 Err(error)
1556 }
1557}
1558
1559#[derive(Debug)]
1560enum RuntimeOwnedEventDomainRunState {
1561 Pending(Box<dyn EventDomainRun>),
1562 Retired,
1563 Failed,
1564}
1565
1566#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1567enum EventDomainRunTerminalState {
1568 Retired,
1569 Failed,
1570}
1571
1572impl fmt::Display for EventDomainRunTerminalState {
1573 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1574 match self {
1575 Self::Retired => formatter.write_str("retired"),
1576 Self::Failed => formatter.write_str("failed"),
1577 }
1578 }
1579}
1580
1581#[derive(Debug)]
1582struct RuntimeOwnedEventDomainRun {
1583 requested_domain: super::EventDomainId,
1584 engine_id: EngineId,
1587 state: RuntimeOwnedEventDomainRunState,
1588}
1589
1590impl RuntimeOwnedEventDomainRun {
1591 fn new(
1592 requested_domain: super::EventDomainId,
1593 engine_id: EngineId,
1594 inner: Box<dyn EventDomainRun>,
1595 ) -> Self {
1596 Self {
1597 requested_domain,
1598 engine_id,
1599 state: RuntimeOwnedEventDomainRunState::Pending(inner),
1600 }
1601 }
1602
1603 fn into_pending(mut self) -> Option<(EngineId, Box<dyn EventDomainRun>)> {
1605 match std::mem::replace(&mut self.state, RuntimeOwnedEventDomainRunState::Retired) {
1606 RuntimeOwnedEventDomainRunState::Pending(run) => Some((self.engine_id.clone(), run)),
1607 RuntimeOwnedEventDomainRunState::Retired | RuntimeOwnedEventDomainRunState::Failed => {
1608 None
1609 }
1610 }
1611 }
1612
1613 fn requested_domain(&self) -> super::EventDomainId {
1614 self.requested_domain
1615 }
1616
1617 fn enable_single_domain_coalescing(&mut self) -> bool {
1622 match &mut self.state {
1623 RuntimeOwnedEventDomainRunState::Pending(run) => run.enable_single_domain_coalescing(),
1624 RuntimeOwnedEventDomainRunState::Retired | RuntimeOwnedEventDomainRunState::Failed => {
1625 false
1626 }
1627 }
1628 }
1629
1630 fn domain(&self, operation: EventDomainOperation) -> Result<super::EventDomainId> {
1631 match &self.state {
1632 RuntimeOwnedEventDomainRunState::Pending(run) => Ok(run.domain()),
1633 RuntimeOwnedEventDomainRunState::Retired => Err(event_domain_run_state_error(
1634 operation,
1635 self.requested_domain,
1636 EventDomainRunTerminalState::Retired,
1637 )),
1638 RuntimeOwnedEventDomainRunState::Failed => Err(event_domain_run_state_error(
1639 operation,
1640 self.requested_domain,
1641 EventDomainRunTerminalState::Failed,
1642 )),
1643 }
1644 }
1645
1646 fn enqueue(
1647 &mut self,
1648 dependencies: &[Arc<dyn EventToken>],
1649 launch: &mut dyn FnMut() -> Result<()>,
1650 ) -> Result<Arc<dyn EventToken>> {
1651 match &mut self.state {
1652 RuntimeOwnedEventDomainRunState::Pending(run) => run.enqueue(dependencies, launch),
1653 RuntimeOwnedEventDomainRunState::Retired => Err(event_domain_run_state_error(
1654 EventDomainOperation::Enqueue,
1655 self.requested_domain,
1656 EventDomainRunTerminalState::Retired,
1657 )),
1658 RuntimeOwnedEventDomainRunState::Failed => Err(event_domain_run_state_error(
1659 EventDomainOperation::Enqueue,
1660 self.requested_domain,
1661 EventDomainRunTerminalState::Failed,
1662 )),
1663 }
1664 }
1665
1666 fn drain(&mut self) -> Result<()> {
1667 let run = match std::mem::replace(&mut self.state, RuntimeOwnedEventDomainRunState::Failed)
1668 {
1669 RuntimeOwnedEventDomainRunState::Pending(run) => run,
1670 RuntimeOwnedEventDomainRunState::Retired => {
1671 self.state = RuntimeOwnedEventDomainRunState::Retired;
1672 return Err(event_domain_run_state_error(
1673 EventDomainOperation::Drain,
1674 self.requested_domain,
1675 EventDomainRunTerminalState::Retired,
1676 ));
1677 }
1678 RuntimeOwnedEventDomainRunState::Failed => {
1679 self.state = RuntimeOwnedEventDomainRunState::Failed;
1680 return Err(event_domain_run_state_error(
1681 EventDomainOperation::Drain,
1682 self.requested_domain,
1683 EventDomainRunTerminalState::Failed,
1684 ));
1685 }
1686 };
1687 let domain = self.requested_domain;
1688 let mut run = run;
1689 let drain_result = std::panic::catch_unwind(AssertUnwindSafe(|| run.drain()));
1690 drop_event_domain_run(run);
1691 match drain_result {
1692 Ok(Ok(())) => {
1693 self.state = RuntimeOwnedEventDomainRunState::Retired;
1694 Ok(())
1695 }
1696 Ok(Err(error)) => {
1697 self.state = RuntimeOwnedEventDomainRunState::Failed;
1698 Err(error)
1699 }
1700 Err(payload) => {
1701 self.state = RuntimeOwnedEventDomainRunState::Failed;
1702 Err(event_domain_error(EventDomainError::DrainPanicked {
1703 operation: EventDomainOperation::Drain,
1704 domain,
1705 message: safe_event_domain_panic_message(payload),
1706 }))
1707 }
1708 }
1709 }
1710}
1711
1712#[derive(Debug, thiserror::Error)]
1715#[error("{operation} used event-domain run {domain:?} after it reached terminal state {state}")]
1716pub(crate) struct EventDomainRunLifecycleError {
1717 operation: EventDomainOperation,
1718 domain: super::EventDomainId,
1719 state: EventDomainRunTerminalState,
1720}
1721
1722fn event_domain_run_state_error(
1723 operation: EventDomainOperation,
1724 domain: super::EventDomainId,
1725 state: EventDomainRunTerminalState,
1726) -> Error {
1727 Error::runtime_state_source(
1728 "Runtime::run",
1729 ErrorPhase::Execution,
1730 EventDomainRunLifecycleError {
1731 operation,
1732 domain,
1733 state,
1734 },
1735 )
1736}
1737
1738fn drop_event_domain_run(run: Box<dyn EventDomainRun>) {
1739 if std::panic::catch_unwind(AssertUnwindSafe(|| drop(run))).is_err() {
1740 }
1742}
1743
1744impl Drop for RuntimeOwnedEventDomainRun {
1745 fn drop(&mut self) {
1746 let state = std::mem::replace(&mut self.state, RuntimeOwnedEventDomainRunState::Failed);
1747 if let RuntimeOwnedEventDomainRunState::Pending(run) = state {
1748 drop_event_domain_run(run);
1749 }
1750 }
1751}
1752
1753fn missing_event_domain_driver(domain: super::EventDomainId) -> Error {
1754 Error::from(EventDomainError::MissingDriver { domain })
1755}
1756
1757fn event_domain_error(source: EventDomainError) -> Error {
1758 Error::from(source)
1759}
1760
1761fn safe_event_domain_panic_message(payload: Box<dyn std::any::Any + Send + 'static>) -> String {
1762 match payload.downcast::<&'static str>() {
1763 Ok(message) => (*message).to_owned(),
1764 Err(payload) => match payload.downcast::<String>() {
1765 Ok(message) => *message,
1766 Err(_) => "non-string panic payload".to_owned(),
1767 },
1768 }
1769}
1770
1771fn scheduled_execution_cleanup_error(primary: Error, cleanup: Error) -> Error {
1772 Error::with_suppressed(primary, cleanup)
1773}
1774
1775fn instruction_execution<'a>(
1776 schedule: &'a ScheduledGraph,
1777 instruction: &ExecInstruction,
1778) -> Result<InstructionExecution<'a>> {
1779 let Some(operation_index) = instruction.semantic_operation_index else {
1780 let location = schedule.root_location();
1781 return Ok(InstructionExecution { location });
1782 };
1783 let location = schedule
1784 .operation_locations()
1785 .get(operation_index)
1786 .ok_or_else(|| {
1787 Error::runtime_state(
1788 "Runtime::run",
1789 ErrorPhase::Execution,
1790 format!(
1791 "instruction references semantic operation {operation_index}, but prepared schedule has {} operations",
1792 schedule.operation_locations().len()
1793 ),
1794 )
1795 })?;
1796 Ok(InstructionExecution { location })
1797}
1798
1799struct InstructionExecution<'a> {
1800 location: &'a ExecutionLocation,
1801}
1802
1803impl<'a> InstructionExecution<'a> {
1804 fn location(&self) -> &'a ExecutionLocation {
1805 self.location
1806 }
1807}
1808
1809pub(crate) struct LocatedExecSlot<'input> {
1810 pub(crate) location: ExecutionLocation,
1811 pub(crate) value: ExecSlot<'input>,
1812}
1813
1814fn stage_instruction_inputs<'input>(
1815 instruction: &ExecInstruction,
1816 location: &ExecutionLocation,
1817 located: &mut [Vec<LocatedExecSlot<'input>>],
1818 staged: &mut [Option<ExecSlot<'input>>],
1819) -> Result<()> {
1820 for &slot in &instruction.input_slots {
1821 stage_slot_input(slot, location, located, staged)?;
1822 }
1823 Ok(())
1824}
1825
1826fn stage_slot_input<'input>(
1828 slot: usize,
1829 location: &ExecutionLocation,
1830 located: &mut [Vec<LocatedExecSlot<'input>>],
1831 staged: &mut [Option<ExecSlot<'input>>],
1832) -> Result<()> {
1833 if staged
1834 .get(slot)
1835 .ok_or(tenferro_tensor::Error::MissingValue { slot })?
1836 .is_some()
1837 {
1838 return Ok(());
1839 }
1840 let values = located
1841 .get_mut(slot)
1842 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1843 let value_index = values
1844 .iter()
1845 .position(|value| &value.location == location)
1846 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1847 staged[slot] = Some(values.swap_remove(value_index).value);
1848 Ok(())
1849}
1850
1851fn validate_region_outputs(
1854 region: &super::region::ElementwiseRegion,
1855 location: &ExecutionLocation,
1856 staged: &[Option<ExecSlot<'_>>],
1857) -> Result<()> {
1858 for &output_slot in ®ion.output_slots {
1859 let output = staged
1860 .get(output_slot)
1861 .and_then(Option::as_ref)
1862 .ok_or(tenferro_tensor::Error::MissingValue { slot: output_slot })?;
1863 let output = output.as_read();
1864 if !location
1865 .witness()
1866 .owns_resident_tensor(&output, location.storage_class())
1867 {
1868 return Err(Error::runtime_state_source(
1869 "Runtime::run",
1870 ErrorPhase::Execution,
1871 super::EngineExecutionContractError::OutputResidencyMismatch {
1872 instruction_index: region.instruction_range.start,
1873 output_slot,
1874 engine_id: location.engine_id().clone(),
1875 storage_class: location.storage_class().clone(),
1876 backend_family: output.backend_family(),
1877 allocation_domain: output.allocation_domain(),
1878 },
1879 ));
1880 }
1881 }
1882 Ok(())
1883}
1884
1885fn retain_region_results<'input>(
1888 region: &super::region::ElementwiseRegion,
1889 location: &ExecutionLocation,
1890 located: &mut [Vec<LocatedExecSlot<'input>>],
1891 staged: &mut [Option<ExecSlot<'input>>],
1892) -> Result<()> {
1893 for (&slot, &last_use) in region.input_slots.iter().zip(region.input_last_use.iter()) {
1894 if last_use {
1895 located[slot].clear();
1896 staged[slot].take();
1897 }
1898 }
1899 for &slot in ®ion.output_slots {
1900 let value = staged
1901 .get_mut(slot)
1902 .and_then(Option::take)
1903 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1904 let values = located
1905 .get_mut(slot)
1906 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1907 values.clear();
1908 values.push(LocatedExecSlot {
1909 location: location.clone(),
1910 value,
1911 });
1912 }
1913 Ok(())
1914}
1915
1916fn validate_instruction_outputs(
1917 instruction_index: usize,
1918 instruction: &ExecInstruction,
1919 location: &ExecutionLocation,
1920 staged: &[Option<ExecSlot<'_>>],
1921) -> Result<()> {
1922 for &output_slot in &instruction.output_slots {
1923 let output = staged
1924 .get(output_slot)
1925 .and_then(Option::as_ref)
1926 .ok_or(tenferro_tensor::Error::MissingValue { slot: output_slot })?;
1927 let output = output.as_read();
1928 if !location
1929 .witness()
1930 .owns_resident_tensor(&output, location.storage_class())
1931 {
1932 return Err(Error::runtime_state_source(
1933 "Runtime::run",
1934 ErrorPhase::Execution,
1935 super::EngineExecutionContractError::OutputResidencyMismatch {
1936 instruction_index,
1937 output_slot,
1938 engine_id: location.engine_id().clone(),
1939 storage_class: location.storage_class().clone(),
1940 backend_family: output.backend_family(),
1941 allocation_domain: output.allocation_domain(),
1942 },
1943 ));
1944 }
1945 }
1946 Ok(())
1947}
1948
1949pub(crate) fn retain_instruction_results<'input>(
1950 instruction: &ExecInstruction,
1951 location: &ExecutionLocation,
1952 located: &mut [Vec<LocatedExecSlot<'input>>],
1953 staged: &mut [Option<ExecSlot<'input>>],
1954) -> Result<()> {
1955 for &slot in &instruction.input_slots {
1956 let is_output = instruction.output_slots.contains(&slot);
1957 let is_last_use = instruction
1958 .input_slots
1959 .iter()
1960 .enumerate()
1961 .any(|(index, &candidate)| {
1962 candidate == slot && instruction.last_use.get(index).copied().unwrap_or(false)
1963 });
1964 if is_last_use {
1965 located[slot].clear();
1966 if !is_output {
1967 staged[slot].take();
1968 }
1969 } else if !is_output && let Some(value) = staged[slot].take() {
1970 located[slot].push(LocatedExecSlot {
1971 location: location.clone(),
1972 value,
1973 });
1974 }
1975 }
1976
1977 for &slot in &instruction.output_slots {
1978 let value = staged
1979 .get_mut(slot)
1980 .and_then(Option::take)
1981 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1982 let values = located
1983 .get_mut(slot)
1984 .ok_or(tenferro_tensor::Error::MissingValue { slot })?;
1985 values.clear();
1986 values.push(LocatedExecSlot {
1987 location: location.clone(),
1988 value,
1989 });
1990 }
1991 Ok(())
1992}
1993
1994fn validate_runtime_input_ingress(
1995 location: &ExecutionLocation,
1996 input: &TensorRead<'_>,
1997 slot: usize,
1998) -> Result<()> {
1999 let accepted = location
2000 .witness()
2001 .accepts_runtime_input(input, location.storage_class());
2002 if accepted {
2003 return Ok(());
2004 }
2005 Err(Error::runtime_state_source(
2006 "Runtime::run",
2007 ErrorPhase::Execution,
2008 super::InputIngressContractError::ResidencyMismatch {
2009 input_slot: slot,
2010 ingress_engine_id: location.engine_id().clone(),
2011 ingress_storage_class: location.storage_class().clone(),
2012 placement: input.placement().clone(),
2013 backend_family: input.backend_family(),
2014 allocation_domain: input.allocation_domain(),
2015 },
2016 ))
2017}
2018
2019fn execute_scheduled_transfer<'input>(
2020 transfer: &ScheduledTransfer,
2021 located: &mut [Vec<LocatedExecSlot<'input>>],
2022) -> Result<()> {
2023 let source = transfer.source_location();
2024 let destination = transfer.destination_location();
2025 let provider = transfer.provider();
2026 let values =
2027 located
2028 .get_mut(transfer.value_slot())
2029 .ok_or(tenferro_tensor::Error::MissingValue {
2030 slot: transfer.value_slot(),
2031 })?;
2032 if values.iter().any(|value| &value.location == destination) {
2033 return Err(Error::runtime_state_source(
2034 "Runtime::run",
2035 ErrorPhase::Execution,
2036 DuplicateTransferDestinationError {
2037 value_slot: transfer.value_slot(),
2038 destination: destination.clone(),
2039 },
2040 ));
2041 }
2042 let transferred = {
2043 let source_value = values
2044 .iter()
2045 .find(|value| &value.location == source)
2046 .ok_or(tenferro_tensor::Error::MissingValue {
2047 slot: transfer.value_slot(),
2048 })?;
2049 let source_read = source_value.value.as_read();
2050 let expected_dtype = source_read.dtype();
2051 let expected_shape = source_read.shape().to_vec();
2052 let transferred =
2053 provider.transfer_blocking(TransferRequest::new(source, destination, source_read))?;
2054 validate_transfer_output(destination, expected_dtype, &expected_shape, &transferred)?;
2055 transferred
2056 };
2057 values.push(LocatedExecSlot {
2058 location: destination.clone(),
2059 value: ExecSlot::Owned(transferred),
2060 });
2061 Ok(())
2062}
2063
2064fn validate_transfer_output(
2065 destination: &ExecutionLocation,
2066 expected_dtype: tenferro_tensor::DType,
2067 expected_shape: &[usize],
2068 output: &Tensor,
2069) -> Result<()> {
2070 let expected_elements = checked_transfer_element_count(expected_shape).map_err(|source| {
2071 Error::runtime_state_source(
2072 "Runtime::run",
2073 ErrorPhase::Execution,
2074 TransferError::ProviderContract { source },
2075 )
2076 })?;
2077 let contract_error = if output.dtype() != expected_dtype {
2078 Some(TransferProviderContractError::DTypeMismatch {
2079 expected: expected_dtype,
2080 actual: output.dtype(),
2081 })
2082 } else if output.shape() != expected_shape {
2083 Some(TransferProviderContractError::ShapeMismatch {
2084 expected: expected_shape.to_vec(),
2085 actual: output.shape().to_vec(),
2086 })
2087 } else if tensor_buffer_len(output) != expected_elements {
2088 Some(TransferProviderContractError::InvalidBufferLength {
2089 expected: expected_elements,
2090 actual: tensor_buffer_len(output),
2091 })
2092 } else if !destination
2093 .witness()
2094 .accepts_input_placement(output.placement(), destination.storage_class())
2095 {
2096 Some(
2097 TransferProviderContractError::DestinationPlacementMismatch {
2098 destination_engine_id: destination.engine_id().clone(),
2099 destination_storage_class: destination.storage_class().clone(),
2100 actual: output.placement().clone(),
2101 },
2102 )
2103 } else if !destination.witness().owns_resident_tensor(
2104 &TensorRead::from_tensor(output),
2105 destination.storage_class(),
2106 ) {
2107 Some(
2108 TransferProviderContractError::DestinationResidencyMismatch {
2109 destination_engine_id: destination.engine_id().clone(),
2110 destination_storage_class: destination.storage_class().clone(),
2111 actual_backend_family: TensorRead::from_tensor(output).backend_family(),
2112 actual_allocation_domain: TensorRead::from_tensor(output).allocation_domain(),
2113 },
2114 )
2115 } else {
2116 None
2117 };
2118 match contract_error {
2119 None => Ok(()),
2120 Some(source) => Err(Error::runtime_state_source(
2121 "Runtime::run",
2122 ErrorPhase::Execution,
2123 TransferError::ProviderContract { source },
2124 )),
2125 }
2126}
2127
2128fn checked_transfer_element_count(
2129 shape: &[usize],
2130) -> std::result::Result<usize, TransferProviderContractError> {
2131 tenferro_tensor::validate::checked_shape_product("Runtime::run", "transfer source shape", shape)
2132 .map_err(|source| TransferProviderContractError::LogicalElementCount { source })
2133}
2134
2135fn tensor_buffer_len(tensor: &Tensor) -> usize {
2136 match tensor.dtype() {
2137 DType::F32 => tensor.as_typed::<f32>().map_or(0, |t| t.buffer().len()),
2138 DType::F64 => tensor.as_typed::<f64>().map_or(0, |t| t.buffer().len()),
2139 DType::I32 => tensor.as_typed::<i32>().map_or(0, |t| t.buffer().len()),
2140 DType::I64 => tensor.as_typed::<i64>().map_or(0, |t| t.buffer().len()),
2141 DType::Bool => tensor.as_typed::<bool>().map_or(0, |t| t.buffer().len()),
2142 DType::C32 => tensor
2143 .as_typed::<Complex32>()
2144 .map_or(0, |t| t.buffer().len()),
2145 DType::C64 => tensor
2146 .as_typed::<Complex64>()
2147 .map_or(0, |t| t.buffer().len()),
2148 DType::External(_) => 0,
2152 }
2153}
2154
2155#[cfg(test)]
2156mod transfer_validation_tests {
2157 use std::error::Error as _;
2158
2159 use super::checked_transfer_element_count;
2160 use crate::TransferProviderContractError;
2161
2162 #[test]
2163 fn transfer_element_count_overflow_is_typed_and_preserves_source() {
2164 let error = checked_transfer_element_count(&[usize::MAX, 2]).unwrap_err();
2165
2166 assert!(matches!(
2167 error,
2168 TransferProviderContractError::LogicalElementCount { .. }
2169 ));
2170 assert!(error.source().is_some());
2171 }
2172}
2173
2174fn input_signature_reads(inputs: &[TensorRead<'_>]) -> Result<InputSignature> {
2175 InputSignature::from_reads(inputs).map_err(|source| prepare_error(Arc::new(source)))
2176}
2177
2178#[derive(Clone, Copy, Debug, Eq, PartialEq)]
2186enum BoundInputCheck {
2187 Validate,
2188 AlreadyValidatedAtPrepare,
2189}
2190
2191fn resolve_input_reads<'a>(
2192 program: &'a CompiledGraph,
2193 inputs: &[TensorRead<'a>],
2194 bound_inputs: BoundInputCheck,
2195) -> Result<RuntimeInputReads<'a>> {
2196 let expected = program.program().inputs().len();
2197 if inputs.len() > expected {
2198 return Err(Error::GraphInputCountMismatch {
2199 expected,
2200 actual: inputs.len(),
2201 });
2202 }
2203 if inputs.is_empty() {
2204 let resolved = semantic_default_inputs(program)?;
2205 if matches!(bound_inputs, BoundInputCheck::AlreadyValidatedAtPrepare) {
2206 return Ok(resolved);
2207 }
2208 validate_ordered_input_metadata_reads(program, &resolved)?;
2209 return Ok(resolved);
2210 }
2211 let resolved = if inputs.len() == expected {
2212 inputs.to_vec()
2213 } else {
2214 let mut explicit = inputs.iter();
2215 let mut resolved = RuntimeInputReads::new();
2216 for value in program.program().inputs() {
2217 if let Some(retained) = program.bindings().tensor_ref_for_input(*value) {
2218 resolved.push(retained.tensor_read()?);
2219 } else {
2220 let read = explicit.next().ok_or_else(|| {
2221 Error::invalid_argument(
2222 "Runtime::run",
2223 ErrorPhase::Execution,
2224 "inputs",
2225 format!(
2226 "expected {expected} inputs, or one explicit input for each unbound placeholder"
2227 ),
2228 )
2229 })?;
2230 resolved.push(read.clone());
2231 }
2232 }
2233 if explicit.next().is_some() {
2234 return Err(Error::invalid_argument(
2235 "Runtime::run",
2236 ErrorPhase::Execution,
2237 "inputs",
2238 format!("expected {expected} inputs, received {}", inputs.len()),
2239 ));
2240 }
2241 resolved
2242 };
2243 validate_ordered_input_metadata_reads(program, &resolved)?;
2244 Ok(resolved)
2245}
2246
2247fn semantic_default_inputs(program: &CompiledGraph) -> Result<RuntimeInputReads<'_>> {
2248 program
2249 .program()
2250 .inputs()
2251 .iter()
2252 .enumerate()
2253 .map(|(input_index, value)| {
2254 let retained = program
2255 .bindings()
2256 .tensor_ref_for_input(*value)
2257 .ok_or_else(|| Error::UnboundPlaceholder {
2258 input_key: format!("semantic input {input_index}"),
2259 })?;
2260 retained.tensor_read()
2261 })
2262 .collect()
2263}
2264
2265fn validate_ordered_input_metadata_reads(
2266 program: &CompiledGraph,
2267 inputs: &[TensorRead<'_>],
2268) -> Result<()> {
2269 let actuals = inputs
2270 .iter()
2271 .map(|input| (input.dtype(), input.shape().to_vec()))
2272 .collect::<Vec<_>>();
2273 validate_ordered_input_metadata_values(program, &actuals, "Runtime::submit")
2274}
2275
2276fn validate_ordered_input_metadata_values(
2277 program: &CompiledGraph,
2278 actuals: &[(tenferro_tensor::DType, Vec<usize>)],
2279 caller: &'static str,
2280) -> Result<()> {
2281 let expected = program.input_count();
2282 if actuals.len() != expected {
2283 return Err(Error::GraphInputCountMismatch {
2284 expected,
2285 actual: actuals.len(),
2286 });
2287 }
2288 let input_shapes: RuntimeInputShapes<'_> =
2289 actuals.iter().map(|(_, shape)| shape.as_slice()).collect();
2290 for (input_value, (actual_dtype, actual_shape)) in
2291 program.program().inputs().iter().zip(actuals)
2292 {
2293 let metadata = program
2294 .program()
2295 .value_metadata(*input_value)
2296 .map_err(|source| Error::runtime_state_source(caller, ErrorPhase::Execution, source))?;
2297 if metadata.dtype() != *actual_dtype {
2298 return Err(Error::PlaceholderDtypeMismatch {
2299 expected: metadata.dtype(),
2300 actual: *actual_dtype,
2301 });
2302 }
2303 if metadata.shape().len() != actual_shape.len() {
2304 return Err(Error::PlaceholderRankMismatch {
2305 expected: metadata.shape().len(),
2306 actual: actual_shape.len(),
2307 });
2308 }
2309 let mut expected_shape: RuntimeShapeScratch = actual_shape.iter().copied().collect();
2310 let mut exact_mismatch = false;
2311 for (axis, (extent, actual_size)) in metadata.shape().iter().zip(actual_shape).enumerate() {
2312 match extent {
2313 ShapeExtent::Exact(expression) => {
2314 let expected = expression.eval(&input_shapes).map_err(|source| {
2315 Error::runtime_state_source(caller, ErrorPhase::Execution, source)
2316 })?;
2317 expected_shape[axis] = expected;
2318 exact_mismatch |= expected != *actual_size;
2319 }
2320 ShapeExtent::UpperBound(expression) => {
2321 let bound = expression.eval(&input_shapes).map_err(|source| {
2322 Error::runtime_state_source(caller, ErrorPhase::Execution, source)
2323 })?;
2324 if *actual_size > bound {
2325 return Err(Error::PlaceholderShapeBoundExceeded {
2326 axis,
2327 bound,
2328 actual: *actual_size,
2329 });
2330 }
2331 }
2332 ShapeExtent::Unknown => {}
2333 }
2334 }
2335 if exact_mismatch {
2336 return Err(Error::PlaceholderShapeMismatch {
2337 expected: expected_shape.into_vec(),
2338 actual: actual_shape.clone(),
2339 });
2340 }
2341 }
2342 Ok(())
2343}
2344
2345fn validate_exec_input_count(program: &ExecProgram, actual: usize) -> Result<()> {
2346 let expected = program.input_slots.len();
2347 if actual != expected {
2348 return Err(Error::runtime_state(
2349 "Runtime::run",
2350 ErrorPhase::Execution,
2351 format!("expected {expected} inputs for execution program, got {actual}"),
2352 ));
2353 }
2354 Ok(())
2355}
2356
2357fn prepare_error(source: Arc<PrepareError>) -> Error {
2358 Error::runtime_state_source(
2359 "Runtime::run",
2360 ErrorPhase::Execution,
2361 SharedPrepareError(source),
2362 )
2363}
2364
2365#[derive(Clone, Debug)]
2366struct SharedPrepareError(Arc<PrepareError>);
2367
2368impl fmt::Display for SharedPrepareError {
2369 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
2370 write!(formatter, "{}", self.0)
2371 }
2372}
2373
2374impl StdError for SharedPrepareError {
2375 fn source(&self) -> Option<&(dyn StdError + 'static)> {
2376 Some(self.0.as_ref())
2377 }
2378}