Skip to main content

tenferro_mpi/
lib.rs

1//! Blocking point-to-point MPI transport for tenferro host tensors.
2//!
3//! This crate moves **one tensor value** between two MPI ranks. It borrows the
4//! caller's communicator and owns nothing else: it never calls `MPI_Init` or
5//! `MPI_Finalize`, never creates `MPI_COMM_WORLD`, never duplicates a communicator,
6//! and never transfers between host and device. Distributed scheduling stays with
7//! the caller (for example hataori); this is transport only.
8//!
9//! # What can be sent
10//!
11//! [`Transport`] covers the three values the adapter accepts, with dtype and shape
12//! preserved:
13//!
14//! - [`TypedTensor<T, R, Host>`](tenferro_tensor::TypedTensor), the statically
15//!   host-owned tensor;
16//! - a host-resident [`TypedTensor<T, R, Dynamic>`](tenferro_tensor::TypedTensor);
17//! - the dtype-erased [`Tensor`](tenferro_tensor::Tensor).
18//!
19//! A `Gpu`-representation tensor does not implement [`Transport`], so an accidental
20//! device transfer is a compile error. Payloads must be compact column-major host
21//! slices; the adapter never materializes a view.
22//!
23//! # Wire format
24//!
25//! Each call sends a header, waits for the receiver's ack, and only then sends the
26//! payload, so a receiver can refuse a tensor without ever receiving its payload
27//! and a sender always learns the outcome. Every frame uses **the caller's single
28//! tag**, which therefore belongs to that pair for the duration of the call. The
29//! format is versioned (v1) and is specified in `docs/design/mpi-transport.md` of
30//! the tenferro-rs repository; the pieces a caller can observe are:
31//!
32//! - supported dtypes are `bool`, `i32`, `i64`, `f32`, `f64`, `Complex32` and
33//!   `Complex64`, plus nothing else: a caller-owned external scalar needs an
34//!   explicit codec, because a process-local `TypeId` cannot be a wire identifier;
35//! - `bool` travels as one byte per element, 0 or 1, validated on receipt;
36//! - the payload is the compact column-major element representation in the host's
37//!   byte order, and the header carries a byte-order marker, so a peer with a
38//!   different byte order is rejected explicitly rather than mis-decoded;
39//! - one message carries at most [`MAX_WIRE_ELEMENTS`] elements, and the payload
40//!   also has to fit one MPI byte count; larger tensors are rejected, never
41//!   chunked or truncated.
42//!
43//! # Backend features
44//!
45//! This is a workspace member with `default = []`: a default build compiles the
46//! wire format, the error taxonomy and the [`MpiBackend`] trait but selects no MPI
47//! binding, so the root `cargo test --workspace` graph needs neither MPI headers
48//! nor libclang.
49//!
50//! Exactly one backend feature is selected when a caller wants transport. Both
51//! build the published [`rsmpi-rt`](https://crates.io/crates/rsmpi-rt) 0.8.2
52//! package, whose library target is still `mpi`:
53//!
54//! - `system` enables rsmpi-rt's bindgen `mpi-sys-backend` (crates.io
55//!   [`rsmpi-sys`](https://crates.io/crates/rsmpi-sys) 0.2.4) and therefore needs
56//!   system MPI headers and libclang at build time.
57//! - `runtime` enables its MPIABI `mpi-rt-sys-backend` (crates.io
58//!   [`mpi-rt-sys`](https://crates.io/crates/mpi-rt-sys) 0.1.0), which loads MPI at
59//!   run time and needs no headers or libclang.
60//!
61//! The two backends are mutually exclusive; rsmpi-rt itself rejects enabling both
62//! backend features at once. `engine` compiles the protocol with no binding at all.
63//!
64//! # Examples
65//!
66//! The transport entry points exist only with a backend, so the example below is
67//! compiled in the `system` and `runtime` lanes:
68//!
69//! ```
70//! # #[cfg(any(feature = "system", feature = "runtime"))]
71//! # fn example() -> Result<(), Box<dyn std::error::Error>> {
72//! use tenferro_mpi::mpi::traits::*;
73//! use tenferro_mpi::{recv, send, Error};
74//! use tenferro_tensor::{DynRank, Host, Tensor, TypedTensor};
75//!
76//! let universe = tenferro_mpi::mpi::initialize().unwrap();
77//! let world = universe.world();
78//!
79//! let typed =
80//!     TypedTensor::<f64, DynRank, Host>::from_host_vec_col_major(vec![2, 2], vec![1.0, 2.0, 3.0, 4.0])?;
81//! let erased = Tensor::from_typed(
82//!     TypedTensor::<f64, DynRank, Host>::from_host_vec_col_major(vec![2, 2], vec![1.0, 2.0, 3.0, 4.0])?
83//!         .into_dynamic(),
84//! );
85//!
86//! // A blocking send to the calling rank could never be received, so it is refused
87//! // before any message.
88//! assert!(matches!(send(world.this_process(), 5, &typed), Err(Error::SelfMessage)));
89//!
90//! // One rank sends while the other receives: a blocking send needs a posted
91//! // receive, so both ranks may not send at each other.
92//! if world.size() >= 2 {
93//!     if world.rank() == 0 {
94//!         send(world.process_at_rank(1), 5, &typed)?;
95//!         send(world.process_at_rank(1), 6, &erased)?;
96//!     } else if world.rank() == 1 {
97//!         let typed_back: TypedTensor<f64, DynRank, Host> = recv(world.process_at_rank(0), 5)?;
98//!         let erased_back: Tensor = recv(world.process_at_rank(0), 6)?;
99//!         assert_eq!(typed_back.get(&[1, 1])?, &4.0);
100//!         assert_eq!(erased_back.shape(), &[2, 2]);
101//!     }
102//! }
103//! # Ok(())
104//! # }
105//! # #[cfg(any(feature = "system", feature = "runtime"))]
106//! # fn main() { example().unwrap(); }
107//! # #[cfg(not(any(feature = "system", feature = "runtime")))]
108//! # fn main() {}
109//! ```
110
111#![deny(missing_docs)]
112// The default (no-backend) build is the wire/error shell: the codec helpers and
113// error codes are exercised only by the engine and its backend tests, so they are
114// intentionally unused there. Keep the root `-D warnings` lane green without
115// weakening the backend builds.
116#![cfg_attr(
117    not(any(feature = "system", feature = "runtime", feature = "engine")),
118    allow(dead_code)
119)]
120
121mod backend;
122#[cfg(any(feature = "system", feature = "runtime", feature = "engine"))]
123pub mod engine;
124mod error;
125mod wire;
126
127pub use backend::MpiBackend;
128#[cfg(feature = "runtime")]
129pub use backend::Rt;
130#[cfg(feature = "system")]
131pub use backend::System;
132#[cfg(any(feature = "system", feature = "runtime", feature = "engine"))]
133pub use engine::Transport;
134pub use error::{AbortReason, Error, RejectReason, Result};
135pub use wire::{Tag, MAX_WIRE_ELEMENTS, MAX_WIRE_RANK, PROTOCOL_VERSION};
136
137/// The MPI binding this crate was built against.
138///
139/// A caller must take its communicator, `Process` handles and `initialize` call from
140/// this re-export, so the adapter and the caller always share one `mpi` identity.
141#[cfg(any(feature = "system", feature = "runtime"))]
142pub use mpi;
143
144/// The backend selected by the one enabled binding feature.
145#[cfg(feature = "system")]
146type ActiveBackend = System;
147#[cfg(feature = "runtime")]
148type ActiveBackend = Rt;
149
150#[cfg(feature = "system")]
151const ACTIVE_BACKEND: ActiveBackend = System;
152#[cfg(feature = "runtime")]
153const ACTIVE_BACKEND: ActiveBackend = Rt;
154
155/// Send one tensor value to `to` on `tag`.
156///
157/// # Examples
158///
159/// ```
160/// use tenferro_mpi::mpi::traits::*;
161/// use tenferro_mpi::send;
162/// use tenferro_tensor::{DynRank, Host, TypedTensor};
163///
164/// let universe = tenferro_mpi::mpi::initialize().unwrap();
165/// let world = universe.world();
166/// let value = TypedTensor::<i64, DynRank, Host>::from_host_vec_col_major(vec![3], vec![1, 2, 3])?;
167/// assert!(matches!(send(world.this_process(), 9, &value), Err(tenferro_mpi::Error::SelfMessage)));
168/// if world.size() >= 2 && world.rank() == 0 {
169///     send(world.process_at_rank(1), 9, &value)?;
170/// }
171/// # Ok::<(), Box<dyn std::error::Error>>(())
172/// ```
173///
174/// # Errors
175///
176/// Returns [`Error::InvalidTag`] for a negative tag and [`Error::SelfMessage`] for a
177/// send to the calling rank; neither sends a message. Returns
178/// [`Error::NotHostResident`], [`Error::NonCompact`], [`Error::UnsupportedDType`],
179/// [`Error::RankTooLarge`] or [`Error::TooManyElements`] when the value cannot be
180/// described on the wire, and [`Error::PeerRejected`] when the receiver refuses it.
181#[cfg(any(feature = "system", feature = "runtime"))]
182pub fn send<X: Transport>(to: mpi::topology::Process<'_>, tag: Tag, value: &X) -> Result<()> {
183    engine::send(&ACTIVE_BACKEND, to, tag, value)
184}
185
186/// Receive one tensor value from `from` on `tag`.
187///
188/// # Examples
189///
190/// ```
191/// use tenferro_mpi::mpi::traits::*;
192/// use tenferro_mpi::recv;
193/// use tenferro_tensor::{DynRank, Host, TypedTensor};
194///
195/// let universe = tenferro_mpi::mpi::initialize().unwrap();
196/// let world = universe.world();
197/// if world.size() >= 2 {
198///     let value = TypedTensor::<i64, DynRank, Host>::from_host_vec_col_major(vec![2], vec![7, 8])?;
199///     if world.rank() == 0 {
200///         tenferro_mpi::send(world.process_at_rank(1), 10, &value)?;
201///     } else if world.rank() == 1 {
202///         let back: TypedTensor<i64, DynRank, Host> = recv(world.process_at_rank(0), 10)?;
203///         assert_eq!(back.as_slice(), &[7, 8]);
204///     }
205/// }
206/// # Ok::<(), Box<dyn std::error::Error>>(())
207/// ```
208///
209/// # Errors
210///
211/// Returns [`Error::InvalidTag`] for a negative tag, [`Error::DTypeMismatch`] or
212/// [`Error::RankMismatch`] when the incoming header does not match this receiver,
213/// [`Error::PeerAborted`] when the sender aborted before offering a payload, and
214/// [`Error::PayloadCountMismatch`] or [`Error::InvalidBoolPayload`] for a payload that
215/// disagrees with its header.
216#[cfg(any(feature = "system", feature = "runtime"))]
217pub fn recv<X: Transport>(from: mpi::topology::Process<'_>, tag: Tag) -> Result<X> {
218    engine::recv(&ACTIVE_BACKEND, from, tag)
219}
220
221/// Receive one tensor value into `out` from `from` on `tag`.
222///
223/// # Examples
224///
225/// ```
226/// use tenferro_mpi::mpi::traits::*;
227/// use tenferro_mpi::recv_into;
228/// use tenferro_tensor::{DynRank, Host, TypedTensor};
229///
230/// let universe = tenferro_mpi::mpi::initialize().unwrap();
231/// let world = universe.world();
232/// let mut out = TypedTensor::<i64, DynRank, Host>::from_host_vec_col_major(vec![2], vec![0, 0])?;
233/// if world.size() >= 2 {
234///     let value = TypedTensor::<i64, DynRank, Host>::from_host_vec_col_major(vec![2], vec![4, 5])?;
235///     if world.rank() == 0 {
236///         tenferro_mpi::send(world.process_at_rank(1), 13, &value)?;
237///     } else if world.rank() == 1 {
238///         recv_into(world.process_at_rank(0), 13, &mut out)?;
239///         assert_eq!(out.as_slice(), &[4, 5]);
240///     }
241/// }
242/// assert_eq!(out.shape(), &[2]);
243/// # Ok::<(), Box<dyn std::error::Error>>(())
244/// ```
245///
246/// # Errors
247///
248/// Returns [`Error::InvalidTag`] for a negative tag, [`Error::DTypeMismatch`],
249/// [`Error::RankMismatch`] or [`Error::ShapeMismatch`] for a destination of a different
250/// dtype or shape, [`Error::NotHostResident`] or [`Error::NonCompact`] for a destination
251/// that cannot be written as a compact host slice, [`Error::PeerAborted`] when the
252/// sender aborted, and [`Error::PayloadCountMismatch`] or [`Error::InvalidBoolPayload`]
253/// for a payload that disagrees with its header.
254#[cfg(any(feature = "system", feature = "runtime"))]
255pub fn recv_into<X: Transport>(
256    from: mpi::topology::Process<'_>,
257    tag: Tag,
258    out: &mut X,
259) -> Result<()> {
260    engine::recv_into(&ACTIVE_BACKEND, from, tag, out)
261}