pub trait TracedTensorAdExt {
// Required methods
fn grad(&self, wrt: &TracedTensor) -> Result<TracedTensor>;
fn grad_optional(&self, wrt: &TracedTensor) -> Result<Option<TracedTensor>>;
fn checkpoint(
&mut self,
compiler: &mut GraphCompiler,
runtime: &Runtime,
engine: &mut dyn ExecutionEngine,
) -> Result<()>;
fn jvp(
&self,
wrt: &TracedTensor,
tangent: &TracedTensor,
) -> Result<TracedTensor>;
fn jvp_optional(
&self,
wrt: &TracedTensor,
tangent: &TracedTensor,
) -> Result<Option<TracedTensor>>;
fn vjp(
&self,
wrt: &TracedTensor,
cotangent: &TracedTensor,
) -> Result<TracedTensor>;
fn vjp_optional(
&self,
wrt: &TracedTensor,
cotangent: &TracedTensor,
) -> Result<Option<TracedTensor>>;
fn grad_many(
&self,
wrts: &[&TracedTensor],
) -> Result<Vec<Option<TracedTensor>>>;
fn value_and_grad(
&self,
wrts: &[&TracedTensor],
) -> Result<(TracedTensor, Vec<Option<TracedTensor>>)>;
fn jvp_many(
&self,
wrt_tangents: &[(&TracedTensor, &TracedTensor)],
) -> Result<Option<TracedTensor>>;
fn vjp_many(
&self,
wrts: &[&TracedTensor],
cotangent: &TracedTensor,
) -> Result<Vec<Option<TracedTensor>>>;
}Required Methods§
Sourcefn grad(&self, wrt: &TracedTensor) -> Result<TracedTensor>
fn grad(&self, wrt: &TracedTensor) -> Result<TracedTensor>
Gradient of a scalar output with respect to a traced input.
For complex scalar outputs, tenferro returns the Hermitian-adjoint
cotangent. To compare seed-1 scalar gradients with JAX’s public
grad values, use the complex conjugate of this result. See
https://tensor4all.org/tenferro-rs/guides/complex-ad.html.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, GraphCompiler, Runtime, TracedTensor};
fn eval(tensor: &TracedTensor) -> tenferro_runtime::Tensor {
let mut compiler = GraphCompiler::new();
let program = compiler.compile(tensor).unwrap();
let mut engine = tenferro_cpu::CpuEngine::serial();
let mut builder = Runtime::builder();
builder
.register_engine(tenferro_cpu::runtime_engine_registration(&engine).unwrap())
.unwrap();
let runtime = builder.build().unwrap();
let target = tenferro_runtime::ExecutionEngine::target(&engine);
let prepared = runtime.prepare(&program, &[], &[target]).unwrap();
let mut outputs = runtime.run(&mut [&mut engine], &prepared, &[]).unwrap();
outputs.pop().unwrap().into_owned().unwrap()
}
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![], vec![3.0_f64]).unwrap();
let loss = (&x * &x).unwrap();
let dx = loss.grad(&x).unwrap();
assert_eq!(eval(&dx).as_slice::<f64>().unwrap(), &[6.0]);§Errors
Returns tenferro_runtime::Error::NonScalarGrad for a non-scalar
output, tenferro_runtime::Error::UnsupportedAdRule when an AD rule
is unavailable, or a typed validation/backend/runtime-state error. An
inactive wrt returns Error::Validation with argument: "wrt"; use
grad_optional to observe that state.
§Deferred errors
Symbolic shape constraints can later produce
tenferro_runtime::Error::ShapeConstraintViolation or
tenferro_runtime::Error::ShapeConstraintEvaluation during compile
or execution.
Sourcefn grad_optional(&self, wrt: &TracedTensor) -> Result<Option<TracedTensor>>
fn grad_optional(&self, wrt: &TracedTensor) -> Result<Option<TracedTensor>>
Like grad, but returns None when wrt is inactive.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, TracedTensor};
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![], vec![3.0_f64]).unwrap();
let y = trace.from_vec_col_major(vec![], vec![4.0_f64]).unwrap();
let loss = (&y * &y).unwrap();
assert!(loss.grad_optional(&x).unwrap().is_none());§Errors
Returns tenferro_runtime::Error::NonScalarGrad for a non-scalar
output, Error::UnsupportedAdRule when an AD rule is unavailable, or
a typed validation/backend/runtime-state error.
§Deferred errors
Symbolic shape constraints can later produce
tenferro_runtime::Error::ShapeConstraintViolation or
tenferro_runtime::Error::ShapeConstraintEvaluation during compile
or execution.
Sourcefn checkpoint(
&mut self,
compiler: &mut GraphCompiler,
runtime: &Runtime,
engine: &mut dyn ExecutionEngine,
) -> Result<()>
fn checkpoint( &mut self, compiler: &mut GraphCompiler, runtime: &Runtime, engine: &mut dyn ExecutionEngine, ) -> Result<()>
Evaluate this tensor and replace its graph with a concrete leaf while preserving the previous graph for AD replay.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, GraphCompiler, Runtime, TracedTensor};
let mut compiler = GraphCompiler::new();
let backend = tenferro_cpu::CpuEngine::serial();
let mut builder = Runtime::builder();
builder
.register_engine(tenferro_cpu::runtime_engine_registration(&backend).unwrap())
.unwrap();
let runtime = builder.build().unwrap();
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![], vec![3.0_f64]).unwrap();
let mut y = (&x * &x).unwrap();
let mut engine = tenferro_cpu::CpuEngine::serial();
y.checkpoint(&mut compiler, &runtime, &mut engine).unwrap();
let value = y.attached_value().unwrap();
let view = value
.tensor_read()?
.tensor_view()
.ok_or("a checkpointed f64 value has a preset-scalar view")?;
let materialized = view.duplicate()?;
assert_eq!(materialized.as_slice::<f64>()?, &[9.0]);§Errors
Returns tenferro_runtime::Error::Validation when checkpoint metadata
is invalid, Error::RuntimeState when graph metadata or runtime
state is unavailable, or a typed backend error from evaluation.
Sourcefn jvp(
&self,
wrt: &TracedTensor,
tangent: &TracedTensor,
) -> Result<TracedTensor>
fn jvp( &self, wrt: &TracedTensor, tangent: &TracedTensor, ) -> Result<TracedTensor>
Forward-mode Jacobian-vector product.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, GraphCompiler, Runtime, TracedTensor};
fn eval(tensor: &TracedTensor) -> tenferro_runtime::Tensor {
let mut compiler = GraphCompiler::new();
let program = compiler.compile(tensor).unwrap();
let mut engine = tenferro_cpu::CpuEngine::serial();
let mut builder = Runtime::builder();
builder
.register_engine(tenferro_cpu::runtime_engine_registration(&engine).unwrap())
.unwrap();
let runtime = builder.build().unwrap();
let target = tenferro_runtime::ExecutionEngine::target(&engine);
let prepared = runtime.prepare(&program, &[], &[target]).unwrap();
let mut outputs = runtime.run(&mut [&mut engine], &prepared, &[]).unwrap();
outputs.pop().unwrap().into_owned().unwrap()
}
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![], vec![3.0_f64]).unwrap();
let tangent = trace.from_vec_col_major(vec![], vec![2.0_f64]).unwrap();
let y = (&x * &x).unwrap();
let dy = y.jvp(&x, &tangent).unwrap();
assert_eq!(eval(&dy).as_slice::<f64>().unwrap(), &[12.0]);§Errors
Returns tenferro_runtime::Error::UnsupportedAdRule when a JVP rule
is unavailable, Error::Validation for incompatible tangent metadata,
or a typed backend/runtime-state error. An inactive wrt returns
Error::Validation with argument: "wrt"; use
jvp_optional to observe that state.
§Deferred errors
Symbolic shape constraints can later produce
tenferro_runtime::Error::ShapeConstraintViolation or
tenferro_runtime::Error::ShapeConstraintEvaluation during compile
or execution.
Sourcefn jvp_optional(
&self,
wrt: &TracedTensor,
tangent: &TracedTensor,
) -> Result<Option<TracedTensor>>
fn jvp_optional( &self, wrt: &TracedTensor, tangent: &TracedTensor, ) -> Result<Option<TracedTensor>>
Like jvp, but returns None when wrt is inactive.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, TracedTensor};
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![], vec![3.0_f64]).unwrap();
let y = trace.from_vec_col_major(vec![], vec![4.0_f64]).unwrap();
let tangent = trace.from_vec_col_major(vec![], vec![1.0_f64]).unwrap();
let loss = (&y * &y).unwrap();
assert!(loss.jvp_optional(&x, &tangent).unwrap().is_none());§Errors
Returns tenferro_runtime::Error::UnsupportedAdRule when a JVP rule
is unavailable, Error::Validation for incompatible tangent metadata,
or a typed backend/runtime-state error.
§Deferred errors
Symbolic shape constraints can later produce
tenferro_runtime::Error::ShapeConstraintViolation or
tenferro_runtime::Error::ShapeConstraintEvaluation during compile
or execution.
Sourcefn vjp(
&self,
wrt: &TracedTensor,
cotangent: &TracedTensor,
) -> Result<TracedTensor>
fn vjp( &self, wrt: &TracedTensor, cotangent: &TracedTensor, ) -> Result<TracedTensor>
Reverse-mode vector-Jacobian product.
Complex cotangents use tenferro’s Hermitian real-inner-product convention. Non-real complex cotangent seeds therefore need an explicit seed-convention comparison when matching JAX. See https://tensor4all.org/tenferro-rs/guides/complex-ad.html.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, GraphCompiler, Runtime, TracedTensor};
fn eval(tensor: &TracedTensor) -> tenferro_runtime::Tensor {
let mut compiler = GraphCompiler::new();
let program = compiler.compile(tensor).unwrap();
let mut engine = tenferro_cpu::CpuEngine::serial();
let mut builder = Runtime::builder();
builder
.register_engine(tenferro_cpu::runtime_engine_registration(&engine).unwrap())
.unwrap();
let runtime = builder.build().unwrap();
let target = tenferro_runtime::ExecutionEngine::target(&engine);
let prepared = runtime.prepare(&program, &[], &[target]).unwrap();
let mut outputs = runtime.run(&mut [&mut engine], &prepared, &[]).unwrap();
outputs.pop().unwrap().into_owned().unwrap()
}
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![], vec![3.0_f64]).unwrap();
let cotangent = trace.from_vec_col_major(vec![], vec![0.5_f64]).unwrap();
let y = (&x * &x).unwrap();
let dx = y.vjp(&x, &cotangent).unwrap();
assert_eq!(eval(&dx).as_slice::<f64>().unwrap(), &[3.0]);§Errors
Returns tenferro_runtime::Error::UnsupportedAdRule when a VJP rule
is unavailable, Error::Validation for incompatible cotangent
metadata, or a typed backend/runtime-state error. An inactive wrt
returns Error::Validation with argument: "wrt"; use
vjp_optional to observe that state.
§Deferred errors
Symbolic shape constraints can later produce
tenferro_runtime::Error::ShapeConstraintViolation or
tenferro_runtime::Error::ShapeConstraintEvaluation during compile
or execution.
Sourcefn vjp_optional(
&self,
wrt: &TracedTensor,
cotangent: &TracedTensor,
) -> Result<Option<TracedTensor>>
fn vjp_optional( &self, wrt: &TracedTensor, cotangent: &TracedTensor, ) -> Result<Option<TracedTensor>>
Like vjp, but returns None when wrt is inactive.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, TracedTensor};
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![], vec![3.0_f64]).unwrap();
let y = trace.from_vec_col_major(vec![], vec![4.0_f64]).unwrap();
let cotangent = trace.from_vec_col_major(vec![], vec![1.0_f64]).unwrap();
let loss = (&y * &y).unwrap();
assert!(loss.vjp_optional(&x, &cotangent).unwrap().is_none());§Errors
Returns tenferro_runtime::Error::UnsupportedAdRule when a VJP rule
is unavailable, Error::Validation for incompatible cotangent
metadata, or a typed backend/runtime-state error.
§Deferred errors
Symbolic shape constraints can later produce
tenferro_runtime::Error::ShapeConstraintViolation or
tenferro_runtime::Error::ShapeConstraintEvaluation during compile
or execution.
Sourcefn grad_many(&self, wrts: &[&TracedTensor]) -> Result<Vec<Option<TracedTensor>>>
fn grad_many(&self, wrts: &[&TracedTensor]) -> Result<Vec<Option<TracedTensor>>>
Reverse-mode gradients for several leaves from one transform.
One snapshot, one program-level transform and one import cover the whole
wrt set, so the backward is a single sweep instead of one per leaf.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, DType};
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
let w = trace.from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
let loss = (&(&x * &w)?).reduce_sum(None)?;
let grads = loss.grad_many(&[&x, &w])?;
assert_eq!(grads.len(), 2);§Errors
Returns Error::NonScalarGrad for a non-scalar output,
Error::UnsupportedAdRule when a rule is unavailable, or a typed
validation/runtime-state error from the snapshot or the transform.
Sourcefn value_and_grad(
&self,
wrts: &[&TracedTensor],
) -> Result<(TracedTensor, Vec<Option<TracedTensor>>)>
fn value_and_grad( &self, wrts: &[&TracedTensor], ) -> Result<(TracedTensor, Vec<Option<TracedTensor>>)>
The scalar output together with reverse-mode gradients for several leaves.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{TraceContext, DType};
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
let loss = (&(&x * &x)?).reduce_sum(None)?;
let (value, grads) = loss.value_and_grad(&[&x])?;
assert_eq!(value.rank(), 0);
assert!(grads[0].is_some());§Errors
Returns Error::NonScalarGrad for a non-scalar output,
Error::UnsupportedAdRule when a rule is unavailable, or a typed
validation/runtime-state error from the snapshot or the transform; the
gradients follow grad_many.
Sourcefn jvp_many(
&self,
wrt_tangents: &[(&TracedTensor, &TracedTensor)],
) -> Result<Option<TracedTensor>>
fn jvp_many( &self, wrt_tangents: &[(&TracedTensor, &TracedTensor)], ) -> Result<Option<TracedTensor>>
Forward-mode JVPs for several leaves from one transform.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{DType, TraceContext};
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
let dx = trace.from_vec_col_major(vec![2], vec![1.0_f64, 0.0])?;
let loss = (&(&x * &x)?).reduce_sum(None)?;
let tangent = loss.jvp_many(&[(&x, &dx)])?;
assert!(tangent.is_some());§Errors
Returns Error::UnsupportedAdRule when a rule is unavailable, or a
typed validation/runtime-state error from the snapshot or the transform.
Sourcefn vjp_many(
&self,
wrts: &[&TracedTensor],
cotangent: &TracedTensor,
) -> Result<Vec<Option<TracedTensor>>>
fn vjp_many( &self, wrts: &[&TracedTensor], cotangent: &TracedTensor, ) -> Result<Vec<Option<TracedTensor>>>
Reverse-mode VJPs for several leaves from one transform.
§Examples
use tenferro_ad::TracedTensorAdExt;
use tenferro_runtime::{DType, TraceContext};
let trace = TraceContext::new();
let x = trace.from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
let loss = (&(&x * &x)?).reduce_sum(None)?;
let seed = trace.from_vec_col_major(vec![], vec![1.0_f64])?;
let grads = loss.vjp_many(&[&x], &seed)?;
assert!(grads[0].is_some());§Errors
Returns Error::UnsupportedAdRule when a rule is unavailable, or a
typed validation/runtime-state error from the snapshot or the transform.
Dyn Compatibility§
This trait is dyn compatible.
In older versions of Rust, dyn compatibility was called "object safety".