diff --git a/Cargo.lock b/Cargo.lock index 85b7f1c..9b0ec7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1207,7 +1207,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "sliding_features" -version = "10.0.1" +version = "11.0.0" dependencies = [ "ballpark", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 9fd8de0..d9fce03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sliding_features" -version = "10.0.1" +version = "11.0.0" authors = ["MathisWellmann "] edition = "2024" license-file = "LICENSE" @@ -82,6 +82,10 @@ harness = false name = "noise_elimination_technology" harness = false +[[bench]] +name = "pipeline" +harness = false + [[bench]] name = "polarized_fractal_efficiency" harness = false diff --git a/README.md b/README.md index 5bc2d91..0b16af1 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,45 @@ for v in &test_values { Each View will first call it's chained View to get it's last value, which will then be used to update the state of the View. Some Views have additional parameters such as ALMA. +### Features at runtime: the `pipeline` module +Chained views are the zero-cost choice when the features are known at compile +time. When they are only known at runtime, write them as text and compile +them with the `pipeline` module, the crate's one way to assemble features at +runtime. Features can then be stored, compared and generated (e.g. by a +search or an LLM agent) as data, without compiling code. A pipeline computes +many features over several input series in one flat program: + +```rust +use sliding_features::pipeline::{Domain, Input, Limits, Pipeline}; + +let source = " + let r = ln_return(price) + vol_ratio = std(r, 20) / std(r, 100) + trend = zscore(ema(price, 10) - ema(price, 50), 200) +"; +let inputs = vec![Input::new("price", Domain::positive())]; +let mut pipeline = Pipeline::compile(source, inputs, Limits::default()).unwrap(); +pipeline.update(&[100.0]); +assert_eq!(pipeline.output(0), None, "warming up"); +``` + +- `name = expr` defines an output, `let name = expr` a helper, and every + function takes an expression and then numeric parameters, e.g. `pfe(x, 20, 5)`. + `pipeline::reference()` renders the language guide and the table of all + functions, ready for a prompt. +- Equal subexpressions are computed once, and an update runs a flat list of + operations with no allocation, recursion or dynamic dispatch. +- Before any data is seen, the compiler derives every node's value range from + the inputs' domains: it rejects calls that can never be ready (`ln` of a + negative value) and constant outputs, and warns about calls that are not + always ready, with line, column and a hint how to fix it. +- Outputs are only ever finite: a division by zero is "not ready" instead of + infinite. `Limits` bounds windows, node count and nesting, so text from an + untrusted author cannot make a pipeline arbitrarily slow or large. +- The `Display` form is canonical, so equal strings mean equal features. +- `pipeline::Builder` assembles the same pipelines from code, and + `pipeline::Feature` turns a one-input, one-output pipeline into a `View`. + ### Examples See examples folder for some code ideas ```shell @@ -92,6 +131,8 @@ A View defines the function which processes value updates. They currently includ * Multiply * Divide * Math functions + * Abs + * Ln * Tanh * GTE - Greater Than or Equal clipping function * LTE - Lower Than or Equal clipping function diff --git a/benches/pipeline.rs b/benches/pipeline.rs new file mode 100644 index 0000000..1cf3ab2 --- /dev/null +++ b/benches/pipeline.rs @@ -0,0 +1,133 @@ +//! A pipeline against the same feature as a hand-written generic chain, and +//! a feature set as one pipeline against one pipeline per feature. + +use std::{ + hint::black_box, + num::NonZeroUsize, +}; + +use criterion::{ + Criterion, + criterion_group, + criterion_main, +}; +use sliding_features::{ + View, + pipeline::{ + Feature, + Pipeline, + }, + pure_functions::{ + Echo, + Subtract, + }, + sliding_windows::{ + Ema, + Sma, + ZScoreStandardization, + }, +}; + +const N: usize = 100_000; + +fn random_walk() -> Vec { + let rng = romu::Rng::from_seed_with_64bit(1); + let mut price = 100.0; + Vec::from_iter((0..N).map(|_| { + price += rng.f64() - 0.5; + price + })) +} + +fn feature(source: &str) -> Feature { + let pipeline: Pipeline = source.parse().expect("compiles"); + Feature::try_from(pipeline).expect("one input and one output") +} + +/// Push every value through a view and read it, as a feature loop does. +fn drive(view: &mut impl View, vals: &[f64]) { + for v in vals { + view.update(*v); + let _ = black_box(view.last()); + } +} + +/// One feature: the cost of interpreting the program against code the +/// compiler monomorphized. +fn one_feature(c: &mut Criterion) { + let vals = random_walk(); + let source = "zscore(ema(x, 10) - sma(x, 50), 200)"; + let window = |n| NonZeroUsize::new(n).expect("non zero"); + let mut group = c.benchmark_group("pipeline_one_feature_100k"); + group.bench_function("static_chain", |b| { + b.iter(|| { + let mut view = ZScoreStandardization::new( + Subtract::new( + Ema::new(Echo::new(), window(10)), + Sma::new(Echo::new(), window(50)), + ), + window(200), + ); + drive(&mut view, &vals); + }) + }); + group.bench_function("pipeline", |b| { + b.iter(|| drive(&mut feature(source), &vals)) + }); + group.finish(); +} + +/// Mostly stateless nodes: the per-operation overhead of the interpreter. +fn stateless_heavy(c: &mut Criterion) { + let vals = random_walk(); + let source = "tanh(abs(-(-(-(-(x * 2 + 1)))) - ema(x, 5)) * 0.01)"; + let mut group = c.benchmark_group("pipeline_stateless_heavy_100k"); + group.bench_function("pipeline", |b| { + b.iter(|| drive(&mut feature(source), &vals)) + }); + group.finish(); +} + +/// A feature set with shared subexpressions: one pipeline computes each of +/// them once, one pipeline per feature computes them per feature. +fn feature_set(c: &mut Criterion) { + let vals = random_walk(); + let sources = [ + "std(ln_return(x), 20) / std(ln_return(x), 100)", + "std(ln_return(x), 20)", + "zscore(ema(x, 10) - ema(x, 50), 200)", + "mad(ema(x, 10) - ema(x, 50), 100)", + "roc(ema(x, 10), 10)", + "rsi(x, 14)", + "ema(rsi(x, 14), 5)", + "tanh(zscore(ln_return(x), 100))", + ]; + let mut group = c.benchmark_group("pipeline_feature_set_100k"); + group.bench_function("pipeline_per_feature", |b| { + b.iter(|| { + let mut features = Vec::from_iter(sources.iter().map(|s| feature(s))); + for v in &vals { + for feature in &mut features { + feature.update(*v); + let _ = black_box(feature.last()); + } + } + }) + }); + group.bench_function("one_pipeline", |b| { + let source = sources.join("\n"); + b.iter(|| { + let mut pipeline: Pipeline = source.parse().expect("compiles"); + for v in &vals { + pipeline.update(&[*v]); + for out in pipeline.outputs() { + let _ = black_box(out); + } + } + }) + }); + group.finish(); +} + +criterion_group!(benches, one_feature, stateless_heavy, feature_set); +criterion_main!(benches); diff --git a/img/cumulative.png b/img/cumulative.png index 70b8ca4..b56068e 100644 Binary files a/img/cumulative.png and b/img/cumulative.png differ diff --git a/img/polarized_fractal_efficiency.png b/img/polarized_fractal_efficiency.png index 5ccb73c..b0b3158 100644 Binary files a/img/polarized_fractal_efficiency.png and b/img/polarized_fractal_efficiency.png differ diff --git a/img/sma.png b/img/sma.png index 377bb1f..d061cab 100644 Binary files a/img/sma.png and b/img/sma.png differ diff --git a/src/lib.rs b/src/lib.rs index 618c5b0..2e87dd5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ //! The sliding_features crate provides modular, chainable sliding windows //! for various signal processing function and technical indicators +pub mod pipeline; pub mod pure_functions; pub mod rolling; pub mod sliding_windows; diff --git a/src/pipeline/analysis.rs b/src/pipeline/analysis.rs new file mode 100644 index 0000000..e6a3c8d --- /dev/null +++ b/src/pipeline/analysis.rs @@ -0,0 +1,210 @@ +//! Checks before a pipeline runs: the domain and warm-up of every node, +//! and what is likely a mistake, worded so the author can fix it. + +use super::{ + BinaryOp, + Diagnostic, + Domain, + Input, + Location, + Requirement, + builder::Binding, + display::Printer, + graph::{ + Graph, + Node, + NodeId, + }, +}; + +/// What the compiler knows about every node before any data is seen. +#[derive(Debug, Clone)] +pub(super) struct Analysis { + domains: Vec, + warm_ups: Vec, + diagnostics: Vec, +} + +impl Analysis { + /// Derive domains and warm-ups in one pass over the nodes (children + /// come first), then check the bindings. Diagnostics point at the + /// node's location in the text form, if any. + pub(super) fn run( + graph: &Graph, + inputs: &[Input], + bindings: &[Binding], + locations: &[Option], + ) -> Self { + assert_eq!(locations.len(), graph.len(), "a location per node"); + let name_of = |id: NodeId| { + bindings + .iter() + .find(|b| b.is_named() && b.node() == id) + .map(Binding::name) + }; + let printer = Printer::new(graph, inputs, name_of); + let mut analysis = Self { + domains: Vec::with_capacity(graph.len()), + warm_ups: Vec::with_capacity(graph.len()), + diagnostics: Vec::new(), + }; + for (id, node) in graph.iter() { + let (domain, warm_up, diagnostic) = analysis.node(&printer, id, node, inputs); + analysis.domains.push(domain); + analysis.warm_ups.push(warm_up); + let at = locations[id.index()]; + analysis.diagnostics.extend(diagnostic.map(|d| d.or_at(at))); + } + assert_eq!(analysis.domains.len(), graph.len(), "a domain per node"); + let findings = analysis.check_bindings(graph, bindings); + analysis.diagnostics.extend(findings); + analysis + } + + /// The values node `id` can take. + pub(super) fn domain(&self, id: NodeId) -> Domain { + self.domains[id.index()] + } + + /// An estimate of the updates node `id` needs before it is ready. + pub(super) fn warm_up(&self, id: NodeId) -> usize { + self.warm_ups[id.index()] + } + + /// Every problem found, in node order, then the bindings'. + pub(super) fn diagnostics(&self) -> &[Diagnostic] { + &self.diagnostics + } + + /// The domain and warm-up of one node, and a problem with it. + fn node<'a>( + &self, + printer: &Printer<'a, impl Fn(NodeId) -> Option<&'a str>>, + id: NodeId, + node: &Node, + inputs: &[Input], + ) -> (Domain, usize, Option) { + let d = |n: &NodeId| self.domains[n.index()]; + let w = |n: &NodeId| self.warm_ups[n.index()]; + match node { + Node::Input(i) => (inputs[*i].domain(), 0, None), + Node::Const(c) => (Domain::point(*c), 0, None), + Node::Neg(a) => (d(a).neg(), w(a), None), + Node::Binary(op, a, b) => { + let warm_up = w(a).max(w(b)); + match binary_domain(*op, a == b, d(a), d(b)) { + Some(domain) => (domain, warm_up, None), + None => { + let message = format!( + "`{}` divides by zero, so it is never ready", + printer.expr(id) + ); + (Domain::REAL, warm_up, Some(Diagnostic::error(message))) + } + } + } + Node::Call(function, a, params) => { + let warm_up = w(a) + function.warm_up(params); + let (arg, requires) = (d(a), function.requires()); + let Some(domain) = function.output_domain(arg, params) else { + let message = format!( + "`{}` is never ready: `{}` needs {} values, but `{}` lies in {arg}", + printer.expr(id), + function.name(), + requirement_words(requires), + printer.name_or_expr(*a), + ); + return (Domain::REAL, warm_up, Some(Diagnostic::error(message))); + }; + let partly = arg.intersect(requires.domain()) != Some(arg); + // The entropy of signs counts values of at least zero. + let one_sign = requires == Requirement::BothSigns + && (arg.lo() >= 0.0 || arg.intersect(Domain::non_negative()).is_none()); + let warning = if partly { + let hint = match printer.input_name(*a) { + Some(name) => format!( + "; declare the domain of input `{name}` if it is always {}", + requirement_words(requires) + ), + None => String::new(), + }; + Some(format!( + "`{}` is not ready while `{}` is not {} (it lies in {arg}){hint}", + printer.expr(id), + printer.name_or_expr(*a), + requirement_words(requires), + )) + } else if one_sign { + Some(format!( + "`{}` is constant: `{}` never changes sign (it lies in {arg})", + printer.expr(id), + printer.name_or_expr(*a), + )) + } else { + None + }; + (domain, warm_up, warning.map(Diagnostic::warning)) + } + } + } + + /// Constant and duplicate outputs, and helpers no output uses. + fn check_bindings(&self, graph: &Graph, bindings: &[Binding]) -> Vec { + let outputs = Vec::from_iter(bindings.iter().filter(|b| b.is_output())); + let reachable = graph.reachable(outputs.iter().map(|b| b.node())); + let mut found = Vec::new(); + for (i, binding) in bindings.iter().enumerate() { + let name = binding.name(); + let at = binding.location(); + if !binding.is_output() { + if !reachable[binding.node().index()] { + found.push( + Diagnostic::warning(format!("`{name}` is not used by any output")) + .or_at(at), + ); + } + continue; + } + if let Some(c) = self.domain(binding.node()).as_point() { + found.push( + Diagnostic::error(format!("output `{name}` is constant: it is always {c}")) + .or_at(at), + ); + } + let earlier = bindings[..i] + .iter() + .find(|b| b.is_output() && b.node() == binding.node()); + if let Some(earlier) = earlier { + let message = format!( + "output `{name}` computes the same as output `{}`", + earlier.name() + ); + found.push(Diagnostic::warning(message).or_at(at)); + } + } + found + } +} + +/// The domain of `a op b`, `None` if it is never finite. `same` says +/// that both operands are one node, so `x - x` is exactly zero. +fn binary_domain(op: BinaryOp, same: bool, lhs: Domain, rhs: Domain) -> Option { + if same { + match op { + BinaryOp::Sub => return Some(Domain::point(0.0)), + // Where `a` is zero the quotient is not ready, elsewhere 1. + BinaryOp::Div if lhs.as_point() != Some(0.0) => return Some(Domain::point(1.0)), + BinaryOp::Mul => return Some(lhs.abs().map_increasing(|v| v * v)), + BinaryOp::Add | BinaryOp::Div => {} + } + } + op.output_domain(lhs, rhs) +} + +fn requirement_words(requires: Requirement) -> &'static str { + match requires { + Requirement::Positive => "positive", + Requirement::NonNegative => "non-negative", + Requirement::Any | Requirement::BothSigns => "finite", + } +} diff --git a/src/pipeline/builder.rs b/src/pipeline/builder.rs new file mode 100644 index 0000000..0b6dbab --- /dev/null +++ b/src/pipeline/builder.rs @@ -0,0 +1,534 @@ +//! Assembling a pipeline node by node: the programmatic way to build one, +//! e.g. for a search, and the target of the text form. + +use super::{ + BinaryOp, + CompileError, + Domain, + Function, + Location, + Pipeline, + Severity, + analysis::Analysis, + display::Printer, + function::Params, + graph::{ + Graph, + Node, + NodeId, + }, + state::Kind, +}; + +/// Resource bounds of a pipeline, so one from an untrusted author (e.g. an +/// LLM) cannot make it arbitrarily slow or large. Plain data: any values +/// are valid, small ones just reject more pipelines. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Limits { + /// The largest window parameter. + pub max_window: usize, + /// The most distinct nodes, inputs and constants included; a shared + /// subexpression counts once. + pub max_nodes: usize, + /// The longest chain of nodes from an input or constant to any node, + /// and the deepest nesting of parentheses and calls in the text form. + pub max_depth: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + max_window: 100_000, + max_nodes: 256, + max_depth: 32, + } + } +} + +impl Limits { + /// No bounds besides the function signatures. + pub fn unbounded() -> Self { + Self { + max_window: usize::MAX, + max_nodes: usize::MAX, + max_depth: usize::MAX, + } + } +} + +/// An input series of a pipeline: its name and the values it takes. The +/// domain lets the compiler check functions with requirements, e.g. that +/// `ln(price)` is defined when prices are declared positive. +#[derive(Debug, Clone, PartialEq)] +pub struct Input { + name: String, + domain: Domain, +} + +impl Input { + /// An input called `name` whose values lie in `domain`. + pub fn new(name: impl Into, domain: Domain) -> Self { + Self { + name: name.into(), + domain, + } + } + + /// An input called `name` that takes any finite value. + pub fn real(name: impl Into) -> Self { + Self::new(name, Domain::REAL) + } + + /// The name expressions refer to the input by. + pub fn name(&self) -> &str { + &self.name + } + + /// The values the input takes. + pub fn domain(&self) -> Domain { + self.domain + } +} + +/// What a binding names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum BindingKind { + /// A node later statements refer to by name (`let name = ...`). + Helper, + /// An output (`name = ...`). + Output, + /// An output written as a bare expression, named by its text. + Anonymous, +} + +/// A named node, see [`BindingKind`]. +#[derive(Debug, Clone, PartialEq)] +pub(super) struct Binding { + name: String, + node: NodeId, + kind: BindingKind, + location: Option, +} + +impl Binding { + pub(super) fn name(&self) -> &str { + &self.name + } + + pub(super) fn node(&self) -> NodeId { + self.node + } + + pub(super) fn kind(&self) -> BindingKind { + self.kind + } + + pub(super) fn is_output(&self) -> bool { + self.kind != BindingKind::Helper + } + + /// Whether expressions can refer to the node by the binding's name. + pub(super) fn is_named(&self) -> bool { + self.kind != BindingKind::Anonymous + } + + /// Where the binding is in the text form. + pub(super) fn location(&self) -> Option { + self.location + } +} + +/// Assembles a [`Pipeline`] node by node. +/// +/// Equal computations are shared: asking twice for `ema(x, 10)` returns +/// the same node, so it is computed once per update. Calls on constants +/// are folded into constants. +/// +/// ``` +/// use sliding_features::pipeline::{BinaryOp, Builder, Function, Input, Limits}; +/// +/// let mut b = Builder::new(vec![Input::real("x")], Limits::default()).unwrap(); +/// let x = b.lookup("x").unwrap(); +/// let fast = b.call(Function::Ema, x, &[10.0]).unwrap(); +/// let slow = b.call(Function::Ema, x, &[50.0]).unwrap(); +/// let cross = b.binary(BinaryOp::Sub, fast, slow).unwrap(); +/// b.output("cross", cross).unwrap(); +/// let mut pipeline = b.build().unwrap(); +/// pipeline.update(&[100.0]); +/// assert_eq!(pipeline.output(0), None, "the EMAs warm up"); +/// for _ in 0..50 { +/// pipeline.update(&[100.0]); +/// } +/// assert_eq!(pipeline.output(0), Some(0.0)); +/// ``` +#[derive(Debug, Clone)] +pub struct Builder { + graph: Graph, + /// The depth of every node, see [`Limits::max_depth`]. + depths: Vec, + inputs: Vec, + /// The node of each input. + input_nodes: Vec, + bindings: Vec, + limits: Limits, + /// Where in the text form each node first appears. + locations: Vec>, + /// Where the text form is, for new nodes, bindings and errors. + location: Option, +} + +impl Builder { + /// A builder over `inputs`, which are fed in this order. + pub fn new(inputs: Vec, limits: Limits) -> Result { + if inputs.is_empty() { + return Err(CompileError::single("a pipeline needs at least one input")); + } + let mut builder = Self { + graph: Graph::default(), + depths: Vec::new(), + inputs: Vec::new(), + input_nodes: Vec::new(), + bindings: Vec::new(), + limits, + locations: Vec::new(), + location: None, + }; + for (i, input) in inputs.into_iter().enumerate() { + builder.check_name(input.name())?; + let id = builder.add(Node::Input(i))?; + assert_eq!(id.index(), i, "inputs are the first nodes"); + builder.inputs.push(input); + builder.input_nodes.push(id); + } + Ok(builder) + } + + /// The input or binding called `name`. + pub fn lookup(&self, name: &str) -> Option { + if let Some(i) = self.inputs.iter().position(|input| input.name() == name) { + return Some(self.input_nodes[i]); + } + self.bindings + .iter() + .find(|binding| binding.is_named() && binding.name == name) + .map(|binding| binding.node) + } + + /// Every name expressions can refer to: inputs, then bindings. + pub(super) fn names(&self) -> impl Iterator { + let inputs = self.inputs.iter().map(Input::name); + let bindings = self + .bindings + .iter() + .filter(|b| b.is_named()) + .map(Binding::name); + inputs.chain(bindings) + } + + /// The limits the builder enforces. + pub(super) fn limits(&self) -> Limits { + self.limits + } + + /// Attribute new nodes, bindings and errors to `location` in the text + /// form. + pub(super) fn set_location(&mut self, location: Location) { + self.location = Some(location); + } + + /// The constant `value`. + pub fn constant(&mut self, value: f64) -> Result { + if !value.is_finite() { + return Err(self.error(format!("the constant {value} is not finite"))); + } + // Adding zero turns -0 into 0, so both are one node. + self.add(Node::Const(value + 0.0)) + } + + /// `-arg`. + pub fn neg(&mut self, arg: NodeId) -> Result { + match *self.graph.get(arg) { + Node::Const(c) => self.constant(-c), + _ => self.add(Node::Neg(arg)), + } + } + + /// `lhs op rhs`. + pub fn binary( + &mut self, + op: BinaryOp, + lhs: NodeId, + rhs: NodeId, + ) -> Result { + if let (Node::Const(a), Node::Const(b)) = (self.graph.get(lhs), self.graph.get(rhs)) { + let (a, b) = (*a, *b); + let value = op.eval(a, b); + if !value.is_finite() { + return Err(self.error(format!("`{a} {} {b}` is not finite", op.symbol()))); + } + return self.constant(value); + } + self.add(Node::Binary(op, lhs, rhs)) + } + + /// `function(arg, params...)`, with parameters checked against the + /// signature and [`Limits::max_window`]. + pub fn call( + &mut self, + function: Function, + arg: NodeId, + params: &[f64], + ) -> Result { + function + .check_params(params, self.limits.max_window) + .map_err(|message| self.error(message))?; + if let Node::Const(c) = *self.graph.get(arg) + && let Kind::Pure(pure) = function.instantiate(Params::new(function, params)) + { + let value = pure.eval(c); + if !value.is_finite() { + return Err(self.error(format!( + "`{}` of the constant {c} is not finite", + function.name() + ))); + } + return self.constant(value); + } + self.add(Node::Call(function, arg, params.into())) + } + + /// Make `node` an output called `name`. Outputs are read in the order + /// they are added. + pub fn output(&mut self, name: &str, node: NodeId) -> Result<(), CompileError> { + self.check_name(name)?; + self.push_binding(name.to_string(), node, BindingKind::Output); + Ok(()) + } + + /// Name `node` without making it an output, so the text form refers to + /// it by `name` (`let name = ...`). + pub fn bind(&mut self, name: &str, node: NodeId) -> Result<(), CompileError> { + self.check_name(name)?; + self.push_binding(name.to_string(), node, BindingKind::Helper); + Ok(()) + } + + /// Make `node` an output named by its expression, as written by a bare + /// expression in the text form. + pub(super) fn anonymous_output(&mut self, node: NodeId) -> Result<(), CompileError> { + let name = self.expr_text(node); + let taken = self + .bindings + .iter() + .any(|b| b.is_output() && b.name() == name); + if taken { + return Err(self.error(format!("the output `{name}` is already defined"))); + } + self.push_binding(name, node, BindingKind::Anonymous); + Ok(()) + } + + /// The text of `node`, referring to named nodes by name. + fn expr_text(&self, node: NodeId) -> String { + let name_of = |id: NodeId| { + self.bindings + .iter() + .find(|b| b.is_named() && b.node() == id) + .map(Binding::name) + }; + Printer::new(&self.graph, &self.inputs, name_of).expr(node) + } + + /// Check the pipeline and return it with fresh state. + /// + /// The check derives the [`Domain`] of every node from the inputs'. + /// It rejects calls that are never ready (e.g. `ln` of a negative + /// argument), divisions by zero and constant outputs, and warns about + /// calls that are not always ready, duplicate outputs and unused + /// helpers, see [`Pipeline::warnings`]. The error lists every problem. + pub fn build(self) -> Result { + if !self.bindings.iter().any(Binding::is_output) { + return Err(CompileError::single("a pipeline needs at least one output")); + } + let analysis = Analysis::run(&self.graph, &self.inputs, &self.bindings, &self.locations); + let diagnostics = analysis.diagnostics(); + if diagnostics.iter().any(|d| d.severity() == Severity::Error) { + return Err(CompileError::new(diagnostics.to_vec())); + } + Ok(Pipeline::new( + self.graph, + self.inputs, + self.bindings, + &analysis, + )) + } + + fn push_binding(&mut self, name: String, node: NodeId, kind: BindingKind) { + assert!(node.index() < self.graph.len(), "a node of this builder"); + self.bindings.push(Binding { + name, + node, + kind, + location: self.location, + }); + } + + /// An error at the current location. + fn error(&self, message: impl Into) -> CompileError { + CompileError::single(message).or_at(self.location) + } + + /// Names are identifiers, so the text form can refer to them, and + /// unique among inputs and bindings. A name may equal a function's: + /// calls are followed by `(`, so `rsi = rsi(x, 14)` is unambiguous. + fn check_name(&self, name: &str) -> Result<(), CompileError> { + let mut chars = name.chars(); + let identifier = chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); + let problem = if !identifier { + "is not an identifier (a letter or `_`, then letters, digits or `_`)" + } else if name == "let" { + "is a keyword" + } else if self.lookup(name).is_some() { + "is already defined" + } else { + return Ok(()); + }; + Err(self.error(format!("the name `{name}` {problem}"))) + } + + /// Intern `node` within the limits. + fn add(&mut self, node: Node) -> Result { + let depth = 1 + node + .children() + .map(|c| self.depths[c.index()]) + .max() + .unwrap_or(0); + if depth > self.limits.max_depth { + return Err(self.error(format!( + "the expression is nested {depth} deep, the limit is {}", + self.limits.max_depth + ))); + } + let id = self + .graph + .intern(node, self.limits.max_nodes) + .ok_or_else(|| { + self.error(format!( + "the pipeline needs more than {} distinct nodes", + self.limits.max_nodes + )) + })?; + if id.index() == self.depths.len() { + self.depths.push(depth); + self.locations.push(self.location); + } + assert_eq!(self.depths.len(), self.graph.len(), "a depth per node"); + Ok(id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn builder() -> Builder { + Builder::new(vec![Input::real("x")], Limits::default()).unwrap() + } + + #[test] + fn equal_calls_share_one_node() { + let mut b = builder(); + let x = b.lookup("x").unwrap(); + let a = b.call(Function::Ema, x, &[10.0]).unwrap(); + let again = b.call(Function::Ema, x, &[10.0]).unwrap(); + assert_eq!(a, again); + let other = b.call(Function::Ema, x, &[11.0]).unwrap(); + assert_ne!(a, other); + } + + #[test] + fn constants_are_folded() { + let mut b = builder(); + let two = b.constant(2.0).unwrap(); + let three = b.constant(3.0).unwrap(); + let six = b.binary(BinaryOp::Mul, two, three).unwrap(); + assert_eq!(six, b.constant(6.0).unwrap()); + let neg = b.neg(six).unwrap(); + assert_eq!(neg, b.constant(-6.0).unwrap()); + let abs = b.call(Function::Abs, neg, &[]).unwrap(); + assert_eq!(abs, six); + assert_eq!(b.constant(-0.0).unwrap(), b.constant(0.0).unwrap()); + let zero = b.constant(0.0).unwrap(); + let err = b.binary(BinaryOp::Div, two, zero).unwrap_err(); + assert_eq!(err.to_string(), "error: `2 / 0` is not finite"); + let err = b.call(Function::Ln, neg, &[]).unwrap_err(); + assert_eq!( + err.to_string(), + "error: `ln` of the constant -6 is not finite" + ); + // Stateful calls on constants are not folded: they are not ready + // until their window fills. + let sma = b.call(Function::Sma, two, &[3.0]).unwrap(); + assert_ne!(sma, two); + } + + #[test] + fn names_are_checked() { + let mut b = builder(); + let x = b.lookup("x").unwrap(); + for (name, problem) in [ + ("x", "is already defined"), + ("let", "is a keyword"), + ("1a", "is not an identifier"), + ("a-b", "is not an identifier"), + ("", "is not an identifier"), + ] { + let err = b.output(name, x).unwrap_err().to_string(); + assert!(err.contains(problem), "{name}: {err}"); + } + b.bind("helper", x).unwrap(); + assert_eq!(b.lookup("helper"), Some(x)); + b.output("sma", x).unwrap(); + assert!(b.output("helper", x).is_err()); + assert!(Builder::new(vec![Input::real("x"), Input::real("x")], Limits::default()).is_err()); + assert!(Builder::new(Vec::new(), Limits::default()).is_err()); + } + + #[test] + fn limits_are_enforced() { + let limits = Limits { + max_window: 10, + max_nodes: 3, + max_depth: 3, + }; + let mut b = Builder::new(vec![Input::real("x")], limits).unwrap(); + let x = b.lookup("x").unwrap(); + assert!(b.call(Function::Sma, x, &[11.0]).is_err()); + let a = b.call(Function::Sma, x, &[10.0]).unwrap(); + let c = b.call(Function::Ema, a, &[10.0]).unwrap(); + let err = b.call(Function::Ema, c, &[10.0]).unwrap_err().to_string(); + assert!(err.contains("nested 4 deep"), "{err}"); + let err = b.call(Function::Ema, x, &[10.0]).unwrap_err().to_string(); + assert!(err.contains("more than 3 distinct nodes"), "{err}"); + } + + #[test] + fn a_pipeline_needs_an_output() { + let mut b = builder(); + let x = b.lookup("x").unwrap(); + b.bind("helper", x).unwrap(); + assert!(b.build().is_err()); + } + + #[test] + #[should_panic(expected = "a node of this builder")] + fn foreign_nodes_are_rejected() { + let mut other = builder(); + let x = other.lookup("x").unwrap(); + let foreign = other.call(Function::Sma, x, &[2.0]).unwrap(); + let _ = builder().output("a", foreign); + } +} diff --git a/src/pipeline/compiled.rs b/src/pipeline/compiled.rs new file mode 100644 index 0000000..a351a1b --- /dev/null +++ b/src/pipeline/compiled.rs @@ -0,0 +1,264 @@ +//! A compiled pipeline, its canonical text form and its use as a single +//! view. + +use std::{ + fmt, + str::FromStr, +}; + +use super::{ + Builder, + CompileError, + Diagnostic, + Domain, + Input, + Limits, + Severity, + analysis::Analysis, + builder::{ + Binding, + BindingKind, + }, + display::Printer, + graph::{ + Graph, + NodeId, + }, + parse::parse, + program::Program, +}; +use crate::View; + +/// Features computed from input series by one flat program. +/// +/// Every update feeds one value per input and runs each distinct node +/// once, in order, with no allocation, recursion or dynamic dispatch. An +/// output is `None` while not ready: until its windows fill, or while an +/// argument is outside what a function accepts (a division by zero, the +/// logarithm of a non-positive value). Ready outputs are always finite. +/// +/// Clone a warmed-up pipeline to branch it; [`Pipeline::reset`] starts it +/// over. +/// +/// The [`fmt::Display`] form is canonical: compiling it yields the same +/// pipeline, so two pipelines that print the same compute the same. +/// +/// ``` +/// use sliding_features::pipeline::Pipeline; +/// +/// let mut p: Pipeline = "zscore(ema(x, 10) - ema(x, 50), 200)".parse().unwrap(); +/// assert_eq!(p.to_string(), "zscore(ema(x, 10) - ema(x, 50), 200)"); +/// p.update(&[100.0]); +/// assert_eq!(p.output(0), None, "warming up"); +/// ``` +#[derive(Debug, Clone)] +pub struct Pipeline { + graph: Graph, + inputs: Vec, + bindings: Vec, + /// The domain and warm-up of each output. + output_domains: Vec, + output_warm_ups: Vec, + warnings: Vec, + program: Program, +} + +impl Pipeline { + pub(super) fn new( + graph: Graph, + inputs: Vec, + bindings: Vec, + analysis: &Analysis, + ) -> Self { + assert!(!inputs.is_empty(), "a pipeline has an input"); + let outputs = output_nodes(&bindings); + let program = Program::lower(&graph, &outputs); + let warnings = analysis.diagnostics().to_vec(); + assert!( + warnings.iter().all(|d| d.severity() == Severity::Warning), + "a pipeline with errors is rejected" + ); + Self { + output_domains: Vec::from_iter(outputs.iter().map(|o| analysis.domain(*o))), + output_warm_ups: Vec::from_iter(outputs.iter().map(|o| analysis.warm_up(*o))), + warnings, + graph, + inputs, + bindings, + program, + } + } + + /// Compile the text form of a pipeline over `inputs`, see + /// [`reference()`](super::reference()) for the language. + /// + /// The error lists every problem with its line and column, worded to + /// be fixed by the author, e.g. an LLM asked to retry. + pub fn compile(source: &str, inputs: Vec, limits: Limits) -> Result { + let builder = Builder::new(inputs, limits)?; + parse(source, builder)?.build() + } + + /// Feed one value per input, in the order of [`Pipeline::inputs`]. A + /// non-finite value counts as not ready. + /// + /// # Panics + /// + /// If the number of values differs from the number of inputs. + #[inline] + pub fn update(&mut self, inputs: &[f64]) { + assert_eq!(inputs.len(), self.inputs.len(), "one value per input"); + self.program.update(inputs); + } + + /// The value of the `i`-th output, `None` while not ready. + /// + /// # Panics + /// + /// If there is no `i`-th output. + #[inline] + pub fn output(&self, i: usize) -> Option { + self.program.output(i) + } + + /// The value of every output, in order. + pub fn outputs(&self) -> impl ExactSizeIterator> + '_ { + (0..self.output_count()).map(|i| self.output(i)) + } + + /// The number of outputs. + pub fn output_count(&self) -> usize { + self.output_bindings().count() + } + + /// The name of every output, in order. + pub fn output_names(&self) -> impl Iterator { + self.output_bindings().map(Binding::name) + } + + /// The index of the output called `name`. + pub fn output_index(&self, name: &str) -> Option { + self.output_names().position(|n| n == name) + } + + /// The values the `i`-th output can take, derived from the inputs' + /// domains before any data is seen. + pub fn output_domain(&self, i: usize) -> Domain { + self.output_domains[i] + } + + /// An estimate of the updates the `i`-th output needs before it is + /// ready: its window lengths summed along its longest chain. Use it to + /// size the history replayed before the output is read. + pub fn output_warm_up(&self, i: usize) -> usize { + self.output_warm_ups[i] + } + + /// The largest warm-up of any output, see [`Pipeline::output_warm_up`]. + pub fn warm_up(&self) -> usize { + self.output_warm_ups.iter().copied().max().unwrap_or(0) + } + + /// What the compiler found suspicious but allowed, e.g. a logarithm + /// of an argument that can be negative. + pub fn warnings(&self) -> &[Diagnostic] { + &self.warnings + } + + /// The inputs, in the order values are fed. + pub fn inputs(&self) -> &[Input] { + &self.inputs + } + + /// The number of operations an update runs: the distinct nodes the + /// outputs read. + pub fn op_count(&self) -> usize { + self.program.len() + } + + /// Start over: every output is not ready until fed again. + pub fn reset(&mut self) { + self.program = Program::lower(&self.graph, &output_nodes(&self.bindings)); + } + + fn output_bindings(&self) -> impl Iterator { + self.bindings.iter().filter(|b| b.is_output()) + } +} + +impl FromStr for Pipeline { + type Err = CompileError; + + /// Compile a pipeline over one input `x` that takes any finite value, + /// within the default [`Limits`]. + fn from_str(source: &str) -> Result { + Self::compile(source, vec![Input::real("x")], Limits::default()) + } +} + +impl fmt::Display for Pipeline { + /// One statement per line, in the order written. A node named by an + /// earlier statement is referred to by that name. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for (k, binding) in self.bindings.iter().enumerate() { + let name_of = |id: NodeId| { + self.bindings[..k] + .iter() + .find(|b| b.is_named() && b.node() == id) + .map(Binding::name) + }; + let expr = Printer::new(&self.graph, &self.inputs, name_of).expr(binding.node()); + if k > 0 { + f.write_str("\n")?; + } + match binding.kind() { + BindingKind::Helper => write!(f, "let {} = {expr}", binding.name())?, + BindingKind::Output => write!(f, "{} = {expr}", binding.name())?, + BindingKind::Anonymous => f.write_str(&expr)?, + } + } + Ok(()) + } +} + +fn output_nodes(bindings: &[Binding]) -> Vec { + Vec::from_iter(bindings.iter().filter(|b| b.is_output()).map(Binding::node)) +} + +/// A pipeline with one input and one output, used as a [`View`], e.g. to +/// chain it with other views. +#[derive(Debug, Clone)] +pub struct Feature(Pipeline); + +impl Feature { + /// The pipeline behind the view. + pub fn pipeline(&self) -> &Pipeline { + &self.0 + } +} + +impl TryFrom for Feature { + type Error = CompileError; + + fn try_from(pipeline: Pipeline) -> Result { + let (inputs, outputs) = (pipeline.inputs().len(), pipeline.output_count()); + if inputs != 1 || outputs != 1 { + return Err(CompileError::single(format!( + "a feature has one input and one output, the pipeline has {inputs} and {outputs}" + ))); + } + Ok(Self(pipeline)) + } +} + +impl View for Feature { + #[inline] + fn update(&mut self, val: f64) { + self.0.update(&[val]); + } + + #[inline] + fn last(&self) -> Option { + self.0.output(0) + } +} diff --git a/src/pipeline/display.rs b/src/pipeline/display.rs new file mode 100644 index 0000000..7af592e --- /dev/null +++ b/src/pipeline/display.rs @@ -0,0 +1,112 @@ +//! The text of a node: canonical, so equal texts mean equal computations. + +use std::fmt::Write as _; + +use super::{ + Input, + graph::{ + Graph, + Node, + NodeId, + }, +}; + +/// Writes the expression of nodes, referring to named nodes by name. +pub(super) struct Printer<'a, F> { + graph: &'a Graph, + inputs: &'a [Input], + /// The name to print instead of a node's expression, if any. + name_of: F, +} + +impl<'a, F> Printer<'a, F> +where + F: Fn(NodeId) -> Option<&'a str>, +{ + pub(super) fn new(graph: &'a Graph, inputs: &'a [Input], name_of: F) -> Self { + Self { + graph, + inputs, + name_of, + } + } + + /// The expression of `id`, with its children by name where they have + /// one. + pub(super) fn expr(&self, id: NodeId) -> String { + let mut out = String::new(); + self.write(&mut out, id, Context::Whole); + out + } + + /// The name of `id` if it has one, else its expression. + pub(super) fn name_or_expr(&self, id: NodeId) -> String { + let mut out = String::new(); + self.write_child(&mut out, id, Context::Whole); + out + } + + /// The name of `id` if it is an input. + pub(super) fn input_name(&self, id: NodeId) -> Option<&'a str> { + match self.graph.get(id) { + Node::Input(i) => Some(self.inputs[*i].name()), + _ => None, + } + } + + fn write_child(&self, out: &mut String, id: NodeId, context: Context) { + // An input keeps its own name even if a binding aliases it. + if matches!(self.graph.get(id), Node::Input(_)) { + return self.write(out, id, context); + } + match (self.name_of)(id) { + Some(name) => out.push_str(name), + None => self.write(out, id, context), + } + } + + fn write(&self, out: &mut String, id: NodeId, context: Context) { + match self.graph.get(id) { + Node::Input(i) => out.push_str(self.inputs[*i].name()), + Node::Const(c) => { + let _ = write!(out, "{c}"); + } + Node::Neg(a) => { + out.push('-'); + self.write_child(out, *a, Context::Operand); + } + Node::Binary(op, a, b) => { + // Every nested operation is parenthesized, so reading the + // text needs no precedence rules. + let parens = context == Context::Operand; + if parens { + out.push('('); + } + self.write_child(out, *a, Context::Operand); + let _ = write!(out, " {} ", op.symbol()); + self.write_child(out, *b, Context::Operand); + if parens { + out.push(')'); + } + } + Node::Call(function, a, params) => { + out.push_str(function.name()); + out.push('('); + self.write_child(out, *a, Context::Whole); + for p in params { + let _ = write!(out, ", {p}"); + } + out.push(')'); + } + } + } +} + +/// Where an expression is written. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Context { + /// On its own: a statement or a call argument. + Whole, + /// As the operand of an operator. + Operand, +} diff --git a/src/pipeline/domain.rs b/src/pipeline/domain.rs new file mode 100644 index 0000000..c377483 --- /dev/null +++ b/src/pipeline/domain.rs @@ -0,0 +1,534 @@ +//! The value range of a node, tracked by the compiler to find problems +//! before a pipeline sees any data. + +use std::fmt; + +/// One end of a [`Domain`]. +#[derive(Debug, Clone, Copy, PartialEq)] +struct Bound { + value: f64, + /// Whether `value` itself belongs to the domain. Infinite bounds are + /// always open, because a node only ever outputs finite values. + closed: bool, +} + +impl Bound { + fn new(value: f64, closed: bool) -> Self { + assert!(!value.is_nan(), "a bound is a number"); + Self { + // Adding zero turns -0 into 0, which prints as `0`. + value: value + 0.0, + closed: closed && value.is_finite(), + } + } +} + +/// A set of real numbers: an interval whose ends are open, closed or +/// infinite, e.g. `(0, inf)` or `[0, 100]`. +/// +/// The compiler derives the domain of every node from the domains of the +/// inputs, to report a function that can never be ready (e.g. `ln` of a +/// negative argument) or an output that is constant before any data is +/// seen, and to tell the range of each output. Domains over-approximate: +/// a node's values always lie inside its domain, but may not cover it. +/// +/// Arithmetic on domains follows real arithmetic, so rounding can step +/// past an open end (a product of two tiny positives can underflow to 0). +/// Functions that saturate in floating point, like `tanh` or `exp`, reach +/// their limits, so their domains include them. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Domain { + lo: Bound, + hi: Bound, +} + +impl Domain { + /// All finite real numbers, `(-inf, inf)`. + pub const REAL: Domain = Domain { + lo: Bound { + value: f64::NEG_INFINITY, + closed: false, + }, + hi: Bound { + value: f64::INFINITY, + closed: false, + }, + }; + + /// The domain between two bounds, `None` if it is empty. + fn new(lo: Bound, hi: Bound) -> Option { + let non_empty = lo.value < hi.value || (lo.value == hi.value && lo.closed && hi.closed); + non_empty.then_some(Self { lo, hi }) + } + + /// The domain between two bounds that rounding may have collapsed + /// onto one value: then that value, which over-approximates, or all + /// numbers if the value overflowed. + fn new_or_point(lo: Bound, hi: Bound) -> Self { + Self::new(lo, hi).unwrap_or_else(|| { + assert_eq!(lo.value, hi.value, "bounds are ordered"); + if lo.value.is_finite() { + Self::point(lo.value) + } else { + Self::REAL + } + }) + } + + /// The positive numbers, `(0, inf)`, e.g. of prices. + pub fn positive() -> Self { + Self { + lo: Bound::new(0.0, false), + hi: Self::REAL.hi, + } + } + + /// The non-negative numbers, `[0, inf)`, e.g. of volumes. + pub fn non_negative() -> Self { + Self { + lo: Bound::new(0.0, true), + hi: Self::REAL.hi, + } + } + + /// The numbers from `lo` to `hi`, both included (infinite ends are + /// open). + /// + /// # Panics + /// + /// If a bound is NaN or `lo > hi`. + pub fn closed(lo: f64, hi: f64) -> Self { + Self::new(Bound::new(lo, true), Bound::new(hi, true)) + .expect("the lower bound must not exceed the upper bound") + } + + /// The numbers strictly between `lo` and `hi`. + /// + /// # Panics + /// + /// If a bound is NaN or `lo >= hi`. + pub fn open(lo: f64, hi: f64) -> Self { + Self::new(Bound::new(lo, false), Bound::new(hi, false)) + .expect("the lower bound must be below the upper bound") + } + + /// The single number `value`. + /// + /// # Panics + /// + /// If `value` is not finite. + pub fn point(value: f64) -> Self { + assert!(value.is_finite(), "a point domain holds a finite value"); + Self::closed(value, value) + } + + /// The lower end, `-inf` when unbounded below. + pub fn lo(&self) -> f64 { + self.lo.value + } + + /// The upper end, `inf` when unbounded above. + pub fn hi(&self) -> f64 { + self.hi.value + } + + /// Whether the domain holds `value`. + pub fn contains(&self, value: f64) -> bool { + let above_lo = value > self.lo.value || (value == self.lo.value && self.lo.closed); + let below_hi = value < self.hi.value || (value == self.hi.value && self.hi.closed); + above_lo && below_hi + } + + /// The only value of a single-value domain, e.g. of a constant. + pub fn as_point(&self) -> Option { + (self.lo.value == self.hi.value).then_some(self.lo.value) + } + + /// The values in both domains, `None` if there are none. + pub fn intersect(self, other: Self) -> Option { + let lo = match self.lo.value.total_cmp(&other.lo.value) { + std::cmp::Ordering::Less => other.lo, + std::cmp::Ordering::Greater => self.lo, + std::cmp::Ordering::Equal => { + Bound::new(self.lo.value, self.lo.closed && other.lo.closed) + } + }; + let hi = match self.hi.value.total_cmp(&other.hi.value) { + std::cmp::Ordering::Less => self.hi, + std::cmp::Ordering::Greater => other.hi, + std::cmp::Ordering::Equal => { + Bound::new(self.hi.value, self.hi.closed && other.hi.closed) + } + }; + Self::new(lo, hi) + } + + /// `-a` for `a` in the domain. + pub(super) fn neg(self) -> Self { + Self { + lo: Bound::new(-self.hi.value, self.hi.closed), + hi: Bound::new(-self.lo.value, self.lo.closed), + } + } + + /// `a + b` for `a` in `self` and `b` in `other`. + pub(super) fn add(self, other: Self) -> Self { + // Lower ends are below `inf` and upper ends above `-inf`, so the + // sums are never `inf - inf`; an overflow becomes an open end. + let lo = Bound::new( + self.lo.value + other.lo.value, + self.lo.closed && other.lo.closed, + ); + let hi = Bound::new( + self.hi.value + other.hi.value, + self.hi.closed && other.hi.closed, + ); + Self::new_or_point(lo, hi) + } + + /// `a - b` for `a` in `self` and `b` in `other`. + pub(super) fn sub(self, other: Self) -> Self { + self.add(other.neg()) + } + + /// `a * b` for `a` in `self` and `b` in `other`. + /// + /// The extremes of a product lie at the corners, with `0 * inf = 0`. + pub(super) fn mul(self, other: Self) -> Self { + let corners = [ + product(self.lo, other.lo), + product(self.lo, other.hi), + product(self.hi, other.lo), + product(self.hi, other.hi), + ]; + let lo = extreme(corners, std::cmp::Ordering::Less); + let hi = extreme(corners, std::cmp::Ordering::Greater); + Self::new_or_point(lo, hi) + } + + /// `a / b` for `a` in `self` and `b` in `other`, or `None` when `b` can + /// be zero (the quotient is then unbounded). + pub(super) fn div(self, other: Self) -> Option { + if other.contains(0.0) { + return None; + } + // Zero is outside, so `other` lies on one side of it; a zero end is + // open and its reciprocal is the infinity on that side. + let positive = other.lo.value >= 0.0; + let recip = |b: Bound| { + let value = if b.value == 0.0 { + if positive { + f64::INFINITY + } else { + f64::NEG_INFINITY + } + } else { + 1.0 / b.value + }; + Bound::new(value, b.closed) + }; + let reciprocal = Self::new_or_point(recip(other.hi), recip(other.lo)); + Some(self.mul(reciprocal)) + } + + /// `|a|` for `a` in the domain. + pub(super) fn abs(self) -> Self { + if self.lo.value >= 0.0 { + return self; + } + if self.hi.value <= 0.0 { + return self.neg(); + } + let (neg, pos) = (-self.lo.value, self.hi.value); + let hi = match neg.total_cmp(&pos) { + std::cmp::Ordering::Less => self.hi, + std::cmp::Ordering::Greater => Bound::new(neg, self.lo.closed), + std::cmp::Ordering::Equal => Bound::new(pos, self.lo.closed || self.hi.closed), + }; + Self::new_or_point(Bound::new(0.0, true), hi) + } + + /// `f(a)` for `a` in the domain and a non-decreasing `f` that is + /// defined on all of it. + /// + /// Finite ends of the result are closed: in floating point `f` reaches + /// its limits (`tanh(20) == 1`, `exp(-800) == 0`). + pub(super) fn map_increasing(self, f: impl Fn(f64) -> f64) -> Self { + let (lo, hi) = (f(self.lo.value), f(self.hi.value)); + assert!(!lo.is_nan(), "f is defined on the domain"); + assert!(!hi.is_nan(), "f is defined on the domain"); + Self::new_or_point(Bound::new(lo, true), Bound::new(hi, true)) + } + + /// The sign of `a` for `a` in the domain: values in `{-1, 0, 1}`. + pub(super) fn sign(self) -> Self { + let sign = |b: Bound, open_zero: f64| { + if b.value > 0.0 { + 1.0 + } else if b.value < 0.0 { + -1.0 + } else if b.closed { + 0.0 + } else { + open_zero + } + }; + Self::closed(sign(self.lo, 1.0), sign(self.hi, -1.0)) + } + + /// `max(a, c)` for `a` in the domain. + pub(super) fn max_with(self, c: f64) -> Self { + assert!(c.is_finite(), "a clipping bound is finite"); + let lo = if c > self.lo.value { + Bound::new(c, true) + } else { + self.lo + }; + let hi = if c >= self.hi.value { + Bound::new(c, true) + } else { + self.hi + }; + Self::new_or_point(lo, hi) + } + + /// `min(a, c)` for `a` in the domain. + pub(super) fn min_with(self, c: f64) -> Self { + self.neg().max_with(-c).neg() + } +} + +/// A corner of a product of two domains, with `0 * inf = 0`: a zero end +/// that belongs to its domain makes the product zero whatever the other +/// factor is. +fn product(a: Bound, b: Bound) -> Bound { + let a_zero = a.value == 0.0; + let b_zero = b.value == 0.0; + let value = if a_zero || b_zero { + 0.0 + } else { + a.value * b.value + }; + let closed = (a.closed && b.closed) || (a_zero && a.closed) || (b_zero && b.closed); + Bound::new(value, closed) +} + +/// The smallest (`Less`) or largest (`Greater`) corner; it belongs to the +/// domain if any corner of that value does. +fn extreme(corners: [Bound; 4], order: std::cmp::Ordering) -> Bound { + corners + .into_iter() + .reduce(|best, c| match c.value.total_cmp(&best.value) { + std::cmp::Ordering::Equal => Bound::new(c.value, c.closed || best.closed), + o if o == order => c, + _ => best, + }) + .expect("four corners") +} + +impl fmt::Display for Domain { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let open = if self.lo.closed { '[' } else { '(' }; + let close = if self.hi.closed { ']' } else { ')' }; + write!(f, "{open}{}, {}{close}", self.lo.value, self.hi.value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display_and_constructors() { + assert_eq!(Domain::REAL.to_string(), "(-inf, inf)"); + assert_eq!(Domain::positive().to_string(), "(0, inf)"); + assert_eq!(Domain::non_negative().to_string(), "[0, inf)"); + assert_eq!(Domain::closed(0.0, 100.0).to_string(), "[0, 100]"); + assert_eq!(Domain::open(-1.0, 1.0).to_string(), "(-1, 1)"); + assert_eq!(Domain::point(2.5).as_point(), Some(2.5)); + assert_eq!(Domain::closed(1.0, f64::INFINITY).to_string(), "[1, inf)"); + assert_eq!(Domain::REAL.as_point(), None); + } + + #[test] + #[should_panic(expected = "must not exceed")] + fn reversed_bounds_panic() { + let _ = Domain::closed(1.0, 0.0); + } + + #[test] + #[should_panic(expected = "must be below")] + fn empty_open_domain_panics() { + let _ = Domain::open(1.0, 1.0); + } + + #[test] + fn contains_respects_open_and_closed_ends() { + let d = Domain::positive(); + assert!(!d.contains(0.0)); + assert!(d.contains(f64::MIN_POSITIVE)); + assert!(!d.contains(-1.0)); + assert!(Domain::non_negative().contains(0.0)); + assert!(!Domain::REAL.contains(f64::INFINITY)); + assert!(Domain::closed(1.0, 2.0).contains(2.0)); + } + + #[test] + fn intersection() { + let r = Domain::REAL.intersect(Domain::positive()); + assert_eq!(r, Some(Domain::positive())); + let neg = Domain::closed(f64::NEG_INFINITY, 0.0); + assert_eq!(neg.intersect(Domain::positive()), None); + assert_eq!( + neg.intersect(Domain::non_negative()), + Some(Domain::point(0.0)) + ); + assert_eq!( + Domain::closed(-1.0, 5.0).intersect(Domain::open(0.0, 10.0)), + Some(Domain::new(Bound::new(0.0, false), Bound::new(5.0, true)).unwrap()) + ); + } + + #[test] + fn arithmetic_examples() { + let pos = Domain::positive(); + assert_eq!(pos.mul(pos), pos); + assert_eq!(pos.add(pos), pos); + assert_eq!(pos.sub(pos), Domain::REAL); + assert_eq!(pos.neg().to_string(), "(-inf, 0)"); + assert_eq!( + Domain::closed(0.0, 5.0).mul(pos), + Domain::non_negative(), + "0 * inf is 0, and 0 is reached" + ); + assert_eq!( + Domain::closed(1.0, 2.0).mul(Domain::closed(-3.0, 4.0)), + Domain::closed(-6.0, 8.0) + ); + assert_eq!( + Domain::closed(1.0, 2.0).div(Domain::closed(-1.0, 1.0)), + None + ); + assert_eq!( + Domain::closed(1.0, 2.0).div(Domain::closed(2.0, 4.0)), + Some(Domain::closed(0.25, 1.0)) + ); + assert_eq!(Domain::point(1.0).div(pos), Some(pos)); + assert_eq!(Domain::point(1.0).div(pos.neg()), Some(pos.neg())); + assert_eq!(Domain::closed(-3.0, 2.0).abs(), Domain::closed(0.0, 3.0)); + assert_eq!(Domain::REAL.abs(), Domain::non_negative()); + assert_eq!(Domain::REAL.sign(), Domain::closed(-1.0, 1.0)); + assert_eq!(pos.sign(), Domain::point(1.0)); + assert_eq!( + Domain::REAL.map_increasing(f64::tanh), + Domain::closed(-1.0, 1.0) + ); + assert_eq!( + Domain::REAL.map_increasing(f64::exp), + Domain::non_negative(), + "exp underflows to 0" + ); + assert_eq!( + Domain::REAL.max_with(-1.0), + Domain::closed(-1.0, f64::INFINITY) + ); + assert_eq!(Domain::closed(0.0, 1.0).min_with(-2.0), Domain::point(-2.0)); + } + + #[test] + fn rounding_collapse_yields_a_point_not_an_empty_domain() { + let far = Domain::open(1000.0, 2000.0); + assert_eq!(far.map_increasing(f64::tanh), Domain::point(1.0)); + let huge = Domain::point(f64::MAX); + assert_eq!(huge.add(huge), Domain::REAL, "an overflow is unbounded"); + } + + /// Bounds that exercise zeros, infinities and open ends. + fn random_domain(rng: &romu::Rng) -> Domain { + const VALUES: [f64; 9] = [ + f64::NEG_INFINITY, + -10.0, + -1.0, + -0.5, + 0.0, + 0.5, + 1.0, + 10.0, + f64::INFINITY, + ]; + loop { + let pick = || VALUES[rng.range_usize(0..VALUES.len())]; + let (a, b) = (pick(), pick()); + let lo = Bound::new(a.min(b), rng.bool()); + let hi = Bound::new(a.max(b), rng.bool()); + if let Some(d) = Domain::new(lo, hi) { + return d; + } + } + } + + /// A value of the domain, often one of its closed ends. + fn random_member(d: Domain, rng: &romu::Rng) -> f64 { + loop { + let v = match rng.range_usize(0..4) { + 0 => d.lo(), + 1 => d.hi(), + _ => { + let lo = d.lo().max(-20.0); + let hi = d.hi().min(20.0); + lo + (hi - lo) * rng.f64() + } + }; + if d.contains(v) { + return v; + } + } + } + + /// Soundness: every result of an operation on members of domains is a + /// member of the operation's domain. + #[test] + fn operations_contain_every_result() { + let rng = romu::Rng::from_seed_with_64bit(3); + for _ in 0..20_000 { + let (da, db) = (random_domain(&rng), random_domain(&rng)); + let (a, b) = (random_member(da, &rng), random_member(db, &rng)); + let c = random_member(Domain::closed(-2.0, 2.0), &rng); + let check = |name: &str, d: Domain, v: f64| { + assert!( + d.contains(v), + "{name}: {v} not in {d} for {a} in {da}, {b} in {db}" + ); + }; + check("neg", da.neg(), -a); + check("add", da.add(db), a + b); + check("sub", da.sub(db), a - b); + check("mul", da.mul(db), a * b); + if let Some(q) = da.div(db) { + check("div", q, a / b); + } else { + assert!(db.contains(0.0)); + } + check("abs", da.abs(), a.abs()); + let sign = if a > 0.0 { + 1.0 + } else if a < 0.0 { + -1.0 + } else { + 0.0 + }; + check("sign", da.sign(), sign); + check("tanh", da.map_increasing(f64::tanh), a.tanh()); + check("exp", da.map_increasing(f64::exp), a.exp()); + check("max_with", da.max_with(c), a.max(c)); + check("min_with", da.min_with(c), a.min(c)); + if let Some(i) = da.intersect(db) { + if da.contains(b) { + check("intersect", i, b); + } + } else { + assert!(!da.contains(b), "{b} in {da} and {db}"); + } + } + } +} diff --git a/src/pipeline/error.rs b/src/pipeline/error.rs new file mode 100644 index 0000000..b670e11 --- /dev/null +++ b/src/pipeline/error.rs @@ -0,0 +1,189 @@ +//! What the compiler reports about a pipeline. + +use std::fmt; + +/// How bad a [`Diagnostic`] is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Severity { + /// The pipeline is rejected. + Error, + /// The pipeline compiles, but likely does not do what was meant. + Warning, +} + +/// Where in the text form a problem is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Location { + offset: usize, + line: usize, + column: usize, +} + +impl Location { + pub(super) fn new(offset: usize, line: usize, column: usize) -> Self { + assert!(line >= 1, "lines count from 1"); + assert!(column >= 1, "columns count from 1"); + Self { + offset, + line, + column, + } + } + + /// The byte offset in the text. + pub fn offset(&self) -> usize { + self.offset + } + + /// The line, counted from 1. + pub fn line(&self) -> usize { + self.line + } + + /// The column in characters, counted from 1. + pub fn column(&self) -> usize { + self.column + } +} + +/// One problem the compiler found, worded to be fixed by whoever wrote the +/// pipeline, a person or an LLM. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Diagnostic { + severity: Severity, + location: Option, + message: String, +} + +impl Diagnostic { + pub(super) fn error(message: impl Into) -> Self { + Self { + severity: Severity::Error, + location: None, + message: message.into(), + } + } + + pub(super) fn warning(message: impl Into) -> Self { + Self { + severity: Severity::Warning, + location: None, + message: message.into(), + } + } + + /// The diagnostic at `location`, unless it already has one. + pub(super) fn or_at(mut self, location: Option) -> Self { + self.location = self.location.or(location); + self + } + + /// How bad the problem is. + pub fn severity(&self) -> Severity { + self.severity + } + + /// Where the problem is in the text form, if the pipeline was written + /// as text. + pub fn location(&self) -> Option { + self.location + } + + /// What is wrong. + pub fn message(&self) -> &str { + &self.message + } +} + +impl fmt::Display for Diagnostic { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let severity = match self.severity { + Severity::Error => "error", + Severity::Warning => "warning", + }; + match self.location { + Some(at) => write!( + f, + "{severity} at {}:{}: {}", + at.line, at.column, self.message + ), + None => write!(f, "{severity}: {}", self.message), + } + } +} + +/// Why a pipeline was rejected: at least one error, and every warning +/// found on the way, one per line in the [`fmt::Display`] form. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompileError { + diagnostics: Vec, +} + +impl CompileError { + pub(super) fn new(diagnostics: Vec) -> Self { + assert!( + diagnostics.iter().any(|d| d.severity == Severity::Error), + "a compile error holds an error" + ); + Self { diagnostics } + } + + pub(super) fn single(message: impl Into) -> Self { + Self::new(vec![Diagnostic::error(message)]) + } + + pub(super) fn at(location: Location, message: impl Into) -> Self { + Self::new(vec![Diagnostic::error(message).or_at(Some(location))]) + } + + /// The error with `location` on every diagnostic that has none. + pub(super) fn or_at(self, location: Option) -> Self { + Self { + diagnostics: Vec::from_iter(self.diagnostics.into_iter().map(|d| d.or_at(location))), + } + } + + /// Every problem found, errors and warnings, in the order found. + pub fn diagnostics(&self) -> &[Diagnostic] { + &self.diagnostics + } +} + +impl fmt::Display for CompileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for (i, diagnostic) in self.diagnostics.iter().enumerate() { + if i > 0 { + f.write_str("\n")?; + } + write!(f, "{diagnostic}")?; + } + Ok(()) + } +} + +impl std::error::Error for CompileError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display_lists_every_diagnostic() { + let err = CompileError::new(vec![Diagnostic::error("a"), Diagnostic::error("b")]); + assert_eq!(err.to_string(), "error: a\nerror: b"); + assert_eq!(err.diagnostics().len(), 2); + let located = err.or_at(Some(Location::new(4, 2, 3))); + assert_eq!(located.to_string(), "error at 2:3: a\nerror at 2:3: b"); + let kept = located.or_at(Some(Location::new(0, 1, 1))); + assert_eq!( + kept.diagnostics()[0].location().map(|l| l.offset()), + Some(4) + ); + } + + #[test] + #[should_panic(expected = "holds an error")] + fn a_compile_error_needs_an_error() { + let _ = CompileError::new(Vec::new()); + } +} diff --git a/src/pipeline/function.rs b/src/pipeline/function.rs new file mode 100644 index 0000000..4341f47 --- /dev/null +++ b/src/pipeline/function.rs @@ -0,0 +1,756 @@ +//! The operators and functions of a pipeline: one table holds each +//! function's name, parameters, argument requirement, output range, runtime +//! form and description, so they cannot drift apart. + +use std::{ + fmt::Write as _, + num::NonZeroUsize, +}; + +use super::{ + Domain, + state::{ + Kind, + Pure, + }, +}; +use crate::{ + pure_functions::Echo, + rolling::{ + Drawdown, + LnReturn, + }, + sliding_windows::{ + Alma, + BinaryEntropy, + CenterOfGravity, + CorrelationTrendIndicator, + Cumulative, + CyberCycle, + EhlersFisherTransform, + Ema, + Lag, + LaguerreFilter, + LaguerreRSI, + MadScaler, + Max, + Min, + MinMaxNormalizer, + MyRSI, + NoiseEliminationTechnology, + PolarizedFractalEfficiency, + ReFlex, + Roc, + RoofingFilter, + Rsi, + Sma, + SuperSmoother, + TrendFlex, + Vst, + WelfordOnline, + ZScoreStandardization, + }, +}; + +/// An arithmetic operator between two expressions. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BinaryOp { + /// `a + b` + Add, + /// `a - b` + Sub, + /// `a * b` + Mul, + /// `a / b`, not ready while `b` is zero. + Div, +} + +impl BinaryOp { + /// The operator's symbol in the text form. + pub fn symbol(self) -> char { + match self { + BinaryOp::Add => '+', + BinaryOp::Sub => '-', + BinaryOp::Mul => '*', + BinaryOp::Div => '/', + } + } + + /// `a op b`. NaN (not ready) operands give NaN, and so does a division + /// by zero once the caller filters the infinity. + #[inline] + pub(super) fn eval(self, a: f64, b: f64) -> f64 { + match self { + BinaryOp::Add => a + b, + BinaryOp::Sub => a - b, + BinaryOp::Mul => a * b, + BinaryOp::Div => a / b, + } + } + + /// The domain of `a op b` for `a` in `lhs` and `b` in `rhs`, `None` + /// when the result is never finite (a division by a zero constant). + pub fn output_domain(self, lhs: Domain, rhs: Domain) -> Option { + Some(match self { + BinaryOp::Add => lhs.add(rhs), + BinaryOp::Sub => lhs.sub(rhs), + BinaryOp::Mul => lhs.mul(rhs), + BinaryOp::Div => match lhs.div(rhs) { + Some(d) => d, + // A divisor that is always zero is never valid; one that + // can be zero leaves the quotient unbounded. + None if rhs.as_point() == Some(0.0) => return None, + None => Domain::REAL, + }, + }) + } +} + +/// The kind of a parameter that follows the expression argument of a call. +/// Parameters are number literals, so they can be checked before the +/// pipeline runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Param { + /// A window length, counted in updates of the argument: an integer of + /// at least [`Function::min_window`]. + Window, + /// A finite real number. + Real, + /// A real number strictly between 0 and 1, e.g. a damping factor. + Fraction, +} + +impl Param { + /// The name of the parameter kind in signatures. + pub fn name(self) -> &'static str { + match self { + Param::Window => "window", + Param::Real => "real", + Param::Fraction => "fraction", + } + } +} + +/// What a function needs of its argument to update. While the argument +/// does not meet it, the call is not ready (or holds its last value). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Requirement { + /// Any finite value. + Any, + /// Values above zero, e.g. for logarithms. + Positive, + /// Values of at least zero, e.g. for square roots. + NonNegative, + /// Any finite value, but an argument that never changes sign makes the + /// output constant, e.g. the entropy of signs. + BothSigns, +} + +impl Requirement { + /// The values the requirement admits. + pub fn domain(self) -> Domain { + match self { + Requirement::Any | Requirement::BothSigns => Domain::REAL, + Requirement::Positive => Domain::positive(), + Requirement::NonNegative => Domain::non_negative(), + } + } + + fn describe(self) -> &'static str { + match self { + Requirement::Any => "any", + Requirement::Positive => "positive", + Requirement::NonNegative => "non-negative", + Requirement::BothSigns => "changes sign", + } + } +} + +/// The checked parameters of a call. +#[derive(Debug, Clone, Copy)] +pub(super) struct Params<'a>(&'a [f64]); + +impl<'a> Params<'a> { + /// Wrap parameters that passed [`Function::check_params`]. + pub(super) fn new(function: Function, params: &'a [f64]) -> Self { + assert_eq!(params.len(), function.params().len(), "checked parameters"); + Self(params) + } + + /// The `i`-th parameter as a window length. + pub(super) fn window(self, i: usize) -> NonZeroUsize { + window_len(self.0[i]).expect("checked window") + } + + /// The `i`-th parameter as a real number. + pub(super) fn real(self, i: usize) -> f64 { + let value = self.0[i]; + assert!(value.is_finite(), "checked parameter"); + value + } +} + +/// The window length `value` denotes: an integer from 1 to 2^53, where +/// every integer is exact as `f64`. +pub(super) fn window_len(value: f64) -> Option { + const MAX_EXACT: f64 = 9_007_199_254_740_992.0; + if value.fract() != 0.0 || !(1.0..=MAX_EXACT).contains(&value) { + return None; + } + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "an integer in [1, 2^53] is exact as usize on 64-bit targets" + )] + let len = value as usize; + NonZeroUsize::new(len) +} + +/// The output domain of a function given its (admitted) argument domain. +type RangeFn = fn(Domain, Params<'_>) -> Domain; + +/// What a call becomes at runtime given its checked parameters. +type BuildFn = fn(Params<'_>) -> Kind; + +/// Declares [`Function`] and its metadata from one table. +macro_rules! functions { + ($($variant:ident { + name: $name:literal, + params: [$($param:ident),*], + min_window: $min:literal, + requires: $requires:ident, + range: $range:expr, + build: $build:expr, + doc: $doc:literal $(,)? + })*) => { + /// A function of a pipeline. Every call takes an expression as its + /// first argument, then its [`Function::params`] as number literals. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub enum Function { + $(#[doc = $doc] $variant,)* + } + + impl Function { + /// Every function, in declaration order. + pub const ALL: &'static [Function] = &[$(Function::$variant,)*]; + + /// The name the function is called by. + pub fn name(self) -> &'static str { + match self { $(Function::$variant => $name,)* } + } + + /// The parameters after the expression argument. + pub fn params(self) -> &'static [Param] { + match self { $(Function::$variant => &[$(Param::$param),*],)* } + } + + /// The smallest window length the function accepts: shorter + /// windows are degenerate (e.g. an RSI over one value). + pub fn min_window(self) -> usize { + match self { $(Function::$variant => $min,)* } + } + + /// What the function needs of its argument. + pub fn requires(self) -> Requirement { + match self { $(Function::$variant => Requirement::$requires,)* } + } + + /// What the function computes, one line. + pub fn description(self) -> &'static str { + match self { $(Function::$variant => $doc,)* } + } + + fn range(self, arg: Domain, params: Params<'_>) -> Domain { + let range: RangeFn = match self { $(Function::$variant => $range,)* }; + range(arg, params) + } + + /// What a call with checked `params` becomes at runtime. + pub(super) fn instantiate(self, params: Params<'_>) -> Kind { + let build: BuildFn = match self { $(Function::$variant => $build,)* }; + build(params) + } + } + }; +} + +functions! { + Abs { + name: "abs", params: [], min_window: 1, requires: Any, + range: |a, _| a.abs(), + build: |_| Kind::Pure(Pure::Abs), + doc: "Absolute value.", + } + Sign { + name: "sign", params: [], min_window: 1, requires: Any, + range: |a, _| a.sign(), + build: |_| Kind::Pure(Pure::Sign), + doc: "Sign: -1, 0 or 1.", + } + Sqrt { + name: "sqrt", params: [], min_window: 1, requires: NonNegative, + range: |a, _| a.map_increasing(f64::sqrt), + build: |_| Kind::Pure(Pure::Sqrt), + doc: "Square root.", + } + Square { + name: "square", params: [], min_window: 1, requires: Any, + range: |a, _| a.abs().map_increasing(|v| v * v), + build: |_| Kind::Pure(Pure::Square), + doc: "The argument times itself.", + } + Exp { + name: "exp", params: [], min_window: 1, requires: Any, + range: |a, _| a.map_increasing(f64::exp), + build: |_| Kind::Pure(Pure::Exp), + doc: "Exponential function; not ready where it overflows.", + } + Ln { + name: "ln", params: [], min_window: 1, requires: Positive, + range: |a, _| a.map_increasing(f64::ln), + build: |_| Kind::Pure(Pure::Ln), + doc: "Natural logarithm.", + } + Tanh { + name: "tanh", params: [], min_window: 1, requires: Any, + range: |a, _| a.map_increasing(f64::tanh), + build: |_| Kind::Pure(Pure::Tanh), + doc: "Hyperbolic tangent, squashes into (-1, 1).", + } + Sigmoid { + name: "sigmoid", params: [], min_window: 1, requires: Any, + range: |a, _| a.map_increasing(sigmoid), + build: |_| Kind::Pure(Pure::Sigmoid), + doc: "Logistic function 1 / (1 + exp(-a)), squashes into (0, 1).", + } + Asinh { + name: "asinh", params: [], min_window: 1, requires: Any, + range: |a, _| a.map_increasing(f64::asinh), + build: |_| Kind::Pure(Pure::Asinh), + doc: "Inverse hyperbolic sine: like a signed logarithm, compresses large magnitudes.", + } + Gte { + name: "gte", params: [Real], min_window: 1, requires: Any, + range: |a, p| a.max_with(p.real(0)), + build: |p| Kind::Pure(Pure::Gte(p.real(0))), + doc: "Clips values below the real to it: max(a, real).", + } + Lte { + name: "lte", params: [Real], min_window: 1, requires: Any, + range: |a, p| a.min_with(p.real(0)), + build: |p| Kind::Pure(Pure::Lte(p.real(0))), + doc: "Clips values above the real to it: min(a, real).", + } + LnReturn { + name: "ln_return", params: [], min_window: 1, requires: Positive, + range: |_, _| Domain::REAL, + build: |_| LnReturn::new(Echo::new()).into(), + doc: "Log return between consecutive updates.", + } + Drawdown { + name: "drawdown", params: [], min_window: 1, requires: Positive, + range: |_, _| Domain::closed(0.0, 1.0), + build: |_| Drawdown::new(Echo::new()).into(), + doc: "Largest relative peak-to-valley decline so far, in [0, 1].", + } + Sma { + name: "sma", params: [Window], min_window: 1, requires: Any, + range: |a, _| a, + build: |p| Sma::new(Echo::new(), p.window(0)).into(), + doc: "Simple moving average.", + } + Ema { + name: "ema", params: [Window], min_window: 1, requires: Any, + range: |a, _| a, + build: |p| Ema::new(Echo::new(), p.window(0)).into(), + doc: "Exponential moving average, alpha = 2 / (window + 1).", + } + Alma { + name: "alma", params: [Window], min_window: 2, requires: Any, + range: |a, _| a, + build: |p| Alma::new(Echo::new(), p.window(0)).into(), + doc: "Arnaud Legoux moving average (sigma 6, offset 0.85).", + } + Roc { + name: "roc", params: [Window], min_window: 1, requires: Any, + range: |_, _| Domain::REAL, + build: |p| Roc::new(Echo::new(), p.window(0)).into(), + doc: "Rate of change in percent against the value `window` updates ago.", + } + Lag { + name: "lag", params: [Window], min_window: 1, requires: Any, + range: |a, _| a, + build: |p| Lag::new(Echo::new(), p.window(0)).into(), + doc: "The value `window` updates ago.", + } + Min { + name: "min", params: [Window], min_window: 1, requires: Any, + range: |a, _| a, + build: |p| Min::new(Echo::new(), p.window(0)).into(), + doc: "Minimum over the window.", + } + Max { + name: "max", params: [Window], min_window: 1, requires: Any, + range: |a, _| a, + build: |p| Max::new(Echo::new(), p.window(0)).into(), + doc: "Maximum over the window.", + } + MinMax { + name: "min_max", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::closed(-1.0, 1.0), + build: |p| MinMaxNormalizer::new(Echo::new(), p.window(0)).into(), + doc: "Position in the [min, max] range of the previous window, scaled to [-1, 1]; new extremes give -1 or 1.", + } + Mad { + name: "mad", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| MadScaler::new(Echo::new(), p.window(0)).into(), + doc: "Robust z-score: (a - median) / (1.4826 MAD) of the previous window.", + } + ZScore { + name: "zscore", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| ZScoreStandardization::new(Echo::new(), p.window(0)).into(), + doc: "Z-score against the mean and standard deviation of the previous window.", + } + Std { + name: "std", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::non_negative(), + build: |p| WelfordOnline::new(Echo::new(), p.window(0)).into(), + doc: "Standard deviation over the window (Welford).", + } + Cumulative { + name: "cumulative", params: [Window], min_window: 1, requires: Any, + range: |a, p| a.mul(Domain::closed(1.0, window_f64(p.window(0)))), + build: |p| Cumulative::new(Echo::new(), p.window(0)).into(), + doc: "Sum over the window.", + } + Rsi { + name: "rsi", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::closed(0.0, 100.0), + build: |p| Rsi::new(Echo::new(), p.window(0)).into(), + doc: "Relative strength index, in [0, 100].", + } + MyRsi { + name: "my_rsi", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::closed(-1.0, 1.0), + build: |p| MyRSI::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' MyRSI, in [-1, 1].", + } + LaguerreRsi { + name: "laguerre_rsi", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::closed(0.0, 1.0), + build: |p| LaguerreRSI::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' Laguerre RSI, in [0, 1].", + } + LaguerreFilter { + name: "laguerre_filter", params: [Fraction], min_window: 1, requires: Any, + range: |_, _| Domain::REAL, + build: |p| LaguerreFilter::new(Echo::new(), p.real(0)).into(), + doc: "John Ehlers' Laguerre filter with damping gamma (the fraction).", + } + SuperSmoother { + name: "super_smoother", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| SuperSmoother::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' two-pole super smoother low-pass filter.", + } + RoofingFilter { + name: "roofing_filter", params: [Window, Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| RoofingFilter::new(Echo::new(), p.window(0), p.window(1)).into(), + doc: "John Ehlers' roofing filter (low-pass window, high-pass window): a band-pass.", + } + ReFlex { + name: "re_flex", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| ReFlex::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' ReFlex zero-lag cycle indicator.", + } + TrendFlex { + name: "trend_flex", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| TrendFlex::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' TrendFlex zero-lag trend indicator.", + } + CyberCycle { + name: "cyber_cycle", params: [Window], min_window: 3, requires: Any, + range: |_, _| Domain::REAL, + build: |p| CyberCycle::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' cyber cycle.", + } + CenterOfGravity { + name: "center_of_gravity", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| CenterOfGravity::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' center of gravity oscillator.", + } + Cti { + name: "cti", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::closed(-1.0, 1.0), + build: |p| CorrelationTrendIndicator::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' correlation trend indicator: correlation with a straight line, in [-1, 1].", + } + Pfe { + name: "pfe", params: [Window, Window], min_window: 3, requires: Any, + range: |_, _| Domain::closed(-1.0, 1.0), + build: |p| PolarizedFractalEfficiency::new(Echo::new(), ema(p.window(1)), p.window(0)).into(), + doc: "Polarized fractal efficiency (window, EMA smoothing window), in [-1, 1]: near 1 in a straight up-trend, near -1 down.", + } + Fisher { + name: "fisher", params: [Window, Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| EhlersFisherTransform::new(Echo::new(), ema(p.window(1)), p.window(0)).into(), + doc: "John Ehlers' Fisher transform (window, EMA smoothing window).", + } + BinaryEntropy { + name: "binary_entropy", params: [Window], min_window: 2, requires: BothSigns, + range: |_, _| Domain::closed(0.0, 1.0), + build: |p| BinaryEntropy::new(Echo::new(), p.window(0)).into(), + doc: "Shannon entropy of the signs over the window, in [0, 1].", + } + Vst { + name: "vst", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::REAL, + build: |p| Vst::new(Echo::new(), p.window(0)).into(), + doc: "Variance stabilizing transform: change scaled by the rolling standard deviation.", + } + Net { + name: "net", params: [Window], min_window: 2, requires: Any, + range: |_, _| Domain::closed(-1.0, 1.0), + build: |p| NoiseEliminationTechnology::new(Echo::new(), p.window(0)).into(), + doc: "John Ehlers' noise elimination technology (Kendall correlation), in [-1, 1].", + } +} + +/// The EMA that smooths `pfe` and `fisher`. +fn ema(window: NonZeroUsize) -> Ema> { + Ema::new(Echo::new(), window) +} + +/// The logistic function. +pub(super) fn sigmoid(v: f64) -> f64 { + 1.0 / (1.0 + (-v).exp()) +} + +/// A window length as `f64`, exact for checked windows (at most 2^53). +fn window_f64(window: NonZeroUsize) -> f64 { + #[allow( + clippy::cast_precision_loss, + reason = "checked windows are at most 2^53, exact as f64" + )] + let value = window.get() as f64; + value +} + +impl Function { + /// The function called `name`, if any. + pub fn from_name(name: &str) -> Option { + Self::ALL.iter().copied().find(|f| f.name() == name) + } + + /// The call signature, e.g. `sma(a, window)`. + pub fn signature(self) -> String { + let mut out = format!("{}(a", self.name()); + for param in self.params() { + let _ = write!(out, ", {}", param.name()); + } + out.push(')'); + out + } + + /// Check parameters against the signature and `max_window`; the error + /// says what is wrong. + pub fn check_params(self, params: &[f64], max_window: usize) -> Result<(), String> { + let expected = self.params(); + if params.len() != expected.len() { + return Err(format!( + "{} takes {} parameter(s) after its argument, got {}", + self.signature(), + expected.len(), + params.len() + )); + } + for (kind, value) in expected.iter().zip(params) { + let valid = match kind { + Param::Window => window_len(*value) + .is_some_and(|w| w.get() >= self.min_window() && w.get() <= max_window), + Param::Real => value.is_finite(), + Param::Fraction => *value > 0.0 && *value < 1.0, + }; + if !valid { + let expected = match kind { + Param::Window => { + format!("an integer in [{}, {max_window}]", self.min_window()) + } + Param::Real => "a finite number".to_string(), + Param::Fraction => "a number strictly between 0 and 1".to_string(), + }; + return Err(format!( + "{}: the {} parameter {value} must be {expected}", + self.signature(), + kind.name() + )); + } + } + Ok(()) + } + + /// The domain of the call's output for an argument in `arg`, `None` if + /// no value of `arg` meets [`Function::requires`] (the call is never + /// ready). + /// + /// # Panics + /// + /// If `params` do not pass [`Function::check_params`]. + pub fn output_domain(self, arg: Domain, params: &[f64]) -> Option { + let admitted = arg.intersect(self.requires().domain())?; + Some(self.range(admitted, Params::new(self, params))) + } + + /// An estimate of the admitted updates of its argument a call needs + /// before it is ready: the sum of its windows (a log return needs one + /// more value). Pipelines sum it along their longest chain. + /// + /// # Panics + /// + /// If `params` do not pass [`Function::check_params`]. + pub fn warm_up(self, params: &[f64]) -> usize { + let params = Params::new(self, params); + let windows: usize = self + .params() + .iter() + .enumerate() + .filter(|(_, kind)| **kind == Param::Window) + .map(|(i, _)| params.window(i).get()) + .sum(); + windows + usize::from(self == Function::LnReturn) + } + + /// A markdown table of every function: signature, what the argument + /// must be, and description. Meant for documentation and for prompting + /// an agent that writes pipelines. + pub fn reference() -> String { + let mut out = String::from("| function | argument | description |\n|---|---|---|\n"); + for function in Self::ALL { + let _ = writeln!( + out, + "| `{}` | {} | {} |", + function.signature(), + function.requires().describe(), + function.description() + ); + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_are_unique_and_round_trip() { + for function in Function::ALL { + assert_eq!(Function::from_name(function.name()), Some(*function)); + assert!(function.min_window() >= 1); + } + let mut names = Vec::from_iter(Function::ALL.iter().map(|f| f.name())); + names.sort_unstable(); + names.dedup(); + assert_eq!(names.len(), Function::ALL.len()); + assert_eq!(Function::from_name("nope"), None); + } + + #[test] + fn the_reference_lists_every_signature() { + let reference = Function::reference(); + assert!(reference.contains("| `pfe(a, window, window)` | any |")); + assert!(reference.contains("| `ln(a)` | positive |")); + assert!(reference.contains("| `laguerre_filter(a, fraction)` |")); + assert_eq!(reference.lines().count(), Function::ALL.len() + 2); + } + + #[test] + fn window_lengths() { + assert_eq!(window_len(3.0).map(NonZeroUsize::get), Some(3)); + for bad in [0.0, -1.0, 2.5, f64::NAN, f64::INFINITY, 1e300] { + assert_eq!(window_len(bad), None, "{bad}"); + } + } + + #[test] + fn parameter_checks() { + assert!(Function::Pfe.check_params(&[3.0, 3.0], 100).is_ok()); + for (function, params) in [ + (Function::Sma, &[0.0][..]), + (Function::Sma, &[2.5]), + (Function::Sma, &[]), + (Function::Sma, &[101.0]), + (Function::Pfe, &[2.0, 3.0]), + (Function::LaguerreFilter, &[1.0]), + (Function::LaguerreFilter, &[0.0]), + (Function::Gte, &[f64::NAN]), + (Function::Abs, &[1.0]), + ] { + let err = function.check_params(params, 100).expect_err("invalid"); + assert!(err.starts_with(&function.signature()), "{err}"); + } + let err = Function::Sma.check_params(&[0.0], 100).unwrap_err(); + assert_eq!( + err, + "sma(a, window): the window parameter 0 must be an integer in [1, 100]" + ); + } + + #[test] + fn output_domains() { + let real = Domain::REAL; + let pos = Domain::positive(); + assert_eq!(Function::Ln.output_domain(pos, &[]), Some(real)); + assert_eq!(Function::Ln.output_domain(real, &[]), Some(real)); + assert_eq!(Function::Ln.output_domain(pos.neg(), &[]), None); + assert_eq!( + Function::Sqrt.output_domain(real, &[]), + Some(Domain::non_negative()) + ); + assert_eq!( + Function::Sma.output_domain(Domain::closed(1.0, 2.0), &[5.0]), + Some(Domain::closed(1.0, 2.0)) + ); + assert_eq!( + Function::Cumulative.output_domain(Domain::closed(1.0, 2.0), &[5.0]), + Some(Domain::closed(1.0, 10.0)) + ); + assert_eq!( + Function::Gte.output_domain(real, &[0.0]), + Some(Domain::non_negative()) + ); + assert_eq!( + Function::Sigmoid.output_domain(real, &[]), + Some(Domain::closed(0.0, 1.0)), + "saturates in floating point" + ); + assert_eq!( + Function::Rsi.output_domain(real, &[14.0]), + Some(Domain::closed(0.0, 100.0)) + ); + } + + #[test] + fn binary_op_domains() { + let pos = Domain::positive(); + assert_eq!(BinaryOp::Add.output_domain(pos, pos), Some(pos)); + assert_eq!(BinaryOp::Div.output_domain(pos, pos), Some(pos)); + assert_eq!( + BinaryOp::Div.output_domain(pos, Domain::REAL), + Some(Domain::REAL) + ); + assert_eq!( + BinaryOp::Div.output_domain(pos, Domain::point(0.0)), + None, + "dividing by zero is never finite" + ); + assert_eq!(BinaryOp::Sub.symbol(), '-'); + } +} diff --git a/src/pipeline/graph.rs b/src/pipeline/graph.rs new file mode 100644 index 0000000..829c097 --- /dev/null +++ b/src/pipeline/graph.rs @@ -0,0 +1,183 @@ +//! A pipeline as a graph: nodes in creation order, so children come before +//! their parents, and structurally equal nodes shared. + +use std::collections::HashMap; + +use super::{ + BinaryOp, + Function, +}; + +/// A node of a pipeline under construction, see +/// [`Builder`](super::Builder). Only valid with the builder that made it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct NodeId(u32); + +impl NodeId { + pub(super) fn index(self) -> usize { + usize::try_from(self.0).expect("u32 fits in usize on supported targets") + } +} + +/// One computation of a pipeline. +#[derive(Debug, Clone, PartialEq)] +pub(super) enum Node { + /// The input series with this index. + Input(usize), + /// A finite constant. + Const(f64), + /// `-a` + Neg(NodeId), + /// `a op b` + Binary(BinaryOp, NodeId, NodeId), + /// A function applied to a node, with checked parameters. + Call(Function, NodeId, Box<[f64]>), +} + +impl Node { + /// The nodes this one reads, left to right. + pub(super) fn children(&self) -> impl Iterator + use<> { + let (a, b) = match self { + Node::Input(_) | Node::Const(_) => (None, None), + Node::Neg(a) | Node::Call(_, a, _) => (Some(*a), None), + Node::Binary(_, a, b) => (Some(*a), Some(*b)), + }; + a.into_iter().chain(b) + } + + fn key(&self) -> Key { + match self { + Node::Input(i) => Key::Input(*i), + Node::Const(c) => Key::Const(c.to_bits()), + Node::Neg(a) => Key::Neg(*a), + Node::Binary(op, a, b) => Key::Binary(*op, *a, *b), + Node::Call(function, a, params) => { + Key::Call(*function, *a, params.iter().map(|p| p.to_bits()).collect()) + } + } + } +} + +/// A [`Node`] with its numbers as bits, so it can be hashed: equal keys +/// compute the same values, so one node serves both. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +enum Key { + Input(usize), + Const(u64), + Neg(NodeId), + Binary(BinaryOp, NodeId, NodeId), + Call(Function, NodeId, Box<[u64]>), +} + +/// The nodes of a pipeline, each computation once. +#[derive(Debug, Clone, Default)] +pub(super) struct Graph { + nodes: Vec, + index: HashMap, +} + +impl Graph { + /// The id of `node`: the existing equal node, or `node` added. `None` + /// if adding it would exceed `max_nodes`. + pub(super) fn intern(&mut self, node: Node, max_nodes: usize) -> Option { + for child in node.children() { + assert!(child.index() < self.nodes.len(), "a node of this graph"); + } + if let Node::Const(c) = node { + assert!(c.is_finite(), "constants are finite"); + } + let key = node.key(); + if let Some(id) = self.index.get(&key) { + return Some(*id); + } + if self.nodes.len() >= max_nodes { + return None; + } + let id = NodeId(u32::try_from(self.nodes.len()).ok()?); + self.nodes.push(node); + self.index.insert(key, id); + assert_eq!(self.nodes.len(), self.index.len(), "every node is indexed"); + Some(id) + } + + /// The node `id`. + pub(super) fn get(&self, id: NodeId) -> &Node { + &self.nodes[id.index()] + } + + /// The number of nodes. + pub(super) fn len(&self) -> usize { + self.nodes.len() + } + + /// Every node with its id, children before parents. + pub(super) fn iter(&self) -> impl Iterator { + self.nodes + .iter() + .enumerate() + .map(|(i, node)| (NodeId(u32::try_from(i).expect("ids fit u32")), node)) + } + + /// Which nodes the `roots` read, directly or not, roots included. + pub(super) fn reachable(&self, roots: impl IntoIterator) -> Vec { + let mut reachable = vec![false; self.nodes.len()]; + for root in roots { + reachable[root.index()] = true; + } + // Parents come after their children, so one backward pass suffices. + for i in (0..self.nodes.len()).rev() { + if reachable[i] { + for child in self.nodes[i].children() { + assert!(child.index() < i, "children come first"); + reachable[child.index()] = true; + } + } + } + reachable + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn equal_nodes_are_shared() { + let mut g = Graph::default(); + let x = g.intern(Node::Input(0), 10).unwrap(); + let a = g.intern(Node::Call(Function::Sma, x, [3.0].into()), 10); + let b = g.intern(Node::Call(Function::Sma, x, [3.0].into()), 10); + let c = g.intern(Node::Call(Function::Sma, x, [4.0].into()), 10); + assert_eq!(a, b); + assert_ne!(a, c); + assert_eq!(g.len(), 3); + } + + #[test] + fn the_node_limit_counts_distinct_nodes() { + let mut g = Graph::default(); + let x = g.intern(Node::Input(0), 2).unwrap(); + let neg = g.intern(Node::Neg(x), 2).unwrap(); + assert_eq!(g.intern(Node::Neg(x), 2), Some(neg), "shared: no new node"); + assert_eq!(g.intern(Node::Neg(neg), 2), None); + } + + #[test] + fn reachability() { + let mut g = Graph::default(); + let x = g.intern(Node::Input(0), 10).unwrap(); + let y = g.intern(Node::Input(1), 10).unwrap(); + let neg = g.intern(Node::Neg(x), 10).unwrap(); + let sum = g.intern(Node::Binary(BinaryOp::Add, neg, x), 10).unwrap(); + assert_eq!(g.reachable([sum]), vec![true, false, true, true]); + assert_eq!(g.reachable([y]), vec![false, true, false, false]); + assert_eq!(Vec::from_iter(g.get(sum).children()), vec![neg, x]); + } + + #[test] + #[should_panic(expected = "a node of this graph")] + fn foreign_nodes_are_rejected() { + let mut g = Graph::default(); + let _ = g.intern(Node::Neg(NodeId(5)), 10); + } +} diff --git a/src/pipeline/lex.rs b/src/pipeline/lex.rs new file mode 100644 index 0000000..dcfdf2c --- /dev/null +++ b/src/pipeline/lex.rs @@ -0,0 +1,236 @@ +//! Splits the text form of a pipeline into tokens with their locations. + +use super::{ + CompileError, + Location, +}; + +#[derive(Debug, Clone, PartialEq)] +pub(super) enum Token { + Number(f64), + Name(String), + Open, + Close, + Comma, + /// `+ - * /` + Op(char), + Assign, + /// The end of a statement: `;` or a line break. + Sep, + End, +} + +impl Token { + /// The token as an error message names it. + pub(super) fn describe(&self) -> String { + match self { + Token::Number(v) => format!("number `{v}`"), + Token::Name(n) => format!("`{n}`"), + Token::Open => "`(`".to_string(), + Token::Close => "`)`".to_string(), + Token::Comma => "`,`".to_string(), + Token::Op(c) => format!("`{c}`"), + Token::Assign => "`=`".to_string(), + Token::Sep => "the end of the line".to_string(), + Token::End => "the end".to_string(), + } + } + + /// Whether a line break after the token continues the statement, as + /// after an operator or inside a call. + fn continues(&self) -> bool { + matches!( + self, + Token::Op(_) | Token::Comma | Token::Assign | Token::Open | Token::Sep + ) + } +} + +/// A token and where it starts. +pub(super) type Spanned = (Token, Location); + +/// Tracks the location while walking the source. +struct Cursor<'a> { + source: &'a str, + offset: usize, + line: usize, + column: usize, +} + +impl Cursor<'_> { + fn location(&self) -> Location { + Location::new(self.offset, self.line, self.column) + } + + fn peek(&self) -> Option { + self.source[self.offset..].chars().next() + } + + fn bump(&mut self) { + let c = self.peek().expect("not at the end"); + self.offset += c.len_utf8(); + if c == '\n' { + self.line += 1; + self.column = 1; + } else { + self.column += 1; + } + } + + fn bump_while(&mut self, pred: impl Fn(char) -> bool) -> &str { + let start = self.offset; + while self.peek().is_some_and(&pred) { + self.bump(); + } + &self.source[start..self.offset] + } +} + +/// The tokens of `source`. Line breaks end a statement only outside +/// parentheses and after a complete expression, so long expressions can +/// span lines. +pub(super) fn lex(source: &str) -> Result, CompileError> { + let mut cursor = Cursor { + source, + offset: 0, + line: 1, + column: 1, + }; + let mut tokens: Vec = Vec::new(); + let mut depth = 0_usize; + while let Some(c) = cursor.peek() { + let at = cursor.location(); + let token = match c { + '#' => { + cursor.bump_while(|c| c != '\n'); + continue; + } + '\n' | ';' => { + cursor.bump(); + let continued = tokens.last().is_none_or(|(t, _)| t.continues()); + if depth > 0 || (c == '\n' && continued) { + continue; + } + Token::Sep + } + c if c.is_whitespace() => { + cursor.bump(); + continue; + } + '(' | ')' | ',' | '+' | '-' | '*' | '/' | '=' => { + cursor.bump(); + match c { + '(' => { + depth += 1; + Token::Open + } + ')' => { + depth = depth.saturating_sub(1); + Token::Close + } + ',' => Token::Comma, + '=' => Token::Assign, + op => Token::Op(op), + } + } + '0'..='9' | '.' => number(&mut cursor, at)?, + c if c.is_ascii_alphabetic() || c == '_' => { + let name = cursor.bump_while(|c| c.is_ascii_alphanumeric() || c == '_'); + Token::Name(name.to_string()) + } + other => { + return Err(CompileError::at(at, format!("unexpected `{other}`"))); + } + }; + tokens.push((token, at)); + } + tokens.push((Token::End, cursor.location())); + Ok(tokens) +} + +/// A number: digits with an optional point, then an optional exponent. +fn number(cursor: &mut Cursor<'_>, at: Location) -> Result { + let start = cursor.offset; + cursor.bump_while(|c| c.is_ascii_digit() || c == '.'); + let rest = &cursor.source[cursor.offset..]; + let mut exponent = rest.chars(); + if matches!(exponent.next(), Some('e' | 'E')) { + let sign = usize::from(matches!(rest[1..].chars().next(), Some('+' | '-'))); + if rest[1 + sign..].starts_with(|c: char| c.is_ascii_digit()) { + for _ in 0..=sign { + cursor.bump(); + } + cursor.bump_while(|c| c.is_ascii_digit()); + } + } + let text = &cursor.source[start..cursor.offset]; + text.parse::() + .map(Token::Number) + .map_err(|_| CompileError::at(at, format!("invalid number `{text}`"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tokens(source: &str) -> Vec { + Vec::from_iter(lex(source).unwrap().into_iter().map(|(t, _)| t)) + } + + #[test] + fn numbers() { + assert_eq!( + tokens("1 2.5 .5 1e3 1.5E-2 3e"), + [ + Token::Number(1.0), + Token::Number(2.5), + Token::Number(0.5), + Token::Number(1000.0), + Token::Number(0.015), + Token::Number(3.0), + Token::Name("e".to_string()), + Token::End + ] + ); + let err = lex("1..2").unwrap_err(); + assert_eq!(err.to_string(), "error at 1:1: invalid number `1..2`"); + } + + #[test] + fn line_breaks_end_statements_only_after_complete_expressions() { + use Token::*; + let name = |n: &str| Name(n.to_string()); + assert_eq!( + tokens("a = x +\n 1\nb = sma(x,\n 2) # comment\n\n"), + [ + name("a"), + Assign, + name("x"), + Op('+'), + Number(1.0), + Sep, + name("b"), + Assign, + name("sma"), + Open, + name("x"), + Comma, + Number(2.0), + Close, + Sep, + End + ] + ); + assert_eq!(tokens("x; y"), [name("x"), Sep, name("y"), End]); + } + + #[test] + fn locations_count_lines_and_columns() { + let spanned = lex("a = 1\n b = 2").unwrap(); + let b = &spanned[4]; + assert_eq!(b.0, Token::Name("b".to_string())); + assert_eq!((b.1.line(), b.1.column(), b.1.offset()), (2, 3, 8)); + let err = lex("x\n $").unwrap_err(); + assert_eq!(err.to_string(), "error at 2:3: unexpected `$`"); + } +} diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs new file mode 100644 index 0000000..2f1c98d --- /dev/null +++ b/src/pipeline/mod.rs @@ -0,0 +1,111 @@ +//! Feature pipelines as data: many features over several input series, +//! written as text (e.g. by an LLM) or assembled by code, checked, and +//! compiled into one flat program at runtime. +//! +//! ``` +//! use sliding_features::pipeline::{Domain, Input, Limits, Pipeline}; +//! +//! let source = " +//! let r = ln_return(price) # a helper, not an output +//! vol_ratio = std(r, 20) / std(r, 100) +//! trend = zscore(ema(price, 10) - ema(price, 50), 200) +//! "; +//! let inputs = vec![Input::new("price", Domain::positive())]; +//! let mut pipeline = Pipeline::compile(source, inputs, Limits::default()).unwrap(); +//! assert_eq!(Vec::from_iter(pipeline.output_names()), ["vol_ratio", "trend"]); +//! for i in 0..300 { +//! pipeline.update(&[100.0 + (f64::from(i) * 0.1).sin()]); +//! } +//! assert!(pipeline.outputs().all(|v| v.is_some())); +//! ``` +//! +//! [`reference()`] describes the language and every function, to put into an +//! LLM's prompt. A [`Builder`] assembles the same pipelines node by node, +//! e.g. for a search. +//! +//! Equal computations are shared, so features that use the same +//! `ema(x, 10)` compute it once. The pipeline runs as a flat list of +//! operations over a value buffer: one pass per update, no allocation, no +//! recursion and no dynamic dispatch. +//! +//! Each [`Function`] declares what it needs of its argument +//! ([`Requirement`]) and the [`Domain`] of its output. From the domains of +//! the inputs the compiler derives every node's domain and warm-up before +//! any data is seen: it rejects calls that are never ready and constant +//! outputs, and warns about calls that are not always ready, each with a +//! [`Location`] and a message that says how to fix it. + +mod analysis; +mod builder; +mod compiled; +mod display; +mod domain; +mod error; +mod function; +mod graph; +mod lex; +mod parse; +mod program; +mod state; +#[cfg(test)] +mod tests; + +pub use builder::{ + Builder, + Input, + Limits, +}; +pub use compiled::{ + Feature, + Pipeline, +}; +pub use domain::Domain; +pub use error::{ + CompileError, + Diagnostic, + Location, + Severity, +}; +pub use function::{ + BinaryOp, + Function, + Param, + Requirement, +}; +pub use graph::NodeId; + +/// The language guide and the table of every function, in markdown: what +/// an LLM needs in its prompt to write pipelines. +pub fn reference() -> String { + let mut out = String::from(LANGUAGE); + out.push_str(&Function::reference()); + out +} + +const LANGUAGE: &str = "\ +# Feature pipelines + +A pipeline computes features (its outputs) from input series, one update at a time. + +- One statement per line (or separated by `;`); `#` starts a comment. +- `name = expr` defines an output. `let name = expr` defines a helper that later \ +statements can use by name. A bare `expr` is an output named by its text. +- An expression is a number, an input or helper name, `a + b`, `a - b`, `a * b`, `a / b` \ +(usual precedence), `-a`, `(a)`, or a call `function(a, param, ...)`: the first argument \ +is an expression, the parameters after it are number literals. +- Operators combine the values of the same update. Equal subexpressions are computed once. +- An output is not ready (no value) until its windows fill, or while an argument is outside \ +what a function accepts (a division by zero, a logarithm of a non-positive value). Ready \ +outputs are always finite. A function whose argument is not ready keeps its last value. +- The compiler rejects calls that can never be ready and constant outputs, and warns about \ +calls that are not always ready. + +```text +let r = ln_return(price) +vol_ratio = std(r, 20) / std(r, 100) +trend = zscore(ema(price, 10) - ema(price, 50), 200) +``` + +## Functions + +"; diff --git a/src/pipeline/parse.rs b/src/pipeline/parse.rs new file mode 100644 index 0000000..ee383bf --- /dev/null +++ b/src/pipeline/parse.rs @@ -0,0 +1,610 @@ +//! Recursive-descent parser of the text form, driving a [`Builder`]. +//! +//! ```text +//! program := (statement (sep statement)*)? +//! statement := 'let' name '=' expr | name '=' expr | expr +//! expr := term (('+' | '-') term)* +//! term := unary (('*' | '/') unary)* +//! unary := '-' unary | atom +//! atom := number | name | name '(' expr (',' param)* ')' | '(' expr ')' +//! param := '-'? number +//! ``` + +use super::{ + BinaryOp, + Builder, + CompileError, + Function, + Location, + graph::NodeId, + lex::{ + Spanned, + Token, + lex, + }, +}; + +/// Parse `source` into `builder`, statement by statement. +pub(super) fn parse(source: &str, builder: Builder) -> Result { + let max_depth = builder.limits().max_depth; + let mut parser = Parser { + tokens: lex(source)?, + pos: 0, + builder, + depth: 0, + max_depth, + }; + parser.program()?; + assert_eq!(parser.depth, 0, "nesting is balanced"); + Ok(parser.builder) +} + +struct Parser { + tokens: Vec, + pos: usize, + builder: Builder, + /// The current nesting of parentheses, calls and negations: bounded + /// so a malicious text cannot overflow the stack. + depth: usize, + max_depth: usize, +} + +impl Parser { + fn peek(&self) -> &Spanned { + &self.tokens[self.pos] + } + + fn peek_second(&self) -> &Token { + &self.tokens[(self.pos + 1).min(self.tokens.len() - 1)].0 + } + + fn next(&mut self) -> Spanned { + let spanned = self.tokens[self.pos].clone(); + if spanned.0 != Token::End { + self.pos += 1; + } + spanned + } + + fn expect(&mut self, expected: &Token, what: &str) -> Result<(), CompileError> { + let (token, at) = self.next(); + if &token == expected { + return Ok(()); + } + Err(CompileError::at( + at, + format!("expected {what}, found {}", token.describe()), + )) + } + + fn program(&mut self) -> Result<(), CompileError> { + loop { + while self.peek().0 == Token::Sep { + self.next(); + } + if self.peek().0 == Token::End { + return Ok(()); + } + self.statement()?; + let (token, at) = self.next(); + match token { + Token::Sep => {} + Token::End => return Ok(()), + other => { + return Err(CompileError::at( + at, + format!( + "expected a new line or `;` after the statement, found {}", + other.describe() + ), + )); + } + } + } + } + + fn statement(&mut self) -> Result<(), CompileError> { + let (first, at) = self.peek().clone(); + match (first, self.peek_second()) { + (Token::Name(keyword), Token::Name(_)) if keyword == "let" => { + self.next(); + let (Token::Name(name), name_at) = self.next() else { + unreachable!("peeked a name"); + }; + self.expect(&Token::Assign, &format!("`=` after `let {name}`"))?; + let node = self.expr()?; + self.builder.set_location(name_at); + self.builder.bind(&name, node) + } + (Token::Name(name), Token::Assign) => { + self.next(); + self.next(); + let node = self.expr()?; + self.builder.set_location(at); + self.builder.output(&name, node) + } + _ => { + let node = self.expr()?; + self.builder.set_location(at); + self.builder.anonymous_output(node) + } + } + } + + fn expr(&mut self) -> Result { + let mut lhs = self.term()?; + while let (Token::Op(c @ ('+' | '-')), at) = self.peek().clone() { + self.next(); + let rhs = self.term()?; + let op = if c == '+' { + BinaryOp::Add + } else { + BinaryOp::Sub + }; + self.builder.set_location(at); + lhs = self.builder.binary(op, lhs, rhs)?; + } + Ok(lhs) + } + + fn term(&mut self) -> Result { + let mut lhs = self.unary()?; + while let (Token::Op(c @ ('*' | '/')), at) = self.peek().clone() { + self.next(); + let rhs = self.unary()?; + let op = if c == '*' { + BinaryOp::Mul + } else { + BinaryOp::Div + }; + self.builder.set_location(at); + lhs = self.builder.binary(op, lhs, rhs)?; + } + Ok(lhs) + } + + fn unary(&mut self) -> Result { + let (token, at) = self.peek().clone(); + if token != Token::Op('-') { + return self.atom(); + } + self.next(); + let inner = self.nested(at, Self::unary)?; + self.builder.set_location(at); + // A negated literal folds into a negative constant. + self.builder.neg(inner) + } + + fn atom(&mut self) -> Result { + let (token, at) = self.next(); + self.builder.set_location(at); + match token { + Token::Number(value) => self.builder.constant(value), + Token::Name(name) if self.peek().0 == Token::Open => { + self.nested(at, |p| p.call(&name, at)) + } + Token::Name(name) => self + .builder + .lookup(&name) + .ok_or_else(|| self.unknown_name(&name, at)), + Token::Open => { + let inner = self.nested(at, Self::expr)?; + self.expect(&Token::Close, "`)`")?; + Ok(inner) + } + other => Err(CompileError::at( + at, + format!("expected an expression, found {}", other.describe()), + )), + } + } + + fn call(&mut self, name: &str, at: Location) -> Result { + let function = Function::from_name(name).ok_or_else(|| self.unknown_function(name, at))?; + self.expect(&Token::Open, &format!("`(` after `{name}`"))?; + let arg = self.expr()?; + let mut params = Vec::new(); + while self.peek().0 == Token::Comma { + self.next(); + params.push(self.param(name)?); + } + self.expect(&Token::Close, "`)` or `,`")?; + self.builder.set_location(at); + self.builder.call(function, arg, ¶ms) + } + + fn param(&mut self, function: &str) -> Result { + let (token, at) = self.next(); + let negative = token == Token::Op('-'); + let token = if negative { self.next().0 } else { token }; + match token { + Token::Number(value) if negative => Ok(-value), + Token::Number(value) => Ok(value), + other => Err(CompileError::at( + at, + format!( + "the parameters of `{function}` after its argument must be numbers, found {}", + other.describe() + ), + )), + } + } + + /// Run `f` one level deeper, failing past the depth limit. + fn nested( + &mut self, + at: Location, + f: impl FnOnce(&mut Self) -> Result, + ) -> Result { + if self.depth >= self.max_depth { + return Err(CompileError::at( + at, + format!( + "the expression is nested deeper than the limit of {}", + self.max_depth + ), + )); + } + self.depth += 1; + let result = f(self); + self.depth -= 1; + result + } + + fn unknown_name(&self, name: &str, at: Location) -> CompileError { + let message = if let Some(function) = Function::from_name(name) { + format!( + "`{name}` is a function, call it as `{}`", + function.signature() + ) + } else { + let known = Vec::from_iter(self.builder.names()); + let hint = match closest(name, known.iter().copied()) { + Some(close) => format!("did you mean `{close}`?"), + None => format!("defined so far: {}", known.join(", ")), + }; + format!("unknown name `{name}`; {hint}") + }; + CompileError::at(at, message) + } + + fn unknown_function(&self, name: &str, at: Location) -> CompileError { + let hint = if self.builder.lookup(name).is_some() { + format!("`{name}` is a value, not a function") + } else if let Some(close) = closest(name, Function::ALL.iter().map(|f| f.name())) { + format!("did you mean `{close}`?") + } else { + "see the function reference".to_string() + }; + CompileError::at(at, format!("unknown function `{name}`; {hint}")) + } +} + +/// The candidate closest to `name` by edit distance, if close enough to be +/// a likely typo: a third of the name may change, but not all of it (`y` +/// is no typo of `x`). +fn closest<'a>(name: &str, candidates: impl Iterator) -> Option<&'a str> { + let len = name.chars().count(); + let limit = (len / 3).max(1); + candidates + .map(|c| (edit_distance(name, c), c)) + .filter(|(d, _)| *d <= limit && *d < len) + .min_by_key(|(d, _)| *d) + .map(|(_, c)| c) +} + +/// The Levenshtein distance between two strings, in characters. +fn edit_distance(a: &str, b: &str) -> usize { + let b: Vec = b.chars().collect(); + let mut row: Vec = (0..=b.len()).collect(); + for (i, ca) in a.chars().enumerate() { + let mut diagonal = row[0]; + row[0] = i + 1; + for (j, cb) in b.iter().enumerate() { + let substitution = diagonal + usize::from(ca != *cb); + diagonal = row[j + 1]; + row[j + 1] = substitution.min(row[j] + 1).min(row[j + 1] + 1); + } + } + row[b.len()] +} + +#[cfg(test)] +mod tests { + use super::{ + super::{ + Domain, + Input, + Limits, + Param, + Pipeline, + }, + *, + }; + + /// Compile over `x`, check that the canonical form compiles to the + /// same canonical form, and return it. + fn canonical(source: &str) -> String { + let pipeline: Pipeline = source.parse().unwrap_or_else(|e| panic!("{source}: {e}")); + let printed = pipeline.to_string(); + let again: Pipeline = printed.parse().unwrap_or_else(|e| panic!("{printed}: {e}")); + assert_eq!(again.to_string(), printed, "{source}"); + printed + } + + fn error(source: &str) -> String { + source.parse::().expect_err(source).to_string() + } + + #[test] + fn precedence_and_associativity() { + assert_eq!(canonical("x + x * 2"), "x + (x * 2)"); + assert_eq!(canonical("(x + x) * 2"), "(x + x) * 2"); + assert_eq!(canonical("x - 1 - x"), "(x - 1) - x"); + assert_eq!(canonical("x / 2 / 4"), "(x / 2) / 4"); + assert_eq!(canonical("-x * 2 + -(x - 1)"), "(-x * 2) + -(x - 1)"); + } + + #[test] + fn calls_unary_minus_and_numbers() { + assert_eq!( + canonical("zscore(ema(x, 10) - ema(x,50), 200)"), + "zscore(ema(x, 10) - ema(x, 50), 200)" + ); + assert_eq!(canonical("x * -0.5"), "x * -0.5"); + assert_eq!(canonical("x * -(2 * 3)"), "x * -6", "constants fold"); + assert_eq!(canonical("gte(x, -1.5e-3)"), "gte(x, -0.0015)"); + assert_eq!(canonical("--x"), "--x"); + assert_eq!(canonical("-(x + 1)"), "-(x + 1)"); + assert_eq!(canonical("pfe(roc(x, 1), 20, 5)"), "pfe(roc(x, 1), 20, 5)"); + assert_eq!( + canonical(" laguerre_filter( x ,0.8 ) "), + "laguerre_filter(x, 0.8)" + ); + } + + #[test] + fn every_function_round_trips() { + for function in Function::ALL { + let mut source = format!("{}(x", function.name()); + for param in function.params() { + let value = match param { + Param::Window => function.min_window().to_string(), + Param::Real | Param::Fraction => "0.5".to_string(), + }; + source.push_str(", "); + source.push_str(&value); + } + source.push(')'); + assert_eq!(canonical(&source), source); + } + } + + #[test] + fn statements_bindings_and_sharing() { + let source = " + # helpers are named in the canonical form + let r = ln_return(x) + a = std(r, 10) / std(r, 20); b = std(ln_return(x), 10) + ema(x, 3) - + x + rsi = rsi(x, 14) + c = ema(rsi, 2) + "; + assert_eq!( + canonical(source), + "let r = ln_return(x)\n\ + a = std(r, 10) / std(r, 20)\n\ + b = std(r, 10)\n\ + ema(x, 3) - x\n\ + rsi = rsi(x, 14)\n\ + c = ema(rsi, 2)" + ); + let p: Pipeline = source.parse().unwrap(); + assert_eq!( + Vec::from_iter(p.output_names()), + ["a", "b", "ema(x, 3) - x", "rsi", "c"] + ); + // x, r, two stds, a, ema, the difference, rsi and its ema. + assert_eq!(p.op_count(), 9); + } + + #[test] + fn several_inputs() { + let inputs = vec![ + Input::new("price", Domain::positive()), + Input::new("volume", Domain::non_negative()), + ]; + let source = "vwap = cumulative(price * volume, 2) / cumulative(volume, 2)"; + let mut p = Pipeline::compile(source, inputs, Limits::default()).unwrap(); + p.update(&[10.0, 1.0]); + p.update(&[20.0, 3.0]); + assert_eq!(p.output(0), Some(17.5)); + assert_eq!( + p.to_string(), + "vwap = cumulative(price * volume, 2) / cumulative(volume, 2)" + ); + } + + #[test] + fn errors_point_at_the_problem() { + for (source, expected) in [ + ( + "sma(x)", + "error at 1:1: sma(a, window) takes 1 parameter(s) after its argument, got 0", + ), + ( + "smaa(x, 3)", + "error at 1:1: unknown function `smaa`; did you mean `sma`?", + ), + ( + "sma(x, y)", + "error at 1:8: the parameters of `sma` after its argument must be numbers, found `y`", + ), + ( + "sma(x, sma(x, 2))", + "error at 1:8: the parameters of `sma` after its argument must be numbers, found `sma`", + ), + ("x +", "error at 1:4: expected an expression, found the end"), + ("(x", "error at 1:3: expected `)`, found the end"), + ( + "x x", + "error at 1:3: expected a new line or `;` after the statement, found `x`", + ), + ("x $ 2", "error at 1:3: unexpected `$`"), + ( + "sma(x, 2.5)", + "error at 1:1: sma(a, window): the window parameter 2.5 must be an integer in [1, 100000]", + ), + ("1..2", "error at 1:1: invalid number `1..2`"), + ( + "a = x\nb = y", + "error at 2:5: unknown name `y`; defined so far: x, a", + ), + ( + "a = x\nb = aa", + "error at 2:5: unknown name `aa`; did you mean `a`?", + ), + ( + "a = sma", + "error at 1:5: `sma` is a function, call it as `sma(a, window)`", + ), + ( + "let a = x\na(x)", + "error at 2:1: unknown function `a`; `a` is a value, not a function", + ), + ( + "let a = x\na = x", + "error at 2:1: the name `a` is already defined", + ), + ("let = 3", "error at 1:1: the name `let` is a keyword"), + ( + "ema(x, 3); ema(x, 3)", + "error at 1:12: the output `ema(x, 3)` is already defined", + ), + ("a = 1 / (2 - 2)", "error at 1:7: `1 / 0` is not finite"), + ("", "error: a pipeline needs at least one output"), + ] { + assert_eq!(error(source), expected, "{source}"); + } + } + + #[test] + fn analysis_findings_point_at_their_expression() { + assert_eq!( + error("a = x\nb = ln(-abs(x) - 1)\nx - x"), + "error at 2:5: `ln(-abs(x) - 1)` is never ready: `ln` needs positive values, \ + but `-abs(x) - 1` lies in (-inf, -1]\n\ + error at 3:1: output `x - x` is constant: it is always 0" + ); + let p: Pipeline = "a = ema(x, 2)\nb = 1 + ln(x)".parse().unwrap(); + let warning = &p.warnings()[0]; + assert_eq!( + warning.location().map(|l| (l.line(), l.column())), + Some((2, 9)) + ); + } + + #[test] + fn deep_nesting_is_an_error_not_a_stack_overflow() { + for source in [ + format!("{}x{}", "(".repeat(100_000), ")".repeat(100_000)), + format!("{}x", "-".repeat(100_000)), + format!("{}x{}", "abs(".repeat(100_000), ")".repeat(100_000)), + ] { + let err = error(&source); + assert!(err.contains("nested deeper than the limit of 32"), "{err}"); + } + } + + /// A random expression over the builder's inputs, `None` if the + /// builder rejects it (e.g. a constant folds to infinity). + fn random_expr(b: &mut super::Builder, rng: &romu::Rng, depth: usize) -> Option { + let leaf = depth == 0 || rng.range_usize(0..4) == 0; + if leaf { + return match rng.range_usize(0..3) { + 0 => b.lookup("y"), + 1 => b.constant(f64::from(rng.range_u8(1..=40)) / 8.0 - 2.5).ok(), + _ => b.lookup("x"), + }; + } + let a = random_expr(b, rng, depth - 1)?; + match rng.range_usize(0..5) { + 0 => b.neg(a).ok(), + 1 | 2 => { + let ops = [BinaryOp::Add, BinaryOp::Sub, BinaryOp::Mul, BinaryOp::Div]; + let rhs = random_expr(b, rng, depth - 1)?; + b.binary(ops[rng.range_usize(0..ops.len())], a, rhs).ok() + } + _ => { + let function = Function::ALL[rng.range_usize(0..Function::ALL.len())]; + let params = Vec::from_iter(function.params().iter().map(|p| match p { + Param::Window => f64::from(rng.range_u8(3..=12)), + Param::Real => f64::from(rng.range_u8(0..=8)) / 4.0 - 1.0, + Param::Fraction => 0.5, + })); + b.call(function, a, ¶ms).ok() + } + } + } + + /// Random pipelines print to text that compiles to the same text and + /// computes the same values. + #[test] + fn random_pipelines_round_trip_through_text() { + let rng = romu::Rng::from_seed_with_64bit(11); + let inputs = || vec![Input::new("x", Domain::positive()), Input::real("y")]; + let mut compared = 0; + for _ in 0..500 { + let mut b = super::Builder::new(inputs(), Limits::default()).unwrap(); + let Some(helper) = random_expr(&mut b, &rng, 3) else { + continue; + }; + if b.bind("h", helper).is_err() { + continue; + } + let Some(out) = random_expr(&mut b, &rng, 4) else { + continue; + }; + let both = b + .lookup("h") + .and_then(|h| b.binary(BinaryOp::Add, h, out).ok()); + let Some(both) = both else { continue }; + if b.output("out", out).is_err() || b.output("both", both).is_err() { + continue; + } + let Ok(mut original) = b.build() else { + continue; + }; + let text = original.to_string(); + let mut parsed = Pipeline::compile(&text, inputs(), Limits::default()) + .unwrap_or_else(|e| panic!("{text}\n{e}")); + assert_eq!(parsed.to_string(), text); + assert_eq!(parsed.op_count(), original.op_count(), "{text}"); + for i in 0..50 { + let v = [100.0 + f64::from(i % 7), f64::from(i % 5) - 2.0]; + original.update(&v); + parsed.update(&v); + let same = original.outputs().zip(parsed.outputs()).all(|(a, b)| { + a == b || a.zip(b).is_some_and(|(a, b)| a.is_nan() && b.is_nan()) + }); + assert!(same, "{text}"); + } + compared += 1; + } + assert!(compared > 100, "only {compared} pipelines compiled"); + } + + #[test] + fn edit_distances() { + assert_eq!(edit_distance("sma", "sma"), 0); + assert_eq!(edit_distance("smaa", "sma"), 1); + assert_eq!(edit_distance("zscroe", "zscore"), 2); + assert_eq!(edit_distance("", "abc"), 3); + assert_eq!( + closest("emaa", ["ema", "sma", "rsi"].into_iter()), + Some("ema") + ); + assert_eq!(closest("foo", ["ema", "sma"].into_iter()), None); + } +} diff --git a/src/pipeline/program.rs b/src/pipeline/program.rs new file mode 100644 index 0000000..38d18a4 --- /dev/null +++ b/src/pipeline/program.rs @@ -0,0 +1,200 @@ +//! The runtime form of a pipeline: a flat list of operations evaluated in +//! one pass, each writing one slot of a value buffer. + +use super::{ + BinaryOp, + Requirement, + function::Params, + graph::{ + Graph, + Node, + NodeId, + }, + state::{ + Kind, + Pure, + State, + }, +}; + +/// One operation. Operands index the value buffer, always below the +/// operation's own slot. +#[derive(Debug, Clone, Copy)] +enum Op { + Input(u32), + Const(f64), + Pure(Pure, u32), + Binary(BinaryOp, u32, u32), + Stateful { + state: u32, + arg: u32, + requires: Requirement, + }, +} + +/// The operations of the nodes the outputs read, their states and their +/// last values; NaN marks a value that is not ready. +#[derive(Debug, Clone)] +pub(super) struct Program { + ops: Vec, + /// Kept apart from `ops`, so the loop over the small operations stays + /// dense and a large view does not widen every slot. + states: Vec, + values: Vec, + outputs: Vec, +} + +impl Program { + /// Lower the nodes that `outputs` read into operations, with fresh + /// states. + pub(super) fn lower(graph: &Graph, outputs: &[NodeId]) -> Self { + assert!(!outputs.is_empty(), "a pipeline has an output"); + let reachable = graph.reachable(outputs.iter().copied()); + let mut slot_of = vec![u32::MAX; graph.len()]; + let mut ops = Vec::new(); + let mut states = Vec::new(); + for (id, node) in graph.iter() { + if !reachable[id.index()] { + continue; + } + let slot = |child: &NodeId| { + let s = slot_of[child.index()]; + assert_ne!(s, u32::MAX, "children are lowered first"); + s + }; + let op = match node { + Node::Input(i) => Op::Input(to_u32(*i)), + Node::Const(c) => Op::Const(*c), + Node::Neg(a) => Op::Pure(Pure::Neg, slot(a)), + Node::Binary(op, a, b) => Op::Binary(*op, slot(a), slot(b)), + Node::Call(function, a, params) => { + match function.instantiate(Params::new(*function, params)) { + Kind::Pure(pure) => Op::Pure(pure, slot(a)), + Kind::Stateful(state) => { + states.push(state); + Op::Stateful { + state: to_u32(states.len() - 1), + arg: slot(a), + requires: function.requires(), + } + } + } + } + }; + slot_of[id.index()] = to_u32(ops.len()); + ops.push(op); + } + let outputs = Vec::from_iter(outputs.iter().map(|o| slot_of[o.index()])); + assert!(outputs.iter().all(|o| (*o as usize) < ops.len())); + Self { + values: vec![f64::NAN; ops.len()], + ops, + states, + outputs, + } + } + + /// Feed one value per input series. + #[inline] + pub(super) fn update(&mut self, inputs: &[f64]) { + run(&self.ops, &mut self.states, &mut self.values, inputs); + } + + /// The `i`-th output, `None` while not ready. + #[inline] + pub(super) fn output(&self, i: usize) -> Option { + let v = self.values[self.outputs[i] as usize]; + (!v.is_nan()).then_some(v) + } + + /// The number of operations run per update. + pub(super) fn len(&self) -> usize { + self.ops.len() + } +} + +fn to_u32(i: usize) -> u32 { + u32::try_from(i).expect("node counts fit u32") +} + +/// Evaluate every operation once, in order. Free of `self`, so the slices +/// stay in registers. +#[inline] +fn run(ops: &[Op], states: &mut [State], values: &mut [f64], inputs: &[f64]) { + assert_eq!(ops.len(), values.len(), "one value per operation"); + for (i, op) in ops.iter().enumerate() { + let v = match *op { + Op::Input(j) => inputs[j as usize], + Op::Const(c) => c, + Op::Pure(f, a) => f.eval(values[a as usize]), + Op::Binary(op, a, b) => op.eval(values[a as usize], values[b as usize]), + Op::Stateful { + state, + arg, + requires, + } => { + let a = values[arg as usize]; + let state = &mut states[state as usize]; + // A view only sees admitted values and otherwise holds its + // last output, like a chained view whose input is not ready. + if admits(requires, a) { + state.update(a) + } else { + state.last() + } + } + }; + // Not ready instead of an infinity: one non-finite value would stay + // in every window downstream for its whole length. + values[i] = if v.is_finite() { v } else { f64::NAN }; + } +} + +/// Whether a value meets the requirement; NaN (not ready) never does. +#[inline] +fn admits(requires: Requirement, v: f64) -> bool { + match requires { + Requirement::Any | Requirement::BothSigns => !v.is_nan(), + Requirement::Positive => v > 0.0, + Requirement::NonNegative => v >= 0.0, + } +} + +#[cfg(test)] +mod tests { + use super::{ + super::Function, + *, + }; + + #[test] + fn admits_rejects_not_ready_values() { + for requires in [ + Requirement::Any, + Requirement::BothSigns, + Requirement::Positive, + Requirement::NonNegative, + ] { + assert!(!admits(requires, f64::NAN)); + } + assert!(admits(Requirement::NonNegative, 0.0)); + assert!(!admits(Requirement::Positive, 0.0)); + assert!(admits(Requirement::Any, -1.0)); + } + + #[test] + fn only_reachable_nodes_are_lowered() { + let mut g = Graph::default(); + let x = g.intern(Node::Input(0), 10).unwrap(); + let unused = g + .intern(Node::Call(Function::Sma, x, [3.0].into()), 10) + .unwrap(); + let ema = g + .intern(Node::Call(Function::Ema, x, [3.0].into()), 10) + .unwrap(); + let program = Program::lower(&g, &[ema]); + assert_eq!(program.len(), 2); + assert_eq!(program.states.len(), 1); + assert_ne!(unused, ema); + } +} diff --git a/src/pipeline/state.rs b/src/pipeline/state.rs new file mode 100644 index 0000000..ebea878 --- /dev/null +++ b/src/pipeline/state.rs @@ -0,0 +1,212 @@ +//! What a call becomes at runtime: a stateless function evaluated inline, +//! or a sliding view of this crate held in one enum. + +use super::function::sigmoid; +use crate::{ + View, + pure_functions::Echo, + rolling::{ + Drawdown, + LnReturn, + }, + sliding_windows::{ + Alma, + BinaryEntropy, + CenterOfGravity, + CorrelationTrendIndicator, + Cumulative, + CyberCycle, + EhlersFisherTransform, + Ema, + Lag, + LaguerreFilter, + LaguerreRSI, + MadScaler, + Max, + Min, + MinMaxNormalizer, + MyRSI, + NoiseEliminationTechnology, + PolarizedFractalEfficiency, + ReFlex, + Roc, + RoofingFilter, + Rsi, + Sma, + SuperSmoother, + TrendFlex, + Vst, + WelfordOnline, + ZScoreStandardization, + }, +}; + +/// A stateless function of one value. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) enum Pure { + Neg, + Abs, + Sign, + Sqrt, + Square, + Exp, + Ln, + Tanh, + Sigmoid, + Asinh, + Gte(f64), + Lte(f64), +} + +impl Pure { + /// The function of `v`. A NaN (not ready) argument gives NaN; outside + /// its domain a function gives NaN or an infinity, which the caller + /// turns into "not ready". + #[inline] + pub(super) fn eval(self, v: f64) -> f64 { + // `f64::max` and `f64::min` would replace a NaN by the bound. + if v.is_nan() { + return v; + } + match self { + Pure::Neg => -v, + Pure::Abs => v.abs(), + // Not `f64::signum`, which is 1 for 0. + Pure::Sign => { + if v > 0.0 { + 1.0 + } else if v < 0.0 { + -1.0 + } else { + 0.0 + } + } + Pure::Sqrt => v.sqrt(), + Pure::Square => v * v, + Pure::Exp => v.exp(), + Pure::Ln => v.ln(), + Pure::Tanh => v.tanh(), + Pure::Sigmoid => sigmoid(v), + Pure::Asinh => v.asinh(), + Pure::Gte(c) => v.max(c), + Pure::Lte(c) => v.min(c), + } + } +} + +/// What a call becomes at runtime. +#[derive(Debug, Clone)] +pub(super) enum Kind { + Pure(Pure), + Stateful(State), +} + +/// Declares [`State`] over the views of this crate, each fed by an [`Echo`] +/// so the pipeline pushes values into it, with uniform `update` and `last`. +macro_rules! states { + ($($variant:ident($ty:ty),)*) => { + /// The state of a call to a sliding view. + #[derive(Debug, Clone)] + pub(super) enum State { + $($variant($ty),)* + } + + impl State { + /// Push `v` and return the output, NaN while not ready. + #[inline] + pub(super) fn update(&mut self, v: f64) -> f64 { + match self { + $(State::$variant(s) => { + s.update(v); + s.last().unwrap_or(f64::NAN) + })* + } + } + + /// The last output, NaN while not ready. + #[inline] + pub(super) fn last(&self) -> f64 { + match self { + $(State::$variant(s) => s.last().unwrap_or(f64::NAN),)* + } + } + } + + $(impl From<$ty> for Kind { + fn from(view: $ty) -> Self { + Kind::Stateful(State::$variant(view)) + } + })* + }; +} + +type E = Echo; + +states! { + LnReturn(LnReturn), + Drawdown(Drawdown), + Sma(Sma), + Ema(Ema), + Alma(Alma), + Roc(Roc), + Lag(Lag), + Min(Min), + Max(Max), + MinMax(MinMaxNormalizer), + Mad(MadScaler), + ZScore(ZScoreStandardization), + Std(WelfordOnline), + Cumulative(Cumulative), + Rsi(Rsi), + MyRsi(MyRSI), + LaguerreRsi(LaguerreRSI), + LaguerreFilter(LaguerreFilter), + SuperSmoother(SuperSmoother), + RoofingFilter(RoofingFilter), + ReFlex(ReFlex), + TrendFlex(TrendFlex), + CyberCycle(CyberCycle), + CenterOfGravity(CenterOfGravity), + Cti(CorrelationTrendIndicator), + Pfe(PolarizedFractalEfficiency>), + Fisher(EhlersFisherTransform>), + BinaryEntropy(BinaryEntropy), + Vst(Vst), + Net(NoiseEliminationTechnology), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pure_functions_propagate_not_ready() { + for f in [ + Pure::Neg, + Pure::Abs, + Pure::Sign, + Pure::Sqrt, + Pure::Square, + Pure::Exp, + Pure::Ln, + Pure::Tanh, + Pure::Sigmoid, + Pure::Asinh, + Pure::Gte(0.0), + Pure::Lte(0.0), + ] { + assert!(f.eval(f64::NAN).is_nan(), "{f:?}"); + } + } + + #[test] + fn pure_function_values() { + assert_eq!(Pure::Sign.eval(0.0), 0.0); + assert_eq!(Pure::Sign.eval(-3.0), -1.0); + assert_eq!(Pure::Gte(1.0).eval(0.5), 1.0); + assert_eq!(Pure::Lte(1.0).eval(0.5), 0.5); + assert!(Pure::Ln.eval(-1.0).is_nan()); + assert_eq!(Pure::Ln.eval(0.0), f64::NEG_INFINITY); + assert_eq!(Pure::Square.eval(-3.0), 9.0); + } +} diff --git a/src/pipeline/tests.rs b/src/pipeline/tests.rs new file mode 100644 index 0000000..703982f --- /dev/null +++ b/src/pipeline/tests.rs @@ -0,0 +1,435 @@ +//! Behaviour of compiled pipelines: they compute what hand-written view +//! chains compute, survive adversarial input and stay in their domains. + +use std::num::NonZeroUsize; + +use super::*; +use crate::{ + View, + pure_functions::{ + Echo, + Subtract, + }, + sliding_windows::{ + Ema, + Sma, + ZScoreStandardization, + }, +}; + +fn window(n: usize) -> NonZeroUsize { + NonZeroUsize::new(n).unwrap() +} + +fn single_input() -> (Builder, NodeId) { + let b = Builder::new(vec![Input::real("x")], Limits::default()).unwrap(); + let x = b.lookup("x").unwrap(); + (b, x) +} + +fn random_walk(seed: u64, len: usize) -> Vec { + let rng = romu::Rng::from_seed_with_64bit(seed); + let mut price = 100.0; + Vec::from_iter((0..len).map(|_| { + price += rng.f64() - 0.5; + price + })) +} + +/// Parameters at the smallest valid values. +fn min_params(function: Function) -> Vec { + Vec::from_iter(function.params().iter().map(|p| match p { + Param::Window => f64::from(u32::try_from(function.min_window()).unwrap()), + Param::Real | Param::Fraction => 0.5, + })) +} + +#[test] +fn matches_the_hand_written_chain() { + let (mut b, x) = single_input(); + let fast = b.call(Function::Ema, x, &[5.0]).unwrap(); + let slow = b.call(Function::Sma, x, &[10.0]).unwrap(); + let diff = b.binary(BinaryOp::Sub, fast, slow).unwrap(); + let z = b.call(Function::ZScore, diff, &[20.0]).unwrap(); + b.output("z", z).unwrap(); + let mut pipeline = b.build().unwrap(); + let mut manual = ZScoreStandardization::new( + Subtract::new( + Ema::new(Echo::new(), window(5)), + Sma::new(Echo::new(), window(10)), + ), + window(20), + ); + for price in random_walk(1, 500) { + pipeline.update(&[price]); + manual.update(price); + assert_eq!(pipeline.output(0), manual.last()); + } + assert!(pipeline.output(0).is_some()); +} + +#[test] +fn shared_nodes_are_computed_once() { + let (mut b, x) = single_input(); + let r = b.call(Function::LnReturn, x, &[]).unwrap(); + let fast = b.call(Function::Std, r, &[10.0]).unwrap(); + let slow = b.call(Function::Std, r, &[60.0]).unwrap(); + let ratio = b.binary(BinaryOp::Div, fast, slow).unwrap(); + b.output("ratio", ratio).unwrap(); + let again = b.call(Function::Std, r, &[10.0]).unwrap(); + b.output("fast", again).unwrap(); + let pipeline = b.build().unwrap(); + // x, ln_return, two stds and the division. + assert_eq!(pipeline.op_count(), 5); + assert_eq!(Vec::from_iter(pipeline.output_names()), ["ratio", "fast"]); + assert_eq!(pipeline.output_index("fast"), Some(1)); +} + +#[test] +fn unused_helpers_are_not_run() { + let (mut b, x) = single_input(); + let helper = b.call(Function::Sma, x, &[10.0]).unwrap(); + b.bind("helper", helper).unwrap(); + let out = b.call(Function::Ema, x, &[10.0]).unwrap(); + b.output("out", out).unwrap(); + assert_eq!(b.build().unwrap().op_count(), 2); +} + +#[test] +fn a_view_holds_its_value_while_its_argument_is_not_ready() { + let (mut b, x) = single_input(); + let one = b.constant(1.0).unwrap(); + let inv = b.binary(BinaryOp::Div, one, x).unwrap(); + let sma = b.call(Function::Sma, inv, &[2.0]).unwrap(); + b.output("inv", inv).unwrap(); + b.output("sma", sma).unwrap(); + let mut p = b.build().unwrap(); + p.update(&[1.0]); + p.update(&[2.0]); + assert_eq!(Vec::from_iter(p.outputs()), [Some(0.5), Some(0.75)]); + p.update(&[0.0]); + assert_eq!(Vec::from_iter(p.outputs()), [None, Some(0.75)]); + p.update(&[4.0]); + assert_eq!(Vec::from_iter(p.outputs()), [Some(0.25), Some(0.375)]); +} + +#[test] +fn requirements_guard_the_views() { + let (mut b, x) = single_input(); + let one = b.constant(1.0).unwrap(); + let shifted = b.binary(BinaryOp::Sub, x, one).unwrap(); + let r = b.call(Function::LnReturn, shifted, &[]).unwrap(); + b.output("r", r).unwrap(); + let mut p = b.build().unwrap(); + for x in [2.0, 0.5, 3.0] { + p.update(&[x]); + } + // 0.5 - 1 < 0 never reached the log return. + assert!((p.output(0).unwrap() - 2.0_f64.ln()).abs() < 1e-12); +} + +#[test] +fn non_finite_input_is_not_ready() { + let (mut b, x) = single_input(); + let sma = b.call(Function::Sma, x, &[1.0]).unwrap(); + b.output("raw", x).unwrap(); + b.output("sma", sma).unwrap(); + let mut p = b.build().unwrap(); + p.update(&[f64::INFINITY]); + assert_eq!(Vec::from_iter(p.outputs()), [None, None]); + p.update(&[2.0]); + assert_eq!(Vec::from_iter(p.outputs()), [Some(2.0), Some(2.0)]); +} + +#[test] +fn several_inputs() { + let inputs = vec![ + Input::new("price", Domain::positive()), + Input::new("volume", Domain::non_negative()), + ]; + let mut b = Builder::new(inputs, Limits::default()).unwrap(); + let (price, volume) = (b.lookup("price").unwrap(), b.lookup("volume").unwrap()); + let turnover = b.binary(BinaryOp::Mul, price, volume).unwrap(); + let num = b.call(Function::Cumulative, turnover, &[2.0]).unwrap(); + let den = b.call(Function::Cumulative, volume, &[2.0]).unwrap(); + let vwap = b.binary(BinaryOp::Div, num, den).unwrap(); + b.output("vwap", vwap).unwrap(); + let mut p = b.build().unwrap(); + p.update(&[10.0, 1.0]); + p.update(&[20.0, 3.0]); + assert_eq!(p.output(0), Some(17.5)); + assert_eq!(p.inputs()[1].name(), "volume"); +} + +#[test] +#[should_panic(expected = "one value per input")] +fn update_checks_the_input_count() { + let (mut b, x) = single_input(); + b.output("raw", x).unwrap(); + b.build().unwrap().update(&[1.0, 2.0]); +} + +#[test] +fn clone_branches_and_reset_starts_over() { + let (mut b, x) = single_input(); + let rsi = b.call(Function::Rsi, x, &[14.0]).unwrap(); + b.output("rsi", rsi).unwrap(); + let mut p = b.build().unwrap(); + let fresh = p.clone(); + let walk = random_walk(2, 200); + for v in &walk[..100] { + p.update(&[*v]); + } + let mut branch = p.clone(); + for v in &walk[100..] { + p.update(&[*v]); + branch.update(&[*v]); + assert_eq!(p.output(0), branch.output(0)); + } + p.reset(); + let mut fresh = fresh; + for v in &walk { + p.update(&[*v]); + fresh.update(&[*v]); + assert_eq!(p.output(0), fresh.output(0)); + } +} + +#[test] +fn a_single_output_pipeline_is_a_view() { + let (mut b, x) = single_input(); + let ema = b.call(Function::Ema, x, &[3.0]).unwrap(); + b.output("ema", ema).unwrap(); + let feature = Feature::try_from(b.build().unwrap()).unwrap(); + let mut chained = Sma::new(feature, window(2)); + let mut manual = Sma::new(Ema::new(Echo::new(), window(3)), window(2)); + for v in random_walk(3, 50) { + chained.update(v); + manual.update(v); + assert_eq!(chained.last(), manual.last()); + } + + let (mut b, x) = single_input(); + b.output("a", x).unwrap(); + b.output("b", x).unwrap(); + let err = Feature::try_from(b.build().unwrap()).unwrap_err(); + assert!(err.to_string().contains("has 1 and 2"), "{err}"); +} + +/// Inputs that break naive indicators: a trend, a constant, zeros, +/// alternating signs, huge values. +fn adversarial_inputs() -> Vec> { + vec![ + random_walk(7, 300), + vec![5.0; 300], + vec![0.0; 300], + Vec::from_iter((0..300).map(|i| if i % 2 == 0 { 1.0 } else { -1.0 })), + Vec::from_iter((0..300).map(|i| 1e12 * f64::from(i % 7))), + ] +} + +/// The arguments every function is tried on: the input, and series of +/// both signs that are exactly zero on constant input. (`x - x` would be +/// rejected as constant.) +fn arguments(b: &mut Builder, x: NodeId) -> Vec { + let roc = b.call(Function::Roc, x, &[1.0]).unwrap(); + let lag = b.call(Function::Lag, x, &[1.0]).unwrap(); + let diff = b.binary(BinaryOp::Sub, x, lag).unwrap(); + vec![x, roc, diff] +} + +/// Every function, at its minimum parameters, on every adversarial input +/// and composed with arguments of both signs, never panics (debug +/// assertions on) and only ever outputs finite values. +#[test] +fn every_function_survives_adversarial_input() { + let mut failures = Vec::new(); + for function in Function::ALL { + for input in adversarial_inputs() { + let run = std::panic::catch_unwind(|| { + let (mut b, x) = single_input(); + for (i, arg) in arguments(&mut b, x).into_iter().enumerate() { + let call = b.call(*function, arg, &min_params(*function)).unwrap(); + b.output(&format!("out{i}"), call).unwrap(); + } + let mut p = b.build().unwrap(); + input.iter().all(|v| { + p.update(&[*v]); + p.outputs().all(|o| o.is_none_or(f64::is_finite)) + }) + }); + match run { + Ok(true) => {} + Ok(false) => failures.push(format!("{}: non-finite output", function.name())), + Err(_) => failures.push(format!( + "{}: panicked on {:?}", + function.name(), + &input[..3] + )), + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} + +/// Every ready output lies in the domain the function declares for an +/// argument in the hull of the input, up to rounding. +#[test] +fn outputs_stay_in_their_declared_domain() { + let mut failures = Vec::new(); + for function in Function::ALL { + let params = min_params(*function); + for input in adversarial_inputs() { + let lo = input.iter().copied().fold(f64::INFINITY, f64::min); + let hi = input.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let (mut b, x) = single_input(); + let call = b.call(*function, x, ¶ms).unwrap(); + b.output("out", call).unwrap(); + let mut p = b.build().unwrap(); + let domain = function.output_domain(Domain::closed(lo, hi), ¶ms); + for v in &input { + p.update(&[*v]); + let Some(out) = p.output(0) else { continue }; + let Some(domain) = domain else { + // A drawdown that never updated reports 0. + if *function != Function::Drawdown { + failures.push(format!("{}: ready but never admitted", function.name())); + } + break; + }; + // Indicators round, e.g. a correlation of 1 + 1e-8. + let tolerance = 1e-6 * out.abs().max(1.0); + let inside = domain.contains(out) + || (out >= domain.lo() - tolerance && out <= domain.hi() + tolerance); + if !inside { + failures.push(format!( + "{}: {out} outside {domain} on [{lo}, {hi}]", + function.name() + )); + break; + } + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} + +/// After its estimated warm-up on a positive random walk, every function +/// is ready, at its minimum parameters and at larger ones. +#[test] +fn the_warm_up_estimate_suffices() { + let mut failures = Vec::new(); + for function in Function::ALL { + let larger = Vec::from_iter(function.params().iter().map(|p| match p { + Param::Window => 10.0, + Param::Real | Param::Fraction => 0.5, + })); + for params in [min_params(*function), larger] { + let (mut b, x) = single_input(); + let inner = b.call(Function::Ema, x, &[3.0]).unwrap(); + let call = b.call(*function, inner, ¶ms).unwrap(); + b.output("out", call).unwrap(); + let mut p = b.build().unwrap(); + let walk = random_walk(4, p.warm_up()); + for v in &walk { + p.update(&[*v]); + } + if p.output(0).is_none() { + failures.push(format!( + "{}{params:?}: not ready after {}", + function.name(), + p.warm_up() + )); + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} + +fn build_errors(b: Builder) -> Vec { + let err = b.build().unwrap_err(); + Vec::from_iter(err.diagnostics().iter().map(ToString::to_string)) +} + +#[test] +fn never_ready_calls_and_constant_outputs_are_errors() { + let (mut b, x) = single_input(); + let abs = b.call(Function::Abs, x, &[]).unwrap(); + let neg = b.neg(abs).unwrap(); + let ln = b.call(Function::Ln, neg, &[]).unwrap(); + b.output("ln", ln).unwrap(); + let zero = b.binary(BinaryOp::Sub, x, x).unwrap(); + let div = b.binary(BinaryOp::Div, x, zero).unwrap(); + b.output("div", div).unwrap(); + b.output("zero", zero).unwrap(); + let sma = b.call(Function::Sma, zero, &[5.0]).unwrap(); + b.output("flat", sma).unwrap(); + assert_eq!( + build_errors(b), + [ + "error: `ln(-abs(x))` is never ready: `ln` needs positive values, but `-abs(x)` lies in (-inf, 0]", + "error: `x / zero` divides by zero, so it is never ready", + "error: output `zero` is constant: it is always 0", + "error: output `flat` is constant: it is always 0", + ] + ); +} + +#[test] +fn suspicious_pipelines_compile_with_warnings() { + let (mut b, x) = single_input(); + let ln = b.call(Function::Ln, x, &[]).unwrap(); + b.output("ln", ln).unwrap(); + let abs = b.call(Function::Abs, x, &[]).unwrap(); + let entropy = b.call(Function::BinaryEntropy, abs, &[10.0]).unwrap(); + b.output("entropy", entropy).unwrap(); + b.output("again", ln).unwrap(); + let unused = b.call(Function::Ema, x, &[3.0]).unwrap(); + b.bind("unused", unused).unwrap(); + let p = b.build().unwrap(); + let warnings = Vec::from_iter(p.warnings().iter().map(ToString::to_string)); + assert_eq!( + warnings, + [ + "warning: `ln(x)` is not ready while `x` is not positive (it lies in (-inf, inf)); \ + declare the domain of input `x` if it is always positive", + "warning: `binary_entropy(abs(x), 10)` is constant: `abs(x)` never changes sign (it lies in [0, inf))", + "warning: output `again` computes the same as output `ln`", + "warning: `unused` is not used by any output", + ] + ); +} + +#[test] +fn declared_input_domains_silence_warnings() { + let inputs = vec![Input::new("price", Domain::positive())]; + let mut b = Builder::new(inputs, Limits::default()).unwrap(); + let price = b.lookup("price").unwrap(); + let ln = b.call(Function::Ln, price, &[]).unwrap(); + let r = b.call(Function::LnReturn, price, &[]).unwrap(); + b.output("ln", ln).unwrap(); + b.output("r", r).unwrap(); + let p = b.build().unwrap(); + assert!(p.warnings().is_empty(), "{:?}", p.warnings()); +} + +#[test] +fn output_domains_and_warm_ups() { + let (mut b, x) = single_input(); + let rsi = b.call(Function::Rsi, x, &[14.0]).unwrap(); + let smooth = b.call(Function::Ema, rsi, &[5.0]).unwrap(); + let tanh = b.call(Function::Tanh, x, &[]).unwrap(); + let lag = b.call(Function::Lag, x, &[100.0]).unwrap(); + let sum = b.binary(BinaryOp::Add, smooth, lag).unwrap(); + b.output("smooth", smooth).unwrap(); + b.output("tanh", tanh).unwrap(); + b.output("sum", sum).unwrap(); + let p = b.build().unwrap(); + assert_eq!(p.output_domain(0), Domain::closed(0.0, 100.0)); + assert_eq!(p.output_domain(1), Domain::closed(-1.0, 1.0)); + assert_eq!(p.output_domain(2), Domain::REAL); + assert_eq!(p.output_warm_up(0), 19); + assert_eq!(p.output_warm_up(1), 0); + assert_eq!(p.output_warm_up(2), 100); + assert_eq!(p.warm_up(), 100); +} diff --git a/src/pure_functions/abs.rs b/src/pure_functions/abs.rs new file mode 100644 index 0000000..978807c --- /dev/null +++ b/src/pure_functions/abs.rs @@ -0,0 +1,59 @@ +use num::Float; + +use crate::View; + +/// The absolute value of the output of its View component, e.g. the size of +/// a return regardless of its direction. +#[derive(Debug, Clone)] +pub struct Abs { + view: V, + _marker: std::marker::PhantomData, +} + +impl Abs +where + V: View, + T: Float, +{ + /// Create a new instance with a chained View + pub fn new(view: V) -> Self { + Self { + view, + _marker: std::marker::PhantomData, + } + } +} + +impl View for Abs +where + V: View, + T: Float, +{ + fn update(&mut self, val: T) { + debug_assert!(val.is_finite(), "value must be finite"); + self.view.update(val); + } + + fn last(&self) -> Option { + self.view.last().map(|v| { + debug_assert!(v.is_finite(), "value must be finite"); + v.abs() + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pure_functions::Echo; + + #[test] + fn abs() { + let mut abs = Abs::new(Echo::new()); + assert_eq!(abs.last(), None); + abs.update(-2.5); + assert_eq!(abs.last(), Some(2.5)); + abs.update(3.0); + assert_eq!(abs.last(), Some(3.0)); + } +} diff --git a/src/pure_functions/ln.rs b/src/pure_functions/ln.rs new file mode 100644 index 0000000..00b989d --- /dev/null +++ b/src/pure_functions/ln.rs @@ -0,0 +1,71 @@ +use num::Float; + +use crate::View; + +/// The natural logarithm of the output of its View component, e.g. log +/// prices, whose differences are returns. +/// +/// The logarithm is only defined for positive values: for zero or negative +/// input, `last` is `None` rather than a non-finite value that would poison +/// every window downstream. +#[derive(Debug, Clone)] +pub struct Ln { + view: V, + _marker: std::marker::PhantomData, +} + +impl Ln +where + V: View, + T: Float, +{ + /// Create a new instance with a chained View + pub fn new(view: V) -> Self { + Self { + view, + _marker: std::marker::PhantomData, + } + } +} + +impl View for Ln +where + V: View, + T: Float, +{ + fn update(&mut self, val: T) { + debug_assert!(val.is_finite(), "value must be finite"); + self.view.update(val); + } + + fn last(&self) -> Option { + let v = self.view.last()?; + debug_assert!(v.is_finite(), "value must be finite"); + (v > T::zero()).then(|| v.ln()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pure_functions::Echo; + + #[test] + fn ln_of_positive_values() { + let mut ln = Ln::new(Echo::new()); + assert_eq!(ln.last(), None); + ln.update(1.0); + assert_eq!(ln.last(), Some(0.0)); + ln.update(std::f64::consts::E); + assert!((ln.last().unwrap() - 1.0).abs() < 1e-12); + } + + #[test] + fn ln_of_non_positive_values_is_none() { + let mut ln = Ln::new(Echo::new()); + ln.update(0.0); + assert_eq!(ln.last(), None); + ln.update(-1.0); + assert_eq!(ln.last(), None); + } +} diff --git a/src/pure_functions/mod.rs b/src/pure_functions/mod.rs index 4a8a4cc..dd69a54 100644 --- a/src/pure_functions/mod.rs +++ b/src/pure_functions/mod.rs @@ -1,21 +1,25 @@ //! This module contains `View` implementations that act like pure functions. //! They don't really rely on an internal state, but rather process value from other `View`s +mod abs; mod add; mod constant; mod divide; mod echo; mod gte; +mod ln; mod lte; mod multiply; mod subtract; mod tanh; +pub use abs::Abs; pub use add::Add; pub use constant::Constant; pub use divide::Divide; pub use echo::Echo; pub use gte::GTE; +pub use ln::Ln; pub use lte::LTE; pub use multiply::Multiply; pub use subtract::Subtract; diff --git a/src/sliding_windows/cumulative.rs b/src/sliding_windows/cumulative.rs index 0ae30cf..6c5f22b 100644 --- a/src/sliding_windows/cumulative.rs +++ b/src/sliding_windows/cumulative.rs @@ -48,19 +48,15 @@ where let Some(val) = self.view.last() else { return }; debug_assert!(val.is_finite(), "value must be finite"); - if self.out.is_none() { - self.out = Some(val); - } - + let out = self.out.get_or_insert(T::zero()); if self.q_vals.len() >= self.window_len.get() { - let old = self.q_vals.pop_front().unwrap(); - let out = self.out.as_mut().expect("Is some at this point"); + let old = self.q_vals.pop_front().expect("the window is full"); *out = *out - old; } self.q_vals.push_back(val); - let out = self.out.as_mut().expect("Is some at this point"); *out = *out + val; debug_assert!(out.is_finite(), "value must be finite"); + debug_assert!(self.q_vals.len() <= self.window_len.get()); } #[inline(always)] @@ -89,4 +85,18 @@ mod tests { let filename = "img/cumulative.png"; plot_values(out, filename).unwrap(); } + + /// The output is the sum of the last `window_len` values; the first + /// value used to be counted twice, forever. + #[test] + fn cumulative_sums_the_window() { + let mut cum = Cumulative::new(Echo::new(), NonZeroUsize::new(2).unwrap()); + assert_eq!(cum.last(), None); + let mut sums = Vec::new(); + for v in [1.0, 2.0, 4.0, 8.0] { + cum.update(v); + sums.push(cum.last().unwrap()); + } + assert_eq!(sums, [1.0, 3.0, 6.0, 12.0]); + } } diff --git a/src/sliding_windows/mad_scaler.rs b/src/sliding_windows/mad_scaler.rs index 9b0d7ca..2b65f71 100644 --- a/src/sliding_windows/mad_scaler.rs +++ b/src/sliding_windows/mad_scaler.rs @@ -38,6 +38,7 @@ use crate::View; /// An update costs `O(log w)` comparisons plus moving at most `w / 2` /// values of the sorted window (none for monotone input such as /// timestamps). +#[derive(Clone)] pub struct MadScaler { view: V, /// Sliding window length (for warm-up tracking). @@ -510,4 +511,22 @@ mod tests { } } } + + /// A clone of a warmed-up scaler continues exactly like the original. + #[test] + fn mad_scaler_clone_continues_identically() { + let mut original = MadScaler::new(Echo::new(), NonZeroUsize::new(16).unwrap()); + let (_, vals) = reference_streams(0, 200).remove(0); + let (head, tail) = vals.split_at(100); + for v in head { + original.update(*v); + } + let mut clone = original.clone(); + for v in tail { + original.update(*v); + clone.update(*v); + assert_eq!(original.last(), clone.last()); + } + assert!(clone.last().is_some()); + } } diff --git a/src/sliding_windows/min_max_normalizer.rs b/src/sliding_windows/min_max_normalizer.rs index 1d9c327..e42ff05 100644 --- a/src/sliding_windows/min_max_normalizer.rs +++ b/src/sliding_windows/min_max_normalizer.rs @@ -14,7 +14,8 @@ use crate::View; /// /// Normalizes values to the [-1, 1] range using the min and max of a sliding /// window of *past* values. The current value is intentionally excluded from -/// the normalization window to avoid lookahead / data-leakage bias. +/// the normalization window to avoid lookahead / data-leakage bias, so it can +/// lie beyond the window's extremes: such a value is clamped to -1 or 1. /// /// No output is emitted until the sliding window is completely filled. /// During warm-up `last()` returns `None`. @@ -99,13 +100,15 @@ where if self.min == self.max { self.out = Some(T::zero()); } else { - self.out = Some( - -T::one() - + ((view_last - self.min) * T::from(2.0).expect("can convert")) - / (self.max - self.min), - ); + let scaled = -T::one() + + ((view_last - self.min) * T::from(2.0).expect("can convert")) + / (self.max - self.min); + // A new extreme lies outside the past window's range. + self.out = Some(scaled.max(-T::one()).min(T::one())); } - debug_assert!(self.out.unwrap().is_finite(), "output must be finite"); + let out = self.out.expect("set above"); + debug_assert!(out.is_finite(), "output must be finite"); + debug_assert!(out.abs() <= T::one(), "output must lie in [-1, 1]"); // Slide the window. let old = self.q_vals.pop_front().expect("Its checked above that the length is >= the non-zero window length, therefore this must be `Some`"); @@ -269,9 +272,10 @@ mod tests { // After all updates, the last value (100) was normalized against // the previous window [60, 70, 80, 90] (min=60, max=90). - // normalized(100) = -1 + 2*(100-60)/(90-60) = -1 + 2*40/30 = 5/3 + // normalized(100) = -1 + 2*(100-60)/(90-60) = 5/3, a new high + // beyond the range, clamped to 1. let got = n.last().unwrap(); - let expected = 5.0 / 3.0; + let expected = 1.0; let diff = (got - expected).abs(); assert!( diff < 1e-12, @@ -440,4 +444,23 @@ mod tests { } } } + + /// New extremes beyond the past window are clamped, so the output + /// stays in the documented [-1, 1] range; it used to exceed it. + #[test] + fn min_max_normalizer_clamps_new_extremes() { + let mut n = MinMaxNormalizer::new(Echo::new(), NonZeroUsize::new(3).unwrap()); + for v in [0.0, 10.0, 5.0] { + n.update(v); + } + n.update(50.0); + assert_eq!(n.last(), Some(1.0)); + n.update(-50.0); + assert_eq!(n.last(), Some(-1.0)); + for v in &TEST_DATA { + n.update(*v); + let out = n.last().unwrap(); + assert!((-1.0..=1.0).contains(&out), "{out}"); + } + } } diff --git a/src/sliding_windows/polarized_fractal_efficiency.rs b/src/sliding_windows/polarized_fractal_efficiency.rs index 34ca07c..d3f9b6b 100644 --- a/src/sliding_windows/polarized_fractal_efficiency.rs +++ b/src/sliding_windows/polarized_fractal_efficiency.rs @@ -31,8 +31,16 @@ where T: Float, { /// Create a new PolarizedFractalEfficiency indicator with a chained view, custom moving - /// average and a window length + /// average and a window length. + /// + /// # Panics + /// + /// If `window_len` is 1: the efficiency of a single value is undefined. pub fn new(view: V, moving_average: M, window_len: NonZeroUsize) -> Self { + assert!( + window_len.get() >= 2, + "a fractal efficiency needs at least two values" + ); Self { view, moving_average, @@ -60,20 +68,8 @@ where } self.q_vals.push_back(val); - let window_len = T::from(self.window_len.get()).expect("can convert"); if self.q_vals.len() >= self.window_len.get() { - let mut s = T::zero(); - let wl: usize = self.window_len.get() - 1; - for i in 0..self.window_len.get() - 2 { - let v_0 = *self.q_vals.get(wl - i).unwrap(); - let v_1 = *self.q_vals.get(wl - i - 1).unwrap(); - s = s + ((v_0 - v_1).powi(2) + T::one()).sqrt(); - } - let mut p = - ((val - *self.q_vals.front().unwrap()).powi(2) + window_len.powi(2)).sqrt() / s; - if val < *self.q_vals.get(self.window_len.get() - 2).unwrap() { - p = -p; - } + let p = efficiency(&self.q_vals); // apply a moving average self.moving_average.update(p); self.out = self.moving_average.last(); @@ -88,6 +84,31 @@ where } } +/// The polarized efficiency of the path through `vals`, one unit of time +/// apart: the straight distance from the first to the last value over the +/// length of the path, negative when the last value is below the first. +/// +/// Both span the same `n = len - 1` steps, so by the triangle inequality the +/// magnitude is at most 1 (up to rounding, which is clamped). +fn efficiency(vals: &VecDeque) -> T { + assert!(vals.len() >= 2, "a path has at least two values"); + let (first, last) = ( + *vals.front().expect("not empty"), + *vals.back().expect("not empty"), + ); + let steps = T::from(vals.len() - 1).expect("can convert"); + let path = vals + .iter() + .zip(vals.iter().skip(1)) + .fold(T::zero(), |sum, (a, b)| { + sum + ((*b - *a).powi(2) + T::one()).sqrt() + }); + debug_assert!(path >= steps, "every step is at least one unit long"); + let straight = ((last - first).powi(2) + steps.powi(2)).sqrt(); + let magnitude = (straight / path).min(T::one()); + if last < first { -magnitude } else { magnitude } +} + #[cfg(test)] mod tests { use super::*; @@ -115,6 +136,49 @@ mod tests { } } + /// A straight path is fully efficient, in its direction; one segment + /// too few in the path length used to push it beyond 1. + #[test] + fn efficiency_of_reference_paths() { + let path = |vals: &[f64]| efficiency(&VecDeque::from_iter(vals.iter().copied())); + assert!((path(&[1.0, 2.0, 3.0, 4.0]) - 1.0).abs() < 1e-12); + assert!((path(&[4.0, 3.0, 2.0, 1.0]) + 1.0).abs() < 1e-12); + assert_eq!(path(&[5.0, 5.0, 5.0]), 1.0, "a flat path is straight"); + // Up 3 then down 2: the direction is that of the whole window, not + // of the last step. Straight sqrt(1 + 4), path sqrt(10) + sqrt(5). + let expected = 5.0_f64.sqrt() / (10.0_f64.sqrt() + 5.0_f64.sqrt()); + assert!((path(&[1.0, 4.0, 2.0]) - expected).abs() < 1e-12); + let zigzag = path(&[0.0, 10.0, 0.0, 10.0, 0.0]); + assert!(zigzag.abs() < 0.11, "{zigzag}"); + } + + #[test] + fn pfe_without_smoothing_stays_in_range() { + for window in [2, 3, 10] { + let mut pfe = PolarizedFractalEfficiency::new( + Echo::new(), + Echo::new(), + NonZeroUsize::new(window).unwrap(), + ); + for v in &TEST_DATA { + pfe.update(*v); + if let Some(val) = pfe.last() { + assert!((-1.0..=1.0).contains(&val), "window {window}: {val}"); + } + } + } + } + + #[test] + #[should_panic(expected = "at least two values")] + fn window_of_one_is_rejected() { + let _ = PolarizedFractalEfficiency::::new( + Echo::new(), + Echo::new(), + NonZeroUsize::new(1).unwrap(), + ); + } + #[test] fn polarized_fractal_efficiency_plot() { let mut pfe = PolarizedFractalEfficiency::new( diff --git a/src/sliding_windows/sma.rs b/src/sliding_windows/sma.rs index 63a9c40..bf5d7c9 100644 --- a/src/sliding_windows/sma.rs +++ b/src/sliding_windows/sma.rs @@ -50,11 +50,12 @@ where let Some(val) = self.view.last() else { return }; debug_assert!(val.is_finite(), "value must be finite"); - if self.q_vals.len() > self.window_len.get() { - let old_val = self.q_vals.pop_front().unwrap(); + if self.q_vals.len() >= self.window_len.get() { + let old_val = self.q_vals.pop_front().expect("the window is full"); self.sum = self.sum - old_val; } self.q_vals.push_back(val); + debug_assert!(self.q_vals.len() <= self.window_len.get()); self.sum = self.sum + val; } @@ -98,6 +99,19 @@ mod tests { } } + /// Exactly `window_len` values are averaged; one more used to stay in + /// the window once it was full. + #[test] + fn sma_averages_the_window() { + let mut sma = Sma::new(Echo::new(), NonZeroUsize::new(2).unwrap()); + let mut out = Vec::new(); + for v in [1.0, 3.0, 5.0, 7.0] { + sma.update(v); + out.push(sma.last()); + } + assert_eq!(out, [None, Some(2.0), Some(4.0), Some(6.0)]); + } + #[test] fn sma_plot() { let mut sma = Sma::new(Echo::new(), NonZeroUsize::new(16).unwrap());