From 07516704346745b1d2b57b168de19098442d021c Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:21:54 +0200 Subject: [PATCH 1/3] feat(multipath): fix the signature thresholds before evaluating them Co-Authored-By: Claude Sonnet 5.5 --- src/navkit/cli.py | 3 + src/navkit/fdir/multipath.py | 57 ++++++++++ src/navkit/multipath_study.py | 199 ++++++++++++++++++++++++++++++++++ 3 files changed, 259 insertions(+) create mode 100644 src/navkit/fdir/multipath.py create mode 100644 src/navkit/multipath_study.py diff --git a/src/navkit/cli.py b/src/navkit/cli.py index bf10c37..f28813c 100644 --- a/src/navkit/cli.py +++ b/src/navkit/cli.py @@ -36,6 +36,7 @@ fault_matrix, figures, mismatch_sweep, + multipath_study, outage_sweep, scene_sweep, seed_sweep, @@ -54,6 +55,7 @@ "attribution": attribution.main, "separability": separability.main, "ramp": slow_ramp.main, + "multipath": multipath_study.main, "faultclass": fault_classification.main, "vision-ramp": vision_ramp.main, } @@ -74,6 +76,7 @@ def _sweep(argv: list[str] | None) -> int: " attribution remove, inflate and grow each inertial error source to see what drives an outage", " separability honest returns after real-IMU outages against spoofs, at the first GNSS gate", " faultclass separate multipath, spoofing and degradation from GNSS innovations", + " multipath evaluate the multipath signature on injected faults", " ramp slow-ramp GNSS spoofs on real recorded IMU data", " vision-ramp slow-ramp GNSS spoofs with a simulated visual second source", ) diff --git a/src/navkit/fdir/multipath.py b/src/navkit/fdir/multipath.py new file mode 100644 index 0000000..01207bf --- /dev/null +++ b/src/navkit/fdir/multipath.py @@ -0,0 +1,57 @@ +"""A multipath signature on a window of GNSS innovations. + +Multipath puts noise on the fixes. Over a window of innovations that shows as an inflated normalised innovation +squared (NIS) with little correlation from one fix to the next. A persistent offset (a step or a ramp, a spoof) also +inflates the NIS, but its innovations are strongly autocorrelated. The signature is both together: a large NIS and a +low lag-1 autocorrelation. + +The two thresholds are round numbers, set from the training runs of the fault-classification study and evaluated on +runs the study never used (``docs/multipath.md``). NIS is about 1 for a healthy channel, so 2 is twice the healthy +value. This is a function on a window of innovations. It is not wired into the filter, and it does not change what +the FDIR layer does (ADR-0022). +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np + +#: A window whose mean NIS is at least this is inflated (a healthy channel is about 1). +NIS_MIN = 2.0 +#: A window whose lag-1 autocorrelation of the whitened innovation is above this is a persistent offset, not noise. +AC1_MAX = 0.8 +#: Fewer innovations than this and no call is made. +MIN_FIXES = 20 + + +@dataclass(frozen=True) +class Signature: + """The two statistics of a window, and the call.""" + + nis: float + ac1: float + n: int + multipath: bool + + +def signature(innovations: Sequence[tuple[float, np.ndarray, np.ndarray]]) -> Signature | None: + """The multipath signature of ``(t, residual, S)`` innovations, or None if the window is too short. + + ``nis`` is the mean of ``r' S^-1 r`` over three degrees of freedom. ``ac1`` is the uncentred lag-1 autocorrelation + of the whitened innovation ``L^-1 r`` with ``S = L L'``, so that a constant offset gives 1 and white noise gives 0. + """ + if len(innovations) < MIN_FIXES: + return None + d2 = np.array([float(i[1] @ np.linalg.solve(i[2], i[1])) for i in innovations]) + z = np.stack([np.linalg.solve(np.linalg.cholesky(i[2]), i[1]) for i in innovations]) + nis = float(d2.mean() / 3.0) + ac1 = float((z[:-1] * z[1:]).sum() / max(float((z * z).sum()), 1e-12)) + return Signature(nis=nis, ac1=ac1, n=len(innovations), multipath=nis >= NIS_MIN and ac1 <= AC1_MAX) + + +def is_multipath(innovations: Sequence[tuple[float, np.ndarray, np.ndarray]]) -> bool: + """True when the window looks like multipath; False when it does not or when it is too short to say.""" + sig = signature(innovations) + return bool(sig and sig.multipath) diff --git a/src/navkit/multipath_study.py b/src/navkit/multipath_study.py new file mode 100644 index 0000000..a4b8f68 --- /dev/null +++ b/src/navkit/multipath_study.py @@ -0,0 +1,199 @@ +"""How small a multipath fault does the signature catch, and does it call anything else? + +The thresholds of :mod:`navkit.fdir.multipath` were fixed before this study. This evaluates them on runs the +fault-classification study never used for the thresholds: its confirmation runs (544, unseen fault sizes and seeds), +plus new runs here: multipath at 0.5 to 2 m, to find the floor, and extra control runs, on seeds 6 and 7. + +Simulated GNSS, injected faults, two datasets, the calibrated bias-walk setting. +""" + +from __future__ import annotations + +import argparse +import csv +import os +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +from .euroc_compare import _ALL_CONFIGS +from .euroc_eval import RunOptions, dataset_loader, run_sequence +from .fault_classification import ONSET_S, SETTINGS, WINDOW_S, rows_from_csv +from .fdir import multipath as mp + +FLOOR_LEVELS_M = (0.5, 1.0, 1.5, 2.0) +SEEDS = (6, 7) +CONFIRM_CSV = "docs/data/fault_classification_confirm.csv" +CSV_COLUMNS = ("dataset", "sequence", "seed", "kind", "level", "nis", "ac1", "multipath") + + +def run_case(seq: Any, config: str, seed: int, level: float) -> dict[str, Any] | None: + """One run: multipath of ``level`` metres (0 for a control), and the signature of the window after the onset.""" + t_start = max(float(seq.imu.t[0]), float(seq.truth.t[0])) + if float(min(seq.imu.t[-1], seq.truth.t[-1])) - t_start < ONSET_S + WINDOW_S + 5.0: + return None + fault = ("multipath", ONSET_S, level) if level > 0 else None + inn: list[Any] = [] + run_sequence(seq, RunOptions(seed=seed, fault=fault, **_ALL_CONFIGS[config]), innovations_out=inn) + onset = t_start + ONSET_S + sig = mp.signature([i for i in inn if onset <= i[0] < onset + WINDOW_S]) + if sig is None: + return None + return { + "dataset": seq.slug, + "sequence": seq.name, + "seed": seed, + "kind": "multipath" if level > 0 else "none", + "level": level, + "nis": sig.nis, + "ac1": sig.ac1, + "multipath": sig.multipath, + } + + +def sweep_rows(dataset: str, root: str | Path) -> list[dict[str, Any]]: + loader, _ = dataset_loader(dataset) + rows: list[dict[str, Any]] = [] + for name in sorted(d for d in os.listdir(root) if (Path(root) / d).is_dir()): + seq = loader(root, name) + for level in (0.0, *FLOOR_LEVELS_M): + for seed in SEEDS: + row = run_case(seq, SETTINGS[dataset], seed, level) + if row is not None: + rows.append(row) + print(f"{dataset} {name}", file=sys.stderr, flush=True) + return rows + + +# --------------------------------------------------------------------- tables + + +def _pct(x: float) -> str: + return f"{100 * x:.0f}%" + + +def _called(row: dict[str, Any]) -> bool: + return str(row["multipath"]) in ("True", "true", "1") + + +def upper_bound(n: int) -> float: + """The rule of three: with no events in ``n`` trials, the 95% upper bound on the rate is about ``3 / n``.""" + return 3.0 / n + + +def confirm_rows(path: str | Path = CONFIRM_CSV) -> list[dict[str, Any]]: + """The confirmation runs of the fault-classification study, with the call recomputed from the stored features.""" + out = [] + for r in rows_from_csv(path): + nis, ac1 = float(r["nis"]), float(r["ac1"]) + out.append({**r, "multipath": nis >= mp.NIS_MIN and ac1 <= mp.AC1_MAX}) + return out + + +def confirm_markdown(rows: list[dict[str, Any]]) -> str: + cr = confirm_rows() + lines = ["| Class | Runs | Called multipath |", "|---|---:|---:|"] + for cls in ("control", "multipath", "spoofing", "degradation"): + sel = [r for r in cr if r["cls"] == cls] + lines.append(f"| {cls} | {len(sel)} | {_pct(sum(_called(r) for r in sel) / len(sel))} |") + return "\n".join(lines) + + +def floor_markdown(rows: list[dict[str, Any]]) -> str: + lines = [ + "| Multipath m | Runs | Called multipath | Median NIS | Median lag-1 autocorrelation |", + "|---:|---:|---:|---:|---:|", + ] + for level in (0.0, *FLOOR_LEVELS_M): + sel = [r for r in rows if float(r["level"]) == level] + if sel: + lines.append( + f"| {level:g} | {len(sel)} | {_pct(sum(_called(r) for r in sel) / len(sel))} | " + f"{np.median([float(r['nis']) for r in sel]):.2f} | {np.median([float(r['ac1']) for r in sel]):.2f} |" + ) + return "\n".join(lines) + + +def false_alarm_markdown(rows: list[dict[str, Any]]) -> str: + cr = [r for r in confirm_rows() if r["cls"] == "control"] + new = [r for r in rows if float(r["level"]) == 0.0] + lines = [ + "| Control runs | Runs | Called multipath | 95% upper bound on the false-alarm rate |", + "|---|---:|---:|---:|", + ] + for label, sel in (("confirmation", cr), ("new", new), ("both", cr + new)): + hits = sum(_called(r) for r in sel) + bound = _pct(upper_bound(len(sel))) if hits == 0 else "see the count" + lines.append(f"| {label} | {len(sel)} | {hits} | {bound} |") + return "\n".join(lines) + + +BLOCKS = { + "multipath-confirm": confirm_markdown, + "multipath-floor": floor_markdown, + "multipath-false-alarms": false_alarm_markdown, +} + +# ------------------------------------------------------------------------- I/O + + +def write_csv(path: str | Path, rows: list[dict[str, Any]]) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("w", newline="", encoding="utf-8") as fh: + writer = csv.DictWriter(fh, fieldnames=CSV_COLUMNS) + writer.writeheader() + writer.writerows(rows) + + +def write_block(path: str | Path, name: str, block: str) -> None: + page = Path(path) + text = page.read_text(encoding="utf-8") + start, end = f"", f"" + if start not in text or end not in text or text.index(start) > text.index(end): + raise ValueError(f"{page} needs the lines {start} and {end}, in that order") + head, rest = text.split(start, 1) + _, tail = rest.split(end, 1) + page.write_text(f"{head}{start}\n\n{block}\n\n{end}{tail}", encoding="utf-8") + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="navkit sweep multipath", description="Evaluate the multipath signature.") + p.add_argument("--euroc-root", default="data/raw/euroc") + p.add_argument("--tumvi-root", default="data/raw/tumvi") + p.add_argument("--csv", default=None, help="write one row per new run") + p.add_argument("--from-csv", default=None, help="rebuild the tables and page from a saved --csv file; no runs") + p.add_argument("--page", default=None, help="fill the multipath blocks of this Markdown page") + p.add_argument("--markdown", action="store_true", help="print the tables") + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.from_csv: + with Path(args.from_csv).open(newline="", encoding="utf-8") as fh: + rows = list(csv.DictReader(fh)) + else: + rows = [] + for ds, root in (("euroc", args.euroc_root), ("tumvi", args.tumvi_root)): + rows += sweep_rows(ds, root) + if args.csv: + write_csv(args.csv, rows) + print(f"wrote {args.csv}", file=sys.stderr) + tables = {name: fn(rows) for name, fn in BLOCKS.items()} + if args.page: + try: + for name, text in tables.items(): + write_block(args.page, name, text) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if args.markdown or not (args.csv or args.page): + print("\n\n".join(tables.values())) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) From 0d1b689e61b66af7ed0bc6fc8b947d410002d712 Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:16:47 +0200 Subject: [PATCH 2/3] feat(multipath): two rules and shipping criteria, fixed before the fresh runs Co-Authored-By: Claude Sonnet 5.5 --- docs/data/multipath_detector.csv | 171 ++++++++++++++++++++++++ src/navkit/fdir/multipath.py | 34 ++++- src/navkit/multipath_study.py | 218 +++++++++++++++++++++++-------- 3 files changed, 360 insertions(+), 63 deletions(-) create mode 100644 docs/data/multipath_detector.csv diff --git a/docs/data/multipath_detector.csv b/docs/data/multipath_detector.csv new file mode 100644 index 0000000..b9c4b42 --- /dev/null +++ b/docs/data/multipath_detector.csv @@ -0,0 +1,171 @@ +dataset,sequence,seed,kind,level,nis,ac1 +euroc,MH_01_easy,6,none,0.0,0.8729455406460268,-0.08967755783042249 +euroc,MH_01_easy,7,none,0.0,0.8579061651794184,-0.020419326855692456 +euroc,MH_01_easy,6,multipath,0.5,1.3224379157029524,0.003153124515192774 +euroc,MH_01_easy,7,multipath,0.5,1.2521245352715888,-0.06313805864765777 +euroc,MH_01_easy,6,multipath,1.0,2.5194806622276036,0.10467334415414868 +euroc,MH_01_easy,7,multipath,1.0,2.3359183149393563,-0.030708113247279577 +euroc,MH_01_easy,6,multipath,1.5,5.216767377937255,0.3729604582195119 +euroc,MH_01_easy,7,multipath,1.5,4.266192493035457,0.10348124756476675 +euroc,MH_01_easy,6,multipath,2.0,7.396272893776651,0.606650680503615 +euroc,MH_01_easy,7,multipath,2.0,5.4759235513666455,0.15986764273211448 +euroc,MH_02_easy,6,none,0.0,0.8780442872941122,-0.08456422725634903 +euroc,MH_02_easy,7,none,0.0,0.8580974021613955,-0.0198585508665714 +euroc,MH_02_easy,6,multipath,0.5,1.3247861475854925,0.004472838867291599 +euroc,MH_02_easy,7,multipath,0.5,1.2510157029136042,-0.06412485171448752 +euroc,MH_02_easy,6,multipath,1.0,2.526021490733104,0.10657487621612813 +euroc,MH_02_easy,7,multipath,1.0,2.350400042583259,-0.024668108943135116 +euroc,MH_02_easy,6,multipath,1.5,5.256720966624979,0.37925040002387916 +euroc,MH_02_easy,7,multipath,1.5,4.570242736740391,0.18158122195093646 +euroc,MH_02_easy,6,multipath,2.0,8.333232383851879,0.6834717461789408 +euroc,MH_02_easy,7,multipath,2.0,6.676063350571446,0.5077591424470911 +euroc,MH_03_medium,6,none,0.0,0.8841123397598145,-0.07400444567616465 +euroc,MH_03_medium,7,none,0.0,0.8903074308693689,0.015516417473938672 +euroc,MH_03_medium,6,multipath,0.5,1.3344593262559965,0.012717186225084562 +euroc,MH_03_medium,7,multipath,0.5,1.2881481053394126,-0.03414244272794145 +euroc,MH_03_medium,6,multipath,1.0,2.5633183011404452,0.11407970123437831 +euroc,MH_03_medium,7,multipath,1.0,2.383127085433388,-0.007379805201068074 +euroc,MH_03_medium,6,multipath,1.5,5.5146138757659395,0.527674407016822 +euroc,MH_03_medium,7,multipath,1.5,4.545464937918619,0.18476280146456914 +euroc,MH_03_medium,6,multipath,2.0,7.671071600844569,0.6628204553702178 +euroc,MH_03_medium,7,multipath,2.0,6.0545618573195865,0.4758724591461825 +euroc,MH_04_difficult,6,none,0.0,0.9056224508765464,-0.05273528615929723 +euroc,MH_04_difficult,7,none,0.0,0.8612082501150562,-0.015465687083370728 +euroc,MH_04_difficult,6,multipath,0.5,1.3421492680908178,0.017838735519366714 +euroc,MH_04_difficult,7,multipath,0.5,1.246738376439555,-0.06704684931410138 +euroc,MH_04_difficult,6,multipath,1.0,2.53702438988106,0.09583150584022801 +euroc,MH_04_difficult,7,multipath,1.0,2.31552879403063,-0.03803943723891981 +euroc,MH_04_difficult,6,multipath,1.5,6.98053505072452,0.6364241156216912 +euroc,MH_04_difficult,7,multipath,1.5,4.193905782862781,0.11007331984412706 +euroc,MH_04_difficult,6,multipath,2.0,8.31547193262591,0.6974137604842752 +euroc,MH_04_difficult,7,multipath,2.0,6.155646586469853,0.2286588024946095 +euroc,MH_05_difficult,6,none,0.0,0.8830572717881298,-0.07750665743222399 +euroc,MH_05_difficult,7,none,0.0,0.8750297365086727,-0.00010043143334992416 +euroc,MH_05_difficult,6,multipath,0.5,1.3293403919667195,0.007864746806772365 +euroc,MH_05_difficult,7,multipath,0.5,1.2669691286629867,-0.05052017724287749 +euroc,MH_05_difficult,6,multipath,1.0,2.5671754468753143,0.12039737684592898 +euroc,MH_05_difficult,7,multipath,1.0,2.3508340331343116,-0.02754618608503417 +euroc,MH_05_difficult,6,multipath,1.5,4.79860850454207,0.2686300737532429 +euroc,MH_05_difficult,7,multipath,1.5,4.442987879037336,0.16037210870217342 +euroc,MH_05_difficult,6,multipath,2.0,7.535464285547729,0.5709124493574161 +euroc,MH_05_difficult,7,multipath,2.0,5.568114607823142,0.1973975521708495 +euroc,V1_01_easy,6,none,0.0,0.878322264590034,-0.08311885015138212 +euroc,V1_01_easy,7,none,0.0,0.8705438107132198,-0.005237722770386122 +euroc,V1_01_easy,6,multipath,0.5,1.3286600748113426,0.007707928918722298 +euroc,V1_01_easy,7,multipath,0.5,1.2678465474474445,-0.049703137843329644 +euroc,V1_01_easy,6,multipath,1.0,2.6094434772328827,0.11755307511336686 +euroc,V1_01_easy,7,multipath,1.0,2.3630074302953568,-0.016808172751700254 +euroc,V1_01_easy,6,multipath,1.5,4.583957886555782,0.24685202697009667 +euroc,V1_01_easy,7,multipath,1.5,4.4782320595647,0.17153876801389611 +euroc,V1_01_easy,6,multipath,2.0,7.17719251812314,0.6299252853083327 +euroc,V1_01_easy,7,multipath,2.0,5.710159432918156,0.39266228841966266 +euroc,V1_02_medium,6,none,0.0,0.870522356436528,-0.09491921062810356 +euroc,V1_02_medium,7,none,0.0,0.8784637142482362,0.0028078946125648725 +euroc,V1_02_medium,6,multipath,0.5,1.306900018368878,-0.00959921362026438 +euroc,V1_02_medium,7,multipath,0.5,1.2643690835166308,-0.054530058284276144 +euroc,V1_02_medium,6,multipath,1.0,2.4924399544096087,0.08705056821409157 +euroc,V1_02_medium,7,multipath,1.0,2.350209841321231,-0.02821115442139086 +euroc,V1_02_medium,6,multipath,1.5,5.658469471217408,0.5893851303537317 +euroc,V1_02_medium,7,multipath,1.5,4.2489295045999365,0.08977737927254471 +euroc,V1_02_medium,6,multipath,2.0,6.484913058770965,0.6253767064765006 +euroc,V1_02_medium,7,multipath,2.0,5.576770806043002,0.13834216774290872 +euroc,V1_03_difficult,6,none,0.0,0.8937437130102173,-0.07001857916213598 +euroc,V1_03_difficult,7,none,0.0,0.8637388149273829,-0.0129122961697465 +euroc,V1_03_difficult,6,multipath,0.5,1.3364541732778215,0.01027743766094656 +euroc,V1_03_difficult,7,multipath,0.5,1.2506454836396796,-0.06486292153363922 +euroc,V1_03_difficult,6,multipath,1.0,2.5879055850654575,0.12571471489440575 +euroc,V1_03_difficult,7,multipath,1.0,2.3416975649030154,-0.026794724097191098 +euroc,V1_03_difficult,6,multipath,1.5,5.101575604543741,0.36837678624864084 +euroc,V1_03_difficult,7,multipath,1.5,4.410379717092746,0.1447882793108138 +euroc,V1_03_difficult,6,multipath,2.0,7.428679025246456,0.6245717156893063 +euroc,V1_03_difficult,7,multipath,2.0,6.9379658526557675,0.5405568977243476 +euroc,V2_01_easy,6,none,0.0,0.8877740203756711,-0.07237479957198434 +euroc,V2_01_easy,7,none,0.0,0.8710397517004305,-0.005032688371892086 +euroc,V2_01_easy,6,multipath,0.5,1.3314283544512175,0.009270135816765534 +euroc,V2_01_easy,7,multipath,0.5,1.2609636799706527,-0.056264879921284816 +euroc,V2_01_easy,6,multipath,1.0,2.5914789439453583,0.1281277720320132 +euroc,V2_01_easy,7,multipath,1.0,2.3991119691956535,0.012275574449911971 +euroc,V2_01_easy,6,multipath,1.5,4.7685520802622685,0.2755323235908758 +euroc,V2_01_easy,7,multipath,1.5,4.396908305233816,0.13035281312489033 +euroc,V2_01_easy,6,multipath,2.0,9.171885811704275,0.6911794760951431 +euroc,V2_01_easy,7,multipath,2.0,5.625374756725582,0.3886419191343642 +euroc,V2_02_medium,6,none,0.0,0.8842810902341595,-0.07776955476961968 +euroc,V2_02_medium,7,none,0.0,0.8941789052203634,0.019033792152768965 +euroc,V2_02_medium,6,multipath,0.5,1.3304044212928046,0.0073686261580382635 +euroc,V2_02_medium,7,multipath,0.5,1.2860971501636738,-0.036772214510843254 +euroc,V2_02_medium,6,multipath,1.0,2.519754766746979,0.09898940760189594 +euroc,V2_02_medium,7,multipath,1.0,2.333557114876127,-0.03637403187038564 +euroc,V2_02_medium,6,multipath,1.5,4.895594214658363,0.5523556845100758 +euroc,V2_02_medium,7,multipath,1.5,4.225969203466302,0.08669819605933839 +euroc,V2_02_medium,6,multipath,2.0,7.841578242097157,0.6593928944324531 +euroc,V2_02_medium,7,multipath,2.0,6.733176537881254,0.5070443813478209 +euroc,V2_03_difficult,6,none,0.0,0.8852339270361224,-0.07858188292693041 +euroc,V2_03_difficult,7,none,0.0,0.8696099016827934,-0.006389020454871797 +euroc,V2_03_difficult,6,multipath,0.5,1.341132573553643,0.014036001600465448 +euroc,V2_03_difficult,7,multipath,0.5,1.2566896418617757,-0.059275992816738 +euroc,V2_03_difficult,6,multipath,1.0,2.5471943204838516,0.10654433241312145 +euroc,V2_03_difficult,7,multipath,1.0,2.3473481698036434,-0.01966289930920687 +euroc,V2_03_difficult,6,multipath,1.5,4.993830355331673,0.3579151338278046 +euroc,V2_03_difficult,7,multipath,1.5,4.207173998408803,0.08766077502324873 +euroc,V2_03_difficult,6,multipath,2.0,8.684028306212651,0.7422982148223641 +euroc,V2_03_difficult,7,multipath,2.0,5.422239819015829,0.13009962939955005 +tumvi,room1,6,none,0.0,0.8859036498625822,-0.07110242838229776 +tumvi,room1,7,none,0.0,0.9900403434078916,0.11391664446962314 +tumvi,room1,6,multipath,0.5,1.3272856979320884,0.007788304632449773 +tumvi,room1,7,multipath,0.5,1.3919952358591239,0.0473213709587938 +tumvi,room1,6,multipath,1.0,2.5798999181618085,0.10697069153911976 +tumvi,room1,7,multipath,1.0,2.4849693603059118,0.02268927925819933 +tumvi,room1,6,multipath,1.5,4.776103438904558,0.24145221446232348 +tumvi,room1,7,multipath,1.5,4.723899281490536,0.15784866363322225 +tumvi,room1,6,multipath,2.0,5.583431101899297,0.34499926221413935 +tumvi,room1,7,multipath,2.0,6.819778385343713,0.6611624576574942 +tumvi,room2,6,none,0.0,0.8970875446444603,-0.05746606202480224 +tumvi,room2,7,none,0.0,0.8802829602376788,0.005588200835351593 +tumvi,room2,6,multipath,0.5,1.3304185537198439,0.010485038503710551 +tumvi,room2,7,multipath,0.5,1.2577180782720918,-0.0567780395864937 +tumvi,room2,6,multipath,1.0,2.8121085447752168,0.2031002660072002 +tumvi,room2,7,multipath,1.0,2.34881504003048,-0.03742797367109027 +tumvi,room2,6,multipath,1.5,4.6664112886560885,0.33916706728675516 +tumvi,room2,7,multipath,1.5,4.066341652315812,-0.01368832924566081 +tumvi,room2,6,multipath,2.0,6.561608864689838,0.485105127717403 +tumvi,room2,7,multipath,2.0,5.800262476158515,0.1234482806861592 +tumvi,room3,6,none,0.0,0.8872530148613523,-0.07454766952156847 +tumvi,room3,7,none,0.0,0.9070465834407134,0.03247200662820063 +tumvi,room3,6,multipath,0.5,1.3487576254914089,0.02334150559405832 +tumvi,room3,7,multipath,0.5,1.3041507308932152,-0.020131763737400363 +tumvi,room3,6,multipath,1.0,3.009174205578566,0.3783723567845301 +tumvi,room3,7,multipath,1.0,2.39398648317332,-0.01573774751406831 +tumvi,room3,6,multipath,1.5,6.010586284932088,0.6457425763707799 +tumvi,room3,7,multipath,1.5,4.064835442767552,-0.012010602720518472 +tumvi,room3,6,multipath,2.0,7.226690495573448,0.5349192972028075 +tumvi,room3,7,multipath,2.0,6.553941855373753,0.229090485801841 +tumvi,room4,6,none,0.0,0.8742175875230714,-0.0889342266384047 +tumvi,room4,7,none,0.0,0.8735725903236341,-0.0027997987543072413 +tumvi,room4,6,multipath,0.5,1.3229725095185805,0.003135650115776606 +tumvi,room4,7,multipath,0.5,1.27477613433796,-0.04456762007967868 +tumvi,room4,6,multipath,1.0,2.5990526202731936,0.12155044872563406 +tumvi,room4,7,multipath,1.0,2.4233578035678343,-5.800556572973478e-06 +tumvi,room4,6,multipath,1.5,4.637206489352162,0.45804998426364774 +tumvi,room4,7,multipath,1.5,4.036207064523542,-0.014115063395224899 +tumvi,room4,6,multipath,2.0,7.6308896090257265,0.5175087955721794 +tumvi,room4,7,multipath,2.0,5.6714556823903015,0.18474316779883804 +tumvi,room5,6,none,0.0,0.9023521266498612,-0.05134241853494778 +tumvi,room5,7,none,0.0,0.9033374235243542,0.0307319215319432 +tumvi,room5,6,multipath,0.5,1.356025022990658,0.028792121742452003 +tumvi,room5,7,multipath,0.5,1.2999558524937636,-0.0249352642196432 +tumvi,room5,6,multipath,1.0,2.57372573966176,0.10710800907195596 +tumvi,room5,7,multipath,1.0,2.453937673402042,0.01167811411310755 +tumvi,room5,6,multipath,1.5,4.464507273411354,0.18777252641675715 +tumvi,room5,7,multipath,1.5,4.168688993352085,0.03215353803330465 +tumvi,room5,6,multipath,2.0,5.929219296594951,0.32005558249024696 +tumvi,room5,7,multipath,2.0,5.875657652197842,0.11995973219839726 +tumvi,room6,6,none,0.0,0.8829730359149859,-0.0761803996641017 +tumvi,room6,7,none,0.0,0.8925588723198935,0.018493710197505453 +tumvi,room6,6,multipath,0.5,1.3393755741431514,0.015855389049321975 +tumvi,room6,7,multipath,0.5,1.282873706628603,-0.03770404912712259 +tumvi,room6,6,multipath,1.0,2.612693999796492,0.12825765873994024 +tumvi,room6,7,multipath,1.0,2.391820416032584,-0.01804725473989612 +tumvi,room6,6,multipath,1.5,4.705126745506019,0.6053236503514928 +tumvi,room6,7,multipath,1.5,4.051458329515348,0.004007504882002368 +tumvi,room6,6,multipath,2.0,7.4825687946116455,0.7315870256420278 +tumvi,room6,7,multipath,2.0,6.601230562515067,0.1946754537242388 diff --git a/src/navkit/fdir/multipath.py b/src/navkit/fdir/multipath.py index 01207bf..c2a76d2 100644 --- a/src/navkit/fdir/multipath.py +++ b/src/navkit/fdir/multipath.py @@ -18,8 +18,12 @@ import numpy as np -#: A window whose mean NIS is at least this is inflated (a healthy channel is about 1). +#: Two rules, fixed before the evaluation on fresh runs. ``sensitive``: a window whose mean NIS is at least 2 (a healthy +#: channel is about 1). ``specific``: at least 6, the value the fault-classification tree learned on its training runs. +#: The first catches smaller multipath and also calls a small step. The second does neither. NIS_MIN = 2.0 +NIS_MIN_SPECIFIC = 6.0 +RULES = ("sensitive", "specific") #: A window whose lag-1 autocorrelation of the whitened innovation is above this is a persistent offset, not noise. AC1_MAX = 0.8 #: Fewer innovations than this and no call is made. @@ -33,7 +37,13 @@ class Signature: nis: float ac1: float n: int - multipath: bool + sensitive: bool + specific: bool + + @property + def multipath(self) -> bool: + """The call under the specific rule, which is the default.""" + return self.specific def signature(innovations: Sequence[tuple[float, np.ndarray, np.ndarray]]) -> Signature | None: @@ -48,10 +58,24 @@ def signature(innovations: Sequence[tuple[float, np.ndarray, np.ndarray]]) -> Si z = np.stack([np.linalg.solve(np.linalg.cholesky(i[2]), i[1]) for i in innovations]) nis = float(d2.mean() / 3.0) ac1 = float((z[:-1] * z[1:]).sum() / max(float((z * z).sum()), 1e-12)) - return Signature(nis=nis, ac1=ac1, n=len(innovations), multipath=nis >= NIS_MIN and ac1 <= AC1_MAX) + return Signature( + nis=nis, + ac1=ac1, + n=len(innovations), + sensitive=call(nis, ac1, "sensitive"), + specific=call(nis, ac1, "specific"), + ) + + +def call(nis: float, ac1: float, rule: str = "specific") -> bool: + """The call of one rule on the two statistics of a window.""" + if rule not in RULES: + raise ValueError(f"rule must be one of {RULES}, got {rule!r}") + floor = NIS_MIN if rule == "sensitive" else NIS_MIN_SPECIFIC + return bool(nis >= floor and ac1 <= AC1_MAX) -def is_multipath(innovations: Sequence[tuple[float, np.ndarray, np.ndarray]]) -> bool: +def is_multipath(innovations: Sequence[tuple[float, np.ndarray, np.ndarray]], rule: str = "specific") -> bool: """True when the window looks like multipath; False when it does not or when it is too short to say.""" sig = signature(innovations) - return bool(sig and sig.multipath) + return bool(sig and call(sig.nis, sig.ac1, rule)) diff --git a/src/navkit/multipath_study.py b/src/navkit/multipath_study.py index a4b8f68..9815909 100644 --- a/src/navkit/multipath_study.py +++ b/src/navkit/multipath_study.py @@ -1,10 +1,18 @@ """How small a multipath fault does the signature catch, and does it call anything else? -The thresholds of :mod:`navkit.fdir.multipath` were fixed before this study. This evaluates them on runs the -fault-classification study never used for the thresholds: its confirmation runs (544, unseen fault sizes and seeds), -plus new runs here: multipath at 0.5 to 2 m, to find the floor, and extra control runs, on seeds 6 and 7. +:mod:`navkit.fdir.multipath` has two rules, fixed before they were tested on fresh runs: ``sensitive`` (NIS at least 2) +and ``specific`` (NIS at least 6), both with a lag-1 autocorrelation of at most 0.8. -Simulated GNSS, injected faults, two datasets, the calibrated bias-walk setting. +Three sets of runs, none of them used to set a threshold except where stated: + +* the **floor** runs: multipath of 0.5 to 2 m and extra controls, seeds 6 and 7; +* the **confirmation** runs of the fault-classification study (unseen sizes and seeds), read from its CSV; +* the **fresh** runs: new seeds (8 and 9), with the mixed set of cases listed in ``FRESH_CASES``. The criteria for + shipping a rule were written before this set was run (``CRITERIA``). + +The 6 in the specific rule came from the classification tree's training runs, so the training runs are not evidence for +it. The floor and confirmation runs were seen before the fresh set was designed. Only the fresh set is a clean test of +the rule that is chosen. Simulated GNSS, injected faults, two datasets, the calibrated bias-walk setting. """ from __future__ import annotations @@ -24,17 +32,43 @@ from .fdir import multipath as mp FLOOR_LEVELS_M = (0.5, 1.0, 1.5, 2.0) -SEEDS = (6, 7) +FLOOR_SEEDS = (6, 7) +FRESH_SEEDS = (8, 9) +#: (kind, level). ``none`` is the control. +FRESH_CASES: tuple[tuple[str, float], ...] = ( + ("none", 0.0), + ("multipath", 1.0), + ("multipath", 2.0), + ("multipath", 3.0), + ("multipath", 5.0), + ("step", 5.0), + ("step", 10.0), + ("ramp", 3.0), + ("ramp", 5.0), + ("accel_bias", 0.7), + ("gyro_bias", 0.02), +) +#: Written before the fresh runs: a rule is shipped only if, on the fresh runs, it calls at least this share of the +#: multipath runs of 3 m and above, and calls no more than this share of the runs of every other kind. +CRITERIA = {"recall_at_3m_and_above": 0.95, "false_call_rate_other": 0.02} +CLASS_OF = { + "none": "control", + "multipath": "multipath", + "ramp": "spoofing", + "step": "spoofing", + "accel_bias": "degradation", + "gyro_bias": "degradation", +} CONFIRM_CSV = "docs/data/fault_classification_confirm.csv" -CSV_COLUMNS = ("dataset", "sequence", "seed", "kind", "level", "nis", "ac1", "multipath") +CSV_COLUMNS = ("dataset", "sequence", "seed", "kind", "level", "nis", "ac1") -def run_case(seq: Any, config: str, seed: int, level: float) -> dict[str, Any] | None: - """One run: multipath of ``level`` metres (0 for a control), and the signature of the window after the onset.""" +def run_case(seq: Any, config: str, seed: int, kind: str, level: float) -> dict[str, Any] | None: + """One run: the fault (``none`` for a control) and the two statistics of the window after the onset.""" t_start = max(float(seq.imu.t[0]), float(seq.truth.t[0])) if float(min(seq.imu.t[-1], seq.truth.t[-1])) - t_start < ONSET_S + WINDOW_S + 5.0: return None - fault = ("multipath", ONSET_S, level) if level > 0 else None + fault = None if kind == "none" else (kind, ONSET_S, level) inn: list[Any] = [] run_sequence(seq, RunOptions(seed=seed, fault=fault, **_ALL_CONFIGS[config]), innovations_out=inn) onset = t_start + ONSET_S @@ -45,28 +79,33 @@ def run_case(seq: Any, config: str, seed: int, level: float) -> dict[str, Any] | "dataset": seq.slug, "sequence": seq.name, "seed": seed, - "kind": "multipath" if level > 0 else "none", + "kind": kind, "level": level, "nis": sig.nis, "ac1": sig.ac1, - "multipath": sig.multipath, } -def sweep_rows(dataset: str, root: str | Path) -> list[dict[str, Any]]: +def sweep_rows( + dataset: str, root: str | Path, cases: tuple[tuple[str, float], ...], seeds: tuple[int, ...] +) -> list[dict[str, Any]]: loader, _ = dataset_loader(dataset) rows: list[dict[str, Any]] = [] for name in sorted(d for d in os.listdir(root) if (Path(root) / d).is_dir()): seq = loader(root, name) - for level in (0.0, *FLOOR_LEVELS_M): - for seed in SEEDS: - row = run_case(seq, SETTINGS[dataset], seed, level) + for kind, level in cases: + for seed in seeds: + row = run_case(seq, SETTINGS[dataset], seed, kind, level) if row is not None: rows.append(row) print(f"{dataset} {name}", file=sys.stderr, flush=True) return rows +def floor_cases() -> tuple[tuple[str, float], ...]: + return (("none", 0.0), *(("multipath", lv) for lv in FLOOR_LEVELS_M)) + + # --------------------------------------------------------------------- tables @@ -74,8 +113,12 @@ def _pct(x: float) -> str: return f"{100 * x:.0f}%" -def _called(row: dict[str, Any]) -> bool: - return str(row["multipath"]) in ("True", "true", "1") +def called(row: dict[str, Any], rule: str) -> bool: + return mp.call(float(row["nis"]), float(row["ac1"]), rule) + + +def cls_of(row: dict[str, Any]) -> str: + return CLASS_OF[row["kind"]] def upper_bound(n: int) -> float: @@ -84,55 +127,106 @@ def upper_bound(n: int) -> float: def confirm_rows(path: str | Path = CONFIRM_CSV) -> list[dict[str, Any]]: - """The confirmation runs of the fault-classification study, with the call recomputed from the stored features.""" - out = [] - for r in rows_from_csv(path): - nis, ac1 = float(r["nis"]), float(r["ac1"]) - out.append({**r, "multipath": nis >= mp.NIS_MIN and ac1 <= mp.AC1_MAX}) - return out + """The confirmation runs of the fault-classification study, as rows of this study.""" + kind_of = {"none": "none"} + return [{**r, "kind": kind_of.get(r["kind"], r["kind"])} for r in rows_from_csv(path)] -def confirm_markdown(rows: list[dict[str, Any]]) -> str: - cr = confirm_rows() - lines = ["| Class | Runs | Called multipath |", "|---|---:|---:|"] - for cls in ("control", "multipath", "spoofing", "degradation"): - sel = [r for r in cr if r["cls"] == cls] - lines.append(f"| {cls} | {len(sel)} | {_pct(sum(_called(r) for r in sel) / len(sel))} |") - return "\n".join(lines) +def _share(sel: list[dict[str, Any]], rule: str) -> float: + return sum(called(r, rule) for r in sel) / len(sel) if sel else float("nan") -def floor_markdown(rows: list[dict[str, Any]]) -> str: +def floor_markdown(floor: list[dict[str, Any]], fresh: list[dict[str, Any]]) -> str: lines = [ - "| Multipath m | Runs | Called multipath | Median NIS | Median lag-1 autocorrelation |", - "|---:|---:|---:|---:|---:|", + "| Multipath m | Runs | Sensitive rule calls | Specific rule calls | Median NIS " + "| Median lag-1 autocorrelation |", + "|---:|---:|---:|---:|---:|---:|", ] for level in (0.0, *FLOOR_LEVELS_M): - sel = [r for r in rows if float(r["level"]) == level] + sel = [r for r in floor if float(r["level"]) == level] if sel: lines.append( - f"| {level:g} | {len(sel)} | {_pct(sum(_called(r) for r in sel) / len(sel))} | " + f"| {level:g} | {len(sel)} | {_pct(_share(sel, 'sensitive'))} | {_pct(_share(sel, 'specific'))} | " f"{np.median([float(r['nis']) for r in sel]):.2f} | {np.median([float(r['ac1']) for r in sel]):.2f} |" ) return "\n".join(lines) -def false_alarm_markdown(rows: list[dict[str, Any]]) -> str: - cr = [r for r in confirm_rows() if r["cls"] == "control"] - new = [r for r in rows if float(r["level"]) == 0.0] +def confirm_markdown(floor: list[dict[str, Any]], fresh: list[dict[str, Any]]) -> str: + cr = confirm_rows() lines = [ - "| Control runs | Runs | Called multipath | 95% upper bound on the false-alarm rate |", + "| Class | Runs | Sensitive rule calls multipath | Specific rule calls multipath |", "|---|---:|---:|---:|", ] - for label, sel in (("confirmation", cr), ("new", new), ("both", cr + new)): - hits = sum(_called(r) for r in sel) - bound = _pct(upper_bound(len(sel))) if hits == 0 else "see the count" - lines.append(f"| {label} | {len(sel)} | {hits} | {bound} |") + for cls in ("control", "multipath", "spoofing", "degradation"): + sel = [r for r in cr if cls_of(r) == cls] + lines.append(f"| {cls} | {len(sel)} | {_pct(_share(sel, 'sensitive'))} | {_pct(_share(sel, 'specific'))} |") + return "\n".join(lines) + + +def fresh_markdown(floor: list[dict[str, Any]], fresh: list[dict[str, Any]]) -> str: + lines = [ + "| Class | Fault | Level | Runs | Sensitive rule calls multipath | Specific rule calls multipath |", + "|---|---|---:|---:|---:|---:|", + ] + for kind, level in FRESH_CASES: + sel = [r for r in fresh if r["kind"] == kind and float(r["level"]) == level] + if sel: + lines.append( + f"| {CLASS_OF[kind]} | {kind} | {level:g} | {len(sel)} | {_pct(_share(sel, 'sensitive'))} | " + f"{_pct(_share(sel, 'specific'))} |" + ) + return "\n".join(lines) + + +def criteria_results(fresh: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """The fresh-run recall and false-call rate of each rule, and whether the criteria are met.""" + out: dict[str, dict[str, Any]] = {} + big = [r for r in fresh if r["kind"] == "multipath" and float(r["level"]) >= 3.0] + other = [r for r in fresh if r["kind"] != "multipath"] + for rule in mp.RULES: + recall, false_rate = _share(big, rule), _share(other, rule) + out[rule] = { + "recall": recall, + "false_call_rate": false_rate, + "met": recall >= CRITERIA["recall_at_3m_and_above"] and false_rate <= CRITERIA["false_call_rate_other"], + } + return out + + +def criteria_markdown(floor: list[dict[str, Any]], fresh: list[dict[str, Any]]) -> str: + res = criteria_results(fresh) + lines = [ + f"| Rule | Recall, multipath of 3 m and above (needs {_pct(CRITERIA['recall_at_3m_and_above'])}) | " + f"Called multipath, all other runs including controls (needs at most {_pct(CRITERIA['false_call_rate_other'])}) " + "| Criteria met |", + "|---|---:|---:|---|", + ] + for rule in mp.RULES: + r = res[rule] + lines.append(f"| {rule} | {_pct(r['recall'])} | {_pct(r['false_call_rate'])} | {'yes' if r['met'] else 'no'} |") + return "\n".join(lines) + + +def false_alarm_markdown(floor: list[dict[str, Any]], fresh: list[dict[str, Any]]) -> str: + ctrl = { + "floor": [r for r in floor if r["kind"] == "none"], + "confirmation": [r for r in confirm_rows() if r["kind"] == "none"], + "fresh": [r for r in fresh if r["kind"] == "none"], + } + lines = ["| Control runs | Runs | Sensitive rule calls | Specific rule calls |", "|---|---:|---:|---:|"] + everything = [r for sel in ctrl.values() for r in sel] + for label, sel in (*ctrl.items(), ("all", everything)): + s, p = (sum(called(r, rule) for r in sel) for rule in mp.RULES) + lines.append(f"| {label} | {len(sel)} | {s} | {p} |") return "\n".join(lines) BLOCKS = { - "multipath-confirm": confirm_markdown, "multipath-floor": floor_markdown, + "multipath-confirm": confirm_markdown, + "multipath-fresh": fresh_markdown, + "multipath-criteria": criteria_markdown, "multipath-false-alarms": false_alarm_markdown, } @@ -143,11 +237,16 @@ def write_csv(path: str | Path, rows: list[dict[str, Any]]) -> None: target = Path(path) target.parent.mkdir(parents=True, exist_ok=True) with target.open("w", newline="", encoding="utf-8") as fh: - writer = csv.DictWriter(fh, fieldnames=CSV_COLUMNS) + writer = csv.DictWriter(fh, fieldnames=CSV_COLUMNS, extrasaction="ignore") writer.writeheader() writer.writerows(rows) +def read_csv(path: str | Path) -> list[dict[str, Any]]: + with Path(path).open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + def write_block(path: str | Path, name: str, block: str) -> None: page = Path(path) text = page.read_text(encoding="utf-8") @@ -163,8 +262,10 @@ def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="navkit sweep multipath", description="Evaluate the multipath signature.") p.add_argument("--euroc-root", default="data/raw/euroc") p.add_argument("--tumvi-root", default="data/raw/tumvi") - p.add_argument("--csv", default=None, help="write one row per new run") - p.add_argument("--from-csv", default=None, help="rebuild the tables and page from a saved --csv file; no runs") + p.add_argument("--set", choices=("floor", "fresh"), default=None, help="run one set and write it with --csv") + p.add_argument("--csv", default=None, help="write one row per run of the set that was run") + p.add_argument("--floor-csv", default="docs/data/multipath_detector.csv") + p.add_argument("--fresh-csv", default="docs/data/multipath_fresh.csv") p.add_argument("--page", default=None, help="fill the multipath blocks of this Markdown page") p.add_argument("--markdown", action="store_true", help="print the tables") return p @@ -172,17 +273,18 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) - if args.from_csv: - with Path(args.from_csv).open(newline="", encoding="utf-8") as fh: - rows = list(csv.DictReader(fh)) - else: - rows = [] + if args.set: + cases = floor_cases() if args.set == "floor" else FRESH_CASES + seeds = FLOOR_SEEDS if args.set == "floor" else FRESH_SEEDS + rows: list[dict[str, Any]] = [] for ds, root in (("euroc", args.euroc_root), ("tumvi", args.tumvi_root)): - rows += sweep_rows(ds, root) - if args.csv: - write_csv(args.csv, rows) - print(f"wrote {args.csv}", file=sys.stderr) - tables = {name: fn(rows) for name, fn in BLOCKS.items()} + rows += sweep_rows(ds, root, cases, seeds) + target = args.csv or (args.floor_csv if args.set == "floor" else args.fresh_csv) + write_csv(target, rows) + print(f"wrote {target}", file=sys.stderr) + return 0 + floor, fresh = read_csv(args.floor_csv), read_csv(args.fresh_csv) + tables = {name: fn(floor, fresh) for name, fn in BLOCKS.items()} if args.page: try: for name, text in tables.items(): @@ -190,7 +292,7 @@ def main(argv: list[str] | None = None) -> int: except ValueError as exc: print(f"error: {exc}", file=sys.stderr) return 1 - if args.markdown or not (args.csv or args.page): + if args.markdown or not args.page: print("\n\n".join(tables.values())) return 0 From afb327c39b04d568e7f6b3a08e5774d66ad40db3 Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:52:03 +0200 Subject: [PATCH 3/3] feat(multipath): evaluate the multipath signature on fresh runs; ADR-0022 Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 + CONSTRAINTS.md | 12 +- CONTRIBUTING.md | 2 +- ROADMAP.md | 4 +- docs/adr/0022-multipath-signature.md | 33 +++ docs/adr/index.md | 1 + docs/data/metrics.json | 10 +- docs/data/multipath_fresh.csv | 375 +++++++++++++++++++++++++++ docs/multipath.md | 123 +++++++++ mkdocs.yml | 2 + src/navkit/multipath_study.py | 13 +- tests/test_multipath.py | 132 ++++++++++ 12 files changed, 696 insertions(+), 17 deletions(-) create mode 100644 docs/adr/0022-multipath-signature.md create mode 100644 docs/data/multipath_fresh.csv create mode 100644 docs/multipath.md create mode 100644 tests/test_multipath.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e0c637..fa33dff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,12 @@ this project uses [semantic versioning](https://semver.org/). The Track A exit test is not met. [ADR-0021](docs/adr/0021-fault-classification-from-innovations.md) is accepted: ship a multipath detector at most, and report every other detected fault as cause unknown. +- `navkit.fdir.multipath` and [`docs/multipath.md`](docs/multipath.md): a multipath signature (NIS and lag-1 + autocorrelation of a window of GNSS innovations) in a sensitive and a specific rule, with shipping criteria written + before the fresh runs. The specific rule met them: it called all multipath of 3 m and above and none of the other + runs, and none of 136 controls. The sensitive rule did not (14% of the other runs). It is a function, not wired into + the filter. [ADR-0022](docs/adr/0022-multipath-signature.md) is accepted. `navkit sweep multipath` runs the study. + ### Changed - Blocker B4 is resolved by retiring the claim. The two tests that compared a published Basalt estimate of TUM VI diff --git a/CONSTRAINTS.md b/CONSTRAINTS.md index a2a5934..d0dd408 100644 --- a/CONSTRAINTS.md +++ b/CONSTRAINTS.md @@ -4,9 +4,9 @@ Invariants this project holds itself to. Every entry is either currently true or is a stated blocker; nothing here is aspirational. Ratchets are measured, not estimated, and the command to re-measure each one is given. -Last measured: 1245 tests collected, all of which pass except the deliberate -xfails and the skips that `pytest -rs` lists, and 94.1% -line coverage (8708/9253 executable lines) on **CPython 3.13**, measured +Last measured: 1259 tests collected, all of which pass except the deliberate +xfails and the skips that `pytest -rs` lists, and 94.2% +line coverage (9039/9598 executable lines) on **CPython 3.13**, measured with `python scripts/coverage_report.py`. The figures come from `docs/data/metrics.json`, which `scripts/metrics.py` checks in CI. Coverage is quoted from that script alone: it is the project's dependency-free tracer (S1 forbids adding `pytest-cov`), and a number taken from a different tool is not comparable with the module figures @@ -83,9 +83,9 @@ These are floors, not goals. Each must not regress; raising one is welcome. | Ratchet | Floor | Current (CPython 3.13) | Re-measure with | |---|---:|---:|---| -| Tests collected | 405 | 1245 | `pytest`, `scripts/metrics.py --check` | +| Tests collected | 405 | 1259 | `pytest`, `scripts/metrics.py --check` | | Tests passing | 300 | all collected, except the deliberate xfails and the listed skips | `pytest -rs` | -| Line coverage | 75% | 94.1% | `scripts/coverage_report.py`, `scripts/metrics.py --check-coverage` | +| Line coverage | 75% | 94.2% | `scripts/coverage_report.py`, `scripts/metrics.py --check-coverage` | | `io/trajectory.py` coverage | 85% | 95.0% | as above | | `analysis/findings.py` coverage | 80% | 99.7% | as above | | `config.py` coverage | 80% | 98.4% | as above | @@ -104,7 +104,7 @@ These are floors, not goals. Each must not regress; raising one is welcome. | `outage_sweep.py` coverage | 80% | 98.4% | as above | | `fdir/nis_monitor.py` coverage | 80% | 98.8% | as above | | `degrade/` coverage | 50% | 78.0% (`config.py`) / 84.2% (`inject.py`) | as above | -| Docs | README + architecture + calibration + every ADR | 21 ADRs, all in the index | `scripts/metrics.py --check` | +| Docs | README + architecture + calibration + every ADR | 22 ADRs, all in the index | `scripts/metrics.py --check` | | Documented tables match the generated benchmark | exact | yes | `scripts/check_doc_tables.py` | | Open blockers documented | all | see ROADMAP | manual | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c7feba..b9121d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,7 +70,7 @@ mypy src --ignore-missing-imports `ruff format src tests scripts` applies formatting; the `--check` form is the gate. -The suite collects 1245 tests and all of them pass on a fresh clone, except the deliberate +The suite collects 1259 tests and all of them pass on a fresh clone, except the deliberate xfails and the skips. `pytest -rs` lists the skips: two need TUM VI reference data that is not vendored (see below), and the doc-table check runs once `python scripts/run_benchmark.py` has written `results/benchmark.json`. The count and the other numbers the docs quote are in diff --git a/ROADMAP.md b/ROADMAP.md index 8978657..0a390dd 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,7 +4,7 @@ Sequenced by dependency, not by ambition. Each item names its exit test. Status markers: `[x]` done, `[~]` in progress, `[ ]` not started, `[!]` blocked. -Measured baseline: 1245 tests collected and 94.1% line coverage on CPython +Measured baseline: 1259 tests collected and 94.2% line coverage on CPython 3.13 (`docs/data/metrics.json`, from `scripts/metrics.py` and `scripts/coverage_report.py`). Every number below re-measures against that, not against a remembered value. @@ -203,6 +203,8 @@ that has already been wrong. This track moves detection forward. - [~] Per-sensor classification from the GNSS innovations is measured ([fault classification](docs/fault_classification.md), ADR-0021 accepted): multipath is separable, spoofing is called in about three quarters of runs, weak faults are missed, and the control false-alarm rate is 18% in cross-validation and 35% on unseen runs. +- [x] Multipath signature as a function, not wired into the filter ([multipath](docs/multipath.md), ADR-0022): the + specific rule calls multipath of about 3 m and above on injected noise and nothing else on fresh runs. - [ ] Per-sensor detection of multipath (elevated innovation variance without a mean shift), spoofing (innovation consistent but GNSS-internally inconsistent, e.g. against the IMU-predicted position), and sensor diff --git a/docs/adr/0022-multipath-signature.md b/docs/adr/0022-multipath-signature.md new file mode 100644 index 0000000..57e2e66 --- /dev/null +++ b/docs/adr/0022-multipath-signature.md @@ -0,0 +1,33 @@ +# ADR-0022: A multipath signature on the GNSS innovations + +- Status: accepted +- Date: 2026-10 +- Related: [ADR-0021](0021-fault-classification-from-innovations.md), [ADR-0005](0005-chi-square-fdir-gating.md), [multipath](../multipath.md) + +## Context + +ADR-0021 chose to ship at most a multipath detector. A rule of two numbers on a window of GNSS innovations (NIS and +lag-1 autocorrelation) was tested in two versions, with the shipping criteria written before the fresh runs. + +## Options + +1. **The sensitive rule** (NIS at least 2). It sees multipath from 1 m and calls small steps multipath too. +2. **The specific rule** (NIS at least 6). It sees multipath from about 3 m and called nothing else on the fresh runs. +3. **Wire a detector into the filter.** The filter would report multipath as a cause on the FDIR event. + +## Decision + +Option 2, as a function and not as filter behaviour. `navkit.fdir.multipath.signature` and `is_multipath` use the +specific rule by default. The sensitive rule stays available as an argument, so the trade-off can be seen. Nothing in +the estimator or the FDIR layer calls it, and no FDIR event changes. Wiring it in (option 3) is a separate step with +its own ADR, because it changes what a user of the filter is told. + +The criteria were met on fresh runs by the specific rule only: 100% of the multipath runs of 3 m and above, and none +of the other runs. The floor is about 3 m. + +## Consequences + +The claim is limited to injected white noise on simulated GNSS, two datasets and 20 s windows. Real multipath is +correlated and may exceed the autocorrelation limit. A real-receiver check is not done. The Track A exit test is +still not met: the multipath call alone does not give 95% detection per type at zero false alarms for spoofing and +degradation. diff --git a/docs/adr/index.md b/docs/adr/index.md index bd010ff..9eda882 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -26,3 +26,4 @@ what it cost, and what was rejected. | [ADR-0019](0019-slow-ramp-spoofing.md) | Slow-ramp GNSS spoofing is not detectable with IMU and GNSS alone | accepted | | [ADR-0020](0020-second-source-against-a-ramp.md) | A second source against a slow-ramp spoof | accepted | | [ADR-0021](0021-fault-classification-from-innovations.md) | Fault classification from the GNSS innovations | accepted | +| [ADR-0022](0022-multipath-signature.md) | A multipath signature on the GNSS innovations | accepted | diff --git a/docs/data/metrics.json b/docs/data/metrics.json index 318fe4c..53c5bd9 100644 --- a/docs/data/metrics.json +++ b/docs/data/metrics.json @@ -1,8 +1,8 @@ { - "adr_count": 21, - "coverage_lines_hit": 8708, - "coverage_lines_total": 9253, - "coverage_percent": 94.11, + "adr_count": 22, + "coverage_lines_hit": 9039, + "coverage_lines_total": 9598, + "coverage_percent": 94.18, "coverage_python": "3.13", - "tests_collected": 1245 + "tests_collected": 1259 } diff --git a/docs/data/multipath_fresh.csv b/docs/data/multipath_fresh.csv new file mode 100644 index 0000000..26927a3 --- /dev/null +++ b/docs/data/multipath_fresh.csv @@ -0,0 +1,375 @@ +dataset,sequence,seed,kind,level,nis,ac1 +euroc,MH_01_easy,8,none,0.0,1.0821274255766509,-0.04671833479131504 +euroc,MH_01_easy,9,none,0.0,0.8243462274684994,-0.012746163179251719 +euroc,MH_01_easy,8,multipath,1.0,2.536612408073726,0.0005856922206142571 +euroc,MH_01_easy,9,multipath,1.0,2.1707144813623556,0.002642963401218713 +euroc,MH_01_easy,8,multipath,2.0,5.695989807189061,0.480933261308903 +euroc,MH_01_easy,9,multipath,2.0,6.153747616001855,0.24039086014795674 +euroc,MH_01_easy,8,multipath,3.0,7.752463624157213,0.5425451232415663 +euroc,MH_01_easy,9,multipath,3.0,7.888722899216976,0.4003436450816639 +euroc,MH_01_easy,8,multipath,5.0,19.234931718908815,0.5920531037833727 +euroc,MH_01_easy,9,multipath,5.0,13.732998460452206,0.25818710510607573 +euroc,MH_01_easy,8,step,5.0,2.3543032748817656,0.5376334730117975 +euroc,MH_01_easy,9,step,5.0,1.813724654578613,0.5398353236634102 +euroc,MH_01_easy,8,step,10.0,6.78354502595331,0.8396571705064111 +euroc,MH_01_easy,9,step,10.0,5.60285981807625,0.8351384202817632 +euroc,MH_01_easy,8,ramp,3.0,1.506397837239683,0.24944084454664783 +euroc,MH_01_easy,9,ramp,3.0,1.1999661277038378,0.30360059185249466 +euroc,MH_01_easy,8,ramp,5.0,15.083102082602908,0.9677551258619294 +euroc,MH_01_easy,9,ramp,5.0,17.392105453033636,0.9778951962263781 +euroc,MH_01_easy,8,accel_bias,0.7,1.3519491090293758,0.16646692270400953 +euroc,MH_01_easy,9,accel_bias,0.7,1.0501784996825232,0.20396394248740266 +euroc,MH_01_easy,8,gyro_bias,0.02,1.8379814197442939,0.39706179359826405 +euroc,MH_01_easy,9,gyro_bias,0.02,1.4190918073711278,0.4215098366631078 +euroc,MH_02_easy,8,none,0.0,1.085410162066888,-0.04408349163776609 +euroc,MH_02_easy,9,none,0.0,0.8218417403471096,-0.015487176321199458 +euroc,MH_02_easy,8,multipath,1.0,2.535420217387666,-0.0019778354392267824 +euroc,MH_02_easy,9,multipath,1.0,2.1560137616134925,-0.003760335929817559 +euroc,MH_02_easy,8,multipath,2.0,6.023274294536911,0.44142810110449726 +euroc,MH_02_easy,9,multipath,2.0,6.414972620729043,0.35520835168565185 +euroc,MH_02_easy,8,multipath,3.0,8.11140339054611,0.500635942584008 +euroc,MH_02_easy,9,multipath,3.0,8.053030317805792,0.4194019715237534 +euroc,MH_02_easy,8,multipath,5.0,19.725838120245022,0.6076293562963023 +euroc,MH_02_easy,9,multipath,5.0,16.74679796384878,0.43352909040751353 +euroc,MH_02_easy,8,step,5.0,2.6605277618500693,0.600483284155907 +euroc,MH_02_easy,9,step,5.0,1.9478194327106626,0.5702221064533322 +euroc,MH_02_easy,8,step,10.0,7.769836901987955,0.8703579360682141 +euroc,MH_02_easy,9,step,10.0,6.094075476284012,0.8482475869270653 +euroc,MH_02_easy,8,ramp,3.0,1.5508878174922263,0.2706058004441704 +euroc,MH_02_easy,9,ramp,3.0,1.18701730579669,0.29679528990204157 +euroc,MH_02_easy,8,ramp,5.0,18.279585544366054,0.9730182191524818 +euroc,MH_02_easy,9,ramp,5.0,16.91770900741455,0.9772326554654635 +euroc,MH_02_easy,8,accel_bias,0.7,1.3708329457522497,0.17744404406844297 +euroc,MH_02_easy,9,accel_bias,0.7,1.0425605220793197,0.19812462450473198 +euroc,MH_02_easy,8,gyro_bias,0.02,1.859104157165832,0.4028723058893664 +euroc,MH_02_easy,9,gyro_bias,0.02,1.40248801628589,0.4146626830438813 +euroc,MH_03_medium,8,none,0.0,1.0914389519173302,-0.03856945818557428 +euroc,MH_03_medium,9,none,0.0,0.8369784182610317,0.004949497776710055 +euroc,MH_03_medium,8,multipath,1.0,2.570035461191862,0.022717694397076507 +euroc,MH_03_medium,9,multipath,1.0,2.163700997896477,-0.0008794656571732043 +euroc,MH_03_medium,8,multipath,2.0,5.823330334488282,0.5531320391280293 +euroc,MH_03_medium,9,multipath,2.0,5.271791704945721,0.07919602448167043 +euroc,MH_03_medium,8,multipath,3.0,7.811256908043545,0.40427756847115237 +euroc,MH_03_medium,9,multipath,3.0,13.540133121461798,0.6251148711602101 +euroc,MH_03_medium,8,multipath,5.0,17.254549489159857,0.5868535398098166 +euroc,MH_03_medium,9,multipath,5.0,14.16667709436919,0.1344861114381258 +euroc,MH_03_medium,8,step,5.0,2.113750493248506,0.4862713457303325 +euroc,MH_03_medium,9,step,5.0,1.317710488147246,0.3207972191762212 +euroc,MH_03_medium,8,step,10.0,5.643579517416021,0.8182249113485517 +euroc,MH_03_medium,9,step,10.0,3.4633958066088115,0.7258823018393734 +euroc,MH_03_medium,8,ramp,3.0,1.5710289458223914,0.2861398787129743 +euroc,MH_03_medium,9,ramp,3.0,1.1987521496856932,0.3025195169608866 +euroc,MH_03_medium,8,ramp,5.0,19.25069341446007,0.9666418443524367 +euroc,MH_03_medium,9,ramp,5.0,2.371945764655122,0.661672655963198 +euroc,MH_03_medium,8,accel_bias,0.7,1.2316486323638707,0.07998661769046923 +euroc,MH_03_medium,9,accel_bias,0.7,1.0190604860944756,0.18075895518399643 +euroc,MH_03_medium,8,gyro_bias,0.02,1.9039330406121548,0.42280706525177525 +euroc,MH_03_medium,9,gyro_bias,0.02,1.3067964989712657,0.35927380611646165 +euroc,MH_04_difficult,8,none,0.0,1.0916153852707768,-0.035498044122619955 +euroc,MH_04_difficult,9,none,0.0,0.8277526654354256,-0.008693367701673413 +euroc,MH_04_difficult,8,multipath,1.0,2.5230556950467276,-0.001447426710358097 +euroc,MH_04_difficult,9,multipath,1.0,2.157592230398664,-0.0013957277629994193 +euroc,MH_04_difficult,8,multipath,2.0,9.649000589397588,0.6721059696726798 +euroc,MH_04_difficult,9,multipath,2.0,5.674929559592922,0.17233342630007317 +euroc,MH_04_difficult,8,multipath,3.0,7.982362063442435,0.5268063801883528 +euroc,MH_04_difficult,9,multipath,3.0,10.621498013994048,0.5326076139267264 +euroc,MH_04_difficult,8,multipath,5.0,17.72086934559169,0.5847975522481929 +euroc,MH_04_difficult,9,multipath,5.0,14.583146044477516,0.2570089397315488 +euroc,MH_04_difficult,8,step,5.0,2.5606129383233793,0.5824028682201264 +euroc,MH_04_difficult,9,step,5.0,2.219642157019299,0.6345432844555867 +euroc,MH_04_difficult,8,step,10.0,7.288660688817941,0.8570089701518282 +euroc,MH_04_difficult,9,step,10.0,6.751070363626236,0.8711222231457502 +euroc,MH_04_difficult,8,ramp,3.0,1.5187105233504683,0.255693415473792 +euroc,MH_04_difficult,9,ramp,3.0,1.386632162326913,0.4186248506102151 +euroc,MH_04_difficult,8,ramp,5.0,11.15013613639927,0.953957009372398 +euroc,MH_04_difficult,9,ramp,5.0,15.53657185764417,0.9774999735023265 +euroc,MH_04_difficult,8,accel_bias,0.7,1.2258130645367082,0.07798387590133063 +euroc,MH_04_difficult,9,accel_bias,0.7,1.0027242034807713,0.16337774211216205 +euroc,MH_04_difficult,8,gyro_bias,0.02,1.7591559694279104,0.37087396789731386 +euroc,MH_04_difficult,9,gyro_bias,0.02,1.1535121404651327,0.2759637977621331 +euroc,MH_05_difficult,8,none,0.0,1.0873736803818508,-0.041418041526935574 +euroc,MH_05_difficult,9,none,0.0,0.8393420726338218,0.0030371164284029614 +euroc,MH_05_difficult,8,multipath,1.0,2.567287835844375,0.006955645921539554 +euroc,MH_05_difficult,9,multipath,1.0,2.163208619507972,0.00258358985958284 +euroc,MH_05_difficult,8,multipath,2.0,5.679330849163798,0.25280979920403474 +euroc,MH_05_difficult,9,multipath,2.0,6.021159657488724,0.3026009657907965 +euroc,MH_05_difficult,8,multipath,3.0,14.198507693807676,0.7199906097389409 +euroc,MH_05_difficult,9,multipath,3.0,10.200197371299302,0.6335485418055085 +euroc,MH_05_difficult,8,multipath,5.0,17.80072025161296,0.5858551319149854 +euroc,MH_05_difficult,9,multipath,5.0,13.889321009386736,0.3872139308684128 +euroc,MH_05_difficult,8,step,5.0,2.4178182469702,0.558248121025792 +euroc,MH_05_difficult,9,step,5.0,2.174179160107112,0.6339631786941153 +euroc,MH_05_difficult,8,step,10.0,6.8347041894268195,0.8476725449172648 +euroc,MH_05_difficult,9,step,10.0,6.350385226894381,0.8642946855648624 +euroc,MH_05_difficult,8,ramp,3.0,1.504464624414779,0.24878701254466284 +euroc,MH_05_difficult,9,ramp,3.0,1.2096706523037069,0.3166480563270635 +euroc,MH_05_difficult,8,ramp,5.0,9.929683492376542,0.9421637231917145 +euroc,MH_05_difficult,9,ramp,5.0,16.133868359644826,0.9882728622790309 +euroc,MH_05_difficult,8,accel_bias,0.7,1.2432535919117282,0.08831323573228388 +euroc,MH_05_difficult,9,accel_bias,0.7,0.9912043454553462,0.1533414157444702 +euroc,MH_05_difficult,8,gyro_bias,0.02,1.4630214733028728,0.2220886448458465 +euroc,MH_05_difficult,9,gyro_bias,0.02,1.1843871707223286,0.29292864471083324 +euroc,V1_01_easy,8,none,0.0,1.0813918913076255,-0.046693581114390384 +euroc,V1_01_easy,9,none,0.0,0.8912440119199774,0.06301529005427223 +euroc,V1_01_easy,8,multipath,1.0,2.545893952248786,0.005724582987621059 +euroc,V1_01_easy,9,multipath,1.0,2.2401578490285816,0.036881990952164004 +euroc,V1_01_easy,8,multipath,2.0,6.381255112367405,0.5892709426393113 +euroc,V1_01_easy,9,multipath,2.0,7.46461137908901,0.7571371459436643 +euroc,V1_01_easy,8,multipath,3.0,8.343951440519573,0.5727265546890948 +euroc,V1_01_easy,9,multipath,3.0,10.94030113825134,0.6736197495352666 +euroc,V1_01_easy,8,multipath,5.0,15.056355594670633,0.5218810168213737 +euroc,V1_01_easy,9,multipath,5.0,11.721046769716082,0.47961903041894194 +euroc,V1_01_easy,8,step,5.0,2.556499450646105,0.5809067860166648 +euroc,V1_01_easy,9,step,5.0,8.237043959740758,0.9713002497924222 +euroc,V1_01_easy,8,step,10.0,7.162510700786108,0.8541193341780183 +euroc,V1_01_easy,9,step,10.0,14.806775327988056,0.9710170782001977 +euroc,V1_01_easy,8,ramp,3.0,1.5156235771762556,0.2544588349365914 +euroc,V1_01_easy,9,ramp,3.0,9.349147611333366,0.9778713588390447 +euroc,V1_01_easy,8,ramp,5.0,11.365194706125095,0.9588900076292793 +euroc,V1_01_easy,9,ramp,5.0,30.0114208441102,0.9916512538044822 +euroc,V1_01_easy,8,accel_bias,0.7,1.3224530657817182,0.15390175910039164 +euroc,V1_01_easy,9,accel_bias,0.7,1.0581173445786496,0.23027405220119107 +euroc,V1_01_easy,8,gyro_bias,0.02,1.6413767592671749,0.31131183342679114 +euroc,V1_01_easy,9,gyro_bias,0.02,1.456742853146748,0.4234085240769716 +euroc,V1_02_medium,8,none,0.0,1.072619553941663,-0.05692534837947885 +euroc,V1_02_medium,9,none,0.0,0.8335274856918904,-0.0004955179371493583 +euroc,V1_02_medium,8,multipath,1.0,2.5381549534578407,0.00612301910736868 +euroc,V1_02_medium,9,multipath,1.0,2.1830565122399737,0.00901118063010577 +euroc,V1_02_medium,8,multipath,2.0,5.9043445600401165,0.45279847098185577 +euroc,V1_02_medium,9,multipath,2.0,5.930375037728576,0.323738862709948 +euroc,V1_02_medium,8,multipath,3.0,8.401346902635181,0.5748674819435756 +euroc,V1_02_medium,9,multipath,3.0,8.103570082295196,0.39989624031448207 +euroc,V1_02_medium,8,multipath,5.0,19.671285702374707,0.5616154848570812 +euroc,V1_02_medium,9,multipath,5.0,14.286691556725737,0.24967286406691783 +euroc,V1_02_medium,8,step,5.0,2.4349620828709693,0.5597164047218107 +euroc,V1_02_medium,9,step,5.0,1.4672601133811936,0.3993734920607498 +euroc,V1_02_medium,8,step,10.0,7.210904331472189,0.8592653852779343 +euroc,V1_02_medium,9,step,10.0,5.194737879294599,0.8236087391412318 +euroc,V1_02_medium,8,ramp,3.0,1.6647845953995375,0.32420283828535057 +euroc,V1_02_medium,9,ramp,3.0,1.2219200397572305,0.31387418437837905 +euroc,V1_02_medium,8,ramp,5.0,30.41871361414107,0.9906245651744945 +euroc,V1_02_medium,9,ramp,5.0,24.33323963535366,0.9901315510300146 +euroc,V1_02_medium,8,accel_bias,0.7,1.311276112706436,0.1336755534810467 +euroc,V1_02_medium,9,accel_bias,0.7,1.0670536657086653,0.21723777727247637 +euroc,V1_02_medium,8,gyro_bias,0.02,1.308112136583597,0.13198601310456276 +euroc,V1_02_medium,9,gyro_bias,0.02,1.0907271092213953,0.2508733281367788 +euroc,V1_03_difficult,8,none,0.0,1.0792009034412262,-0.04860425984780361 +euroc,V1_03_difficult,9,none,0.0,0.8123582176603351,-0.02403442083064143 +euroc,V1_03_difficult,8,multipath,1.0,2.5629976872251157,0.016054495688309526 +euroc,V1_03_difficult,9,multipath,1.0,2.158252358625665,-0.0016057586747122157 +euroc,V1_03_difficult,8,multipath,2.0,6.337591967199919,0.30239912655948237 +euroc,V1_03_difficult,9,multipath,2.0,5.56372852579149,0.1527052164585831 +euroc,V1_03_difficult,8,multipath,3.0,7.930054107085504,0.5080428573935131 +euroc,V1_03_difficult,9,multipath,3.0,10.405138608225263,0.5202342968526212 +euroc,V1_03_difficult,8,multipath,5.0,11.71241035233251,0.25480693835398144 +euroc,V1_03_difficult,9,multipath,5.0,14.720371193262325,0.1736193101411058 +euroc,V1_03_difficult,8,step,5.0,2.354101680780492,0.5454553898549266 +euroc,V1_03_difficult,9,step,5.0,1.771051619440547,0.5358998238555361 +euroc,V1_03_difficult,8,step,10.0,6.929692386680287,0.8489293141478121 +euroc,V1_03_difficult,9,step,10.0,5.555772693810856,0.8338338904066863 +euroc,V1_03_difficult,8,ramp,3.0,1.3415780606915817,0.1572832346076823 +euroc,V1_03_difficult,9,ramp,3.0,1.0450392239339152,0.1983811525035739 +euroc,V1_03_difficult,8,ramp,5.0,15.418745405262092,0.9815442363170543 +euroc,V1_03_difficult,9,ramp,5.0,16.437988194891542,0.986521327536931 +euroc,V1_03_difficult,8,accel_bias,0.7,1.2380831117085078,0.08167434106452987 +euroc,V1_03_difficult,9,accel_bias,0.7,1.0263342748000628,0.1889362342821334 +euroc,V1_03_difficult,8,gyro_bias,0.02,4.904435479985171,0.7622547615023972 +euroc,V1_03_difficult,9,gyro_bias,0.02,1.720333386836808,0.5278036650521831 +euroc,V2_01_easy,8,none,0.0,1.0842502724602452,-0.04491131584629918 +euroc,V2_01_easy,9,none,0.0,0.8151358087880046,-0.020819867954211244 +euroc,V2_01_easy,8,multipath,1.0,2.571746130850475,0.017884128605420005 +euroc,V2_01_easy,9,multipath,1.0,2.1521482110862244,-0.008147049125351441 +euroc,V2_01_easy,8,multipath,2.0,6.738165607811408,0.6191948430586002 +euroc,V2_01_easy,9,multipath,2.0,5.647387400108567,0.15308169329133614 +euroc,V2_01_easy,8,multipath,3.0,8.174857872630168,0.4747857360527346 +euroc,V2_01_easy,9,multipath,3.0,9.00406951591179,0.39770746413574104 +euroc,V2_01_easy,8,multipath,5.0,17.23566602836934,0.5684442755869586 +euroc,V2_01_easy,9,multipath,5.0,12.202608039244327,0.18870596790706137 +euroc,V2_01_easy,8,step,5.0,1.904438357957363,0.4332951633733166 +euroc,V2_01_easy,9,step,5.0,2.058677409207485,0.6108091688041097 +euroc,V2_01_easy,8,step,10.0,5.323194568098313,0.7864110200109803 +euroc,V2_01_easy,9,step,10.0,6.0760085534053445,0.8505792703523366 +euroc,V2_01_easy,8,ramp,3.0,1.3367191629339914,0.15155619634491987 +euroc,V2_01_easy,9,ramp,3.0,1.240719674244865,0.342494052432671 +euroc,V2_01_easy,8,ramp,5.0,7.536424828222882,0.9262844154763856 +euroc,V2_01_easy,9,ramp,5.0,15.655193169133254,0.985093574585042 +euroc,V2_01_easy,8,accel_bias,0.7,1.308309232865717,0.13121589404447964 +euroc,V2_01_easy,9,accel_bias,0.7,1.117479251999835,0.26632961125990395 +euroc,V2_01_easy,8,gyro_bias,0.02,10.285375270656665,0.9048900229602495 +euroc,V2_01_easy,9,gyro_bias,0.02,1.6625660147094947,0.5111096779006545 +euroc,V2_02_medium,8,none,0.0,1.2090319714821105,0.0636987640900488 +euroc,V2_02_medium,9,none,0.0,1.105831805137769,0.2598253641281991 +euroc,V2_02_medium,8,multipath,1.0,2.861637257399442,0.12857601054150952 +euroc,V2_02_medium,9,multipath,1.0,5.01518184537669,0.7085053926270483 +euroc,V2_02_medium,8,multipath,2.0,9.420864332672027,0.8878408381732136 +euroc,V2_02_medium,9,multipath,2.0,12.190677478169404,0.7461747589389338 +euroc,V2_02_medium,8,multipath,3.0,11.496943356384769,0.7066529800626676 +euroc,V2_02_medium,9,multipath,3.0,10.792120656575795,0.49677452836338 +euroc,V2_02_medium,8,multipath,5.0,12.684330913611333,0.5356364231328257 +euroc,V2_02_medium,9,multipath,5.0,16.6610744672944,0.4891128079641751 +euroc,V2_02_medium,8,step,5.0,1.9654671128129546,0.44024118968976594 +euroc,V2_02_medium,9,step,5.0,12.6233103284672,0.970322489197914 +euroc,V2_02_medium,8,step,10.0,5.15665586121724,0.7737221873341034 +euroc,V2_02_medium,9,step,10.0,13.155534365537733,0.9579452994498251 +euroc,V2_02_medium,8,ramp,3.0,1.5674508763026214,0.28045619136816613 +euroc,V2_02_medium,9,ramp,3.0,1.5541836602789456,0.46295378182416763 +euroc,V2_02_medium,8,ramp,5.0,2.31645879231658,0.526460408178374 +euroc,V2_02_medium,9,ramp,5.0,6.2318795614742415,0.9113455177137145 +euroc,V2_02_medium,8,accel_bias,0.7,1.4994465258977094,0.25034651513094736 +euroc,V2_02_medium,9,accel_bias,0.7,1.3239841822222889,0.3784694803433394 +euroc,V2_02_medium,8,gyro_bias,0.02,2.604151502636608,0.5766155019204915 +euroc,V2_02_medium,9,gyro_bias,0.02,15.33913374532883,0.9478650717460038 +euroc,V2_03_difficult,8,none,0.0,1.0752675133147822,-0.05132386776833098 +euroc,V2_03_difficult,9,none,0.0,0.8414852644221185,0.006225198365483302 +euroc,V2_03_difficult,8,multipath,1.0,2.5175220607967375,-0.006138338967026559 +euroc,V2_03_difficult,9,multipath,1.0,2.1870989840450483,0.010078796869375968 +euroc,V2_03_difficult,8,multipath,2.0,5.698954764794814,0.4500055577493233 +euroc,V2_03_difficult,9,multipath,2.0,6.81522792535047,0.2822155055661514 +euroc,V2_03_difficult,8,multipath,3.0,8.463574639897521,0.13050132790669375 +euroc,V2_03_difficult,9,multipath,3.0,8.055305423384214,0.4246930181212231 +euroc,V2_03_difficult,8,multipath,5.0,30.80763507475487,0.707780209763106 +euroc,V2_03_difficult,9,multipath,5.0,13.565053961621977,0.28821884106492934 +euroc,V2_03_difficult,8,step,5.0,1.934576019371395,0.44409299442647787 +euroc,V2_03_difficult,9,step,5.0,1.391500102019041,0.35426917688235016 +euroc,V2_03_difficult,8,step,10.0,5.081261051342719,0.7746053768435678 +euroc,V2_03_difficult,9,step,10.0,4.075245564529784,0.7651768197199649 +euroc,V2_03_difficult,8,ramp,3.0,1.3356848516484325,0.15391790502047686 +euroc,V2_03_difficult,9,ramp,3.0,1.1476638693156023,0.2684271323588377 +euroc,V2_03_difficult,8,ramp,5.0,1.9238162391140115,0.41348932744657335 +euroc,V2_03_difficult,9,ramp,5.0,1.9573348910475425,0.5787833305918441 +euroc,V2_03_difficult,8,accel_bias,0.7,1.2669652373761862,0.11343042212392898 +euroc,V2_03_difficult,9,accel_bias,0.7,1.0325528686409235,0.18813596144812766 +euroc,V2_03_difficult,8,gyro_bias,0.02,1.8202108777413253,0.3875816764565502 +euroc,V2_03_difficult,9,gyro_bias,0.02,1.267114580287547,0.34242286765908053 +tumvi,room1,8,none,0.0,1.1006623851072026,-0.028430118894454282 +tumvi,room1,9,none,0.0,0.8562874318167824,0.02456876759842019 +tumvi,room1,8,multipath,1.0,2.5846378144347693,0.014083310963173416 +tumvi,room1,9,multipath,1.0,2.2105423700701494,0.01747015564336544 +tumvi,room1,8,multipath,2.0,7.770980406328356,0.22864002761545132 +tumvi,room1,9,multipath,2.0,7.97986017786424,0.7148931672720689 +tumvi,room1,8,multipath,3.0,8.304156621041994,0.16925536905567107 +tumvi,room1,9,multipath,3.0,9.490979737376646,0.5507236338598249 +tumvi,room1,8,multipath,5.0,26.042928806957757,0.5816614402639936 +tumvi,room1,9,multipath,5.0,15.447757065768245,0.3959664127111759 +tumvi,room1,8,step,5.0,2.6149832367488446,0.5836890874730092 +tumvi,room1,9,step,5.0,2.346256313397482,0.640533445593456 +tumvi,room1,8,step,10.0,13.176439099543282,0.938508319529361 +tumvi,room1,9,step,10.0,7.467026140381052,0.8795333810250701 +tumvi,room1,8,ramp,3.0,16.989750073966245,0.9378216120414224 +tumvi,room1,9,ramp,3.0,2.4628244417684204,0.663647331080339 +tumvi,room1,8,ramp,5.0,19.652252663983337,0.9871313572077579 +tumvi,room1,9,ramp,5.0,23.810755855863587,0.9894806643443879 +tumvi,room1,8,accel_bias,0.7,27.169380293564682,0.9663927389319387 +tumvi,room1,9,accel_bias,0.7,52.911509479587544,0.983764609748269 +tumvi,room1,8,gyro_bias,0.02,1.215111414711316,0.0660038966589963 +tumvi,room1,9,gyro_bias,0.02,1.0122937674020471,0.183660861530402 +tumvi,room2,8,none,0.0,1.1130141704808365,-0.021795800763067665 +tumvi,room2,9,none,0.0,0.8558811030284347,0.026573274428533116 +tumvi,room2,8,multipath,1.0,2.563319339247754,-0.00019099531843472258 +tumvi,room2,9,multipath,1.0,2.1442168374434223,-0.005218538552728286 +tumvi,room2,8,multipath,2.0,6.457226052478451,0.06580750687478813 +tumvi,room2,9,multipath,2.0,6.675985449082753,0.1532593041186908 +tumvi,room2,8,multipath,3.0,10.292496191656321,0.3939434063764731 +tumvi,room2,9,multipath,3.0,9.18582264636848,0.1603909177923941 +tumvi,room2,8,multipath,5.0,17.085965880692864,0.26855842423310033 +tumvi,room2,9,multipath,5.0,15.684925035243582,0.06699126293260292 +tumvi,room2,8,step,5.0,5.149839809297229,0.8602863029010394 +tumvi,room2,9,step,5.0,4.0906875713240485,0.8260520974146106 +tumvi,room2,8,step,10.0,17.437505734385628,0.9726779403191679 +tumvi,room2,9,step,10.0,14.561671333295566,0.9510527114512289 +tumvi,room2,8,ramp,3.0,53.356163266529485,0.9889555165209795 +tumvi,room2,9,ramp,3.0,48.991498839704434,0.9896190679375956 +tumvi,room2,8,ramp,5.0,158.292071433709,0.9934863813270648 +tumvi,room2,9,ramp,5.0,150.3000132729736,0.9936201092207828 +tumvi,room2,8,accel_bias,0.7,36.622144466901325,0.97640642013378 +tumvi,room2,9,accel_bias,0.7,49.13999754436628,0.9835568078490494 +tumvi,room2,8,gyro_bias,0.02,15.979600832412368,0.9392312897139972 +tumvi,room2,9,gyro_bias,0.02,6.206500849918701,0.8759327384422984 +tumvi,room3,8,none,0.0,1.1306979354034785,-0.0006302946444247058 +tumvi,room3,9,none,0.0,0.8399718095829866,0.009509393346553784 +tumvi,room3,8,multipath,1.0,2.6202568725985995,0.027538007807073526 +tumvi,room3,9,multipath,1.0,2.14918413878499,-0.007898965740995943 +tumvi,room3,8,multipath,2.0,6.238746020141341,0.12106198770024278 +tumvi,room3,9,multipath,2.0,6.839499904877676,0.14547651889859062 +tumvi,room3,8,multipath,3.0,11.534433963681701,0.4707334423686784 +tumvi,room3,9,multipath,3.0,12.436357974401753,0.5460178030090562 +tumvi,room3,8,multipath,5.0,17.417920391253308,0.3353503563533207 +tumvi,room3,9,multipath,5.0,18.56621947111215,0.3373968897784969 +tumvi,room3,8,step,5.0,4.691877698412903,0.8193553831120646 +tumvi,room3,9,step,5.0,2.6501570614925565,0.6887974090944263 +tumvi,room3,8,step,10.0,14.386914469676368,0.9512525713769825 +tumvi,room3,9,step,10.0,9.650562637182178,0.9146552485092287 +tumvi,room3,8,ramp,3.0,22.8708805422789,0.9807812299008414 +tumvi,room3,9,ramp,3.0,15.770005029098513,0.9758550632803283 +tumvi,room3,8,ramp,5.0,98.48253781903848,0.9930705872716548 +tumvi,room3,9,ramp,5.0,89.3783372179583,0.9933391649292612 +tumvi,room3,8,accel_bias,0.7,19.649554830343387,0.9503923433192731 +tumvi,room3,9,accel_bias,0.7,27.569221595743787,0.9729088744926582 +tumvi,room3,8,gyro_bias,0.02,2.162019747908316,0.48998047572074943 +tumvi,room3,9,gyro_bias,0.02,1.3876517596779305,0.40708616859411884 +tumvi,room4,8,none,0.0,1.0874295040596276,-0.040877354703821325 +tumvi,room4,9,none,0.0,0.8546797812810887,0.025569697119724944 +tumvi,room4,8,multipath,1.0,2.5911107998136056,0.012252158065658183 +tumvi,room4,9,multipath,1.0,2.1933092709769837,0.01445864600366998 +tumvi,room4,8,multipath,2.0,7.889021445047205,0.47103212706143177 +tumvi,room4,9,multipath,2.0,6.779765959018051,0.16170531665057133 +tumvi,room4,8,multipath,3.0,6.909506551301997,0.2889646625312422 +tumvi,room4,9,multipath,3.0,7.789158551869697,0.20273468329907854 +tumvi,room4,8,multipath,5.0,22.567398390580916,0.5085613569135262 +tumvi,room4,9,multipath,5.0,12.61483896077155,0.19234326985123829 +tumvi,room4,8,step,5.0,2.6494925825545157,0.5858813843545384 +tumvi,room4,9,step,5.0,2.0883106152402724,0.5940018425135333 +tumvi,room4,8,step,10.0,8.286210369436239,0.8745106096033576 +tumvi,room4,9,step,10.0,6.5900559327664405,0.8574139873701155 +tumvi,room4,8,ramp,3.0,2.106792490348665,0.46799993278432245 +tumvi,room4,9,ramp,3.0,2.2537961990124113,0.6362240362968438 +tumvi,room4,8,ramp,5.0,34.776269485860844,0.9885496861812167 +tumvi,room4,9,ramp,5.0,24.47729890533955,0.9888317396227114 +tumvi,room4,8,accel_bias,0.7,23.56704485707535,0.9608692634674114 +tumvi,room4,9,accel_bias,0.7,35.30876777854238,0.9782090777161591 +tumvi,room4,8,gyro_bias,0.02,1.8813968538000534,0.4127915921073799 +tumvi,room4,9,gyro_bias,0.02,1.4276924145747518,0.4247032943953707 +tumvi,room5,8,none,0.0,1.1468669623991694,0.011874903861594897 +tumvi,room5,9,none,0.0,0.8868421645865604,0.06107419197289426 +tumvi,room5,8,multipath,1.0,2.8527880789998705,0.12459376066666893 +tumvi,room5,9,multipath,1.0,2.379046580457882,0.0938834925028281 +tumvi,room5,8,multipath,2.0,10.751120620430045,0.8727281849887641 +tumvi,room5,9,multipath,2.0,11.16978244602344,0.7735137073078783 +tumvi,room5,8,multipath,3.0,11.266381444270904,0.593494658636693 +tumvi,room5,9,multipath,3.0,21.58806824900876,0.7728230997636699 +tumvi,room5,8,multipath,5.0,14.846924717533064,0.3763529417033872 +tumvi,room5,9,multipath,5.0,23.42867939600184,0.5283576621060367 +tumvi,room5,8,step,5.0,1.9752458428386752,0.44258402071096864 +tumvi,room5,9,step,5.0,1.4590664676578389,0.38926804624137423 +tumvi,room5,8,step,10.0,32.90850328425035,0.969581222490296 +tumvi,room5,9,step,10.0,5.050641209933226,0.8296272940233608 +tumvi,room5,8,ramp,3.0,3.8782135885730575,0.6858740144940729 +tumvi,room5,9,ramp,3.0,1.6899781089448693,0.5034716334145818 +tumvi,room5,8,ramp,5.0,61.53588497105486,0.9803124758511855 +tumvi,room5,9,ramp,5.0,18.83943825551437,0.9844259066761962 +tumvi,room5,8,accel_bias,0.7,2.2859531163187863,0.5131071457784483 +tumvi,room5,9,accel_bias,0.7,1.8970612743246218,0.5597968737123742 +tumvi,room5,8,gyro_bias,0.02,11.774684574184995,0.9136339685194295 +tumvi,room5,9,gyro_bias,0.02,11.621689985168643,0.9309458518260947 +tumvi,room6,8,none,0.0,1.118340089634935,-0.014696323536570325 +tumvi,room6,9,none,0.0,1.1884671440997594,0.2896887307153488 +tumvi,room6,8,multipath,1.0,2.550467791297373,0.00014748028081835448 +tumvi,room6,9,multipath,1.0,2.8984677998074972,0.27582141246822917 +tumvi,room6,8,multipath,2.0,6.52231770682512,0.27446301278691126 +tumvi,room6,9,multipath,2.0,8.19169191249975,0.3105107756270337 +tumvi,room6,8,multipath,3.0,10.44344979445962,0.5415558806748999 +tumvi,room6,9,multipath,3.0,10.836941685349094,0.48285129493289114 +tumvi,room6,8,multipath,5.0,40.03338811084634,0.7321017936009996 +tumvi,room6,9,multipath,5.0,22.34543028988779,0.34066301792197673 +tumvi,room6,8,step,5.0,2.3731113949402824,0.5385161930704878 +tumvi,room6,9,step,5.0,1.9894303767105255,0.5612280008114846 +tumvi,room6,8,step,10.0,6.516280868907088,0.828094798438709 +tumvi,room6,9,step,10.0,5.666152283628141,0.8270366897269945 +tumvi,room6,8,ramp,3.0,1.9903594999325367,0.44179325635307426 +tumvi,room6,9,ramp,3.0,3.3618441093659706,0.7425018494480041 +tumvi,room6,8,ramp,5.0,14.449663937393984,0.980918071720304 +tumvi,room6,9,ramp,5.0,23.744655773698444,0.9889514813612186 +tumvi,room6,8,accel_bias,0.7,7.739529603731095,0.8730345713414651 +tumvi,room6,9,accel_bias,0.7,29.06383986706337,0.9768906682255647 +tumvi,room6,8,gyro_bias,0.02,35.39936248871269,0.966734373019244 +tumvi,room6,9,gyro_bias,0.02,35.50503210784493,0.9709380106060864 diff --git a/docs/multipath.md b/docs/multipath.md new file mode 100644 index 0000000..7db8c9e --- /dev/null +++ b/docs/multipath.md @@ -0,0 +1,123 @@ +# A multipath signature on the GNSS innovations + +[ADR-0021](adr/0021-fault-classification-from-innovations.md) chose to ship at most a multipath detector, because +multipath was the one cause the [fault-classification study](fault_classification.md) separated cleanly. This page +measures a simple detector: two numbers computed on a window of GNSS innovations. It is a function in +`navkit.fdir.multipath`. It is not wired into the filter ([ADR-0022](adr/0022-multipath-signature.md)). + +## The signature + +Over a 20 s window of innovations: + +- **NIS**: the mean normalised innovation squared, divided by 3. About 1 when the channel is healthy. +- **Lag-1 autocorrelation** of the whitened innovation. Near 0 for noise, near 1 for a persistent offset. + +Multipath is noise on the fixes: a large NIS with little correlation. A step or a ramp also inflates the NIS but its +innovations are strongly autocorrelated. The call is a large NIS and an autocorrelation of at most 0.8. + +Two rules differ in the NIS they need. The **sensitive** rule needs 2, twice the healthy value. The **specific** rule +needs 6, the value the classification tree learned on its training runs. Both were fixed in code, and the shipping +criteria were written, before the fresh runs were made. + +## What counts as success + +A rule is shipped only if, on fresh runs, it calls at least 95% of the multipath runs of 3 m and above, and at most 2% +of every other run, controls included. The criteria are in `CRITERIA` in `navkit.multipath_study`. + +## The floor + +Multipath of 0.5 to 2 m, and extra controls, on seeds 6 and 7. These runs were seen before the fresh set was +designed. + + + +| Multipath m | Runs | Sensitive rule calls | Specific rule calls | Median NIS | Median lag-1 autocorrelation | +|---:|---:|---:|---:|---:|---:| +| 0 | 34 | 0% | 0% | 0.88 | -0.04 | +| 0.5 | 34 | 0% | 0% | 1.31 | -0.00 | +| 1 | 34 | 100% | 0% | 2.49 | 0.05 | +| 1.5 | 34 | 100% | 6% | 4.58 | 0.19 | +| 2 | 34 | 100% | 68% | 6.64 | 0.51 | + + + +## The confirmation runs of the classification study + +Unseen sizes and seeds, by true class. The 6 in the specific rule came from that study's training runs. These runs +were seen before the rules were fixed. + + + +| Class | Runs | Sensitive rule calls multipath | Specific rule calls multipath | +|---|---:|---:|---:| +| control | 68 | 0% | 0% | +| multipath | 68 | 100% | 100% | +| spoofing | 238 | 12% | 0% | +| degradation | 170 | 4% | 0% | + + + +## The fresh runs + +New seeds (8 and 9) and a mixed set of cases, run after the rules and the criteria were fixed. This is the clean +test of the rule that is chosen. + + + +| Class | Fault | Level | Runs | Sensitive rule calls multipath | Specific rule calls multipath | +|---|---|---:|---:|---:|---:| +| control | none | 0 | 34 | 0% | 0% | +| multipath | multipath | 1 | 34 | 100% | 0% | +| multipath | multipath | 2 | 34 | 94% | 65% | +| multipath | multipath | 3 | 34 | 100% | 100% | +| multipath | multipath | 5 | 34 | 100% | 100% | +| spoofing | step | 5 | 34 | 50% | 0% | +| spoofing | step | 10 | 34 | 15% | 0% | +| spoofing | ramp | 3 | 34 | 15% | 0% | +| spoofing | ramp | 5 | 34 | 6% | 0% | +| degradation | accel_bias | 0.7 | 34 | 3% | 0% | +| degradation | gyro_bias | 0.02 | 34 | 9% | 0% | + + + + + +| Rule | Recall, multipath of 3 m and above (needs 95%) | Called multipath, all other runs including controls (needs at most 2%) | Criteria met | +|---|---:|---:|---| +| sensitive | 100% | 14% | no | +| specific | 100% | 0% | yes | + + + +## Control runs + + + +| Control runs | Runs | Sensitive rule calls | Specific rule calls | 95% upper bound on the specific rule's rate | +|---|---:|---:|---:|---:| +| floor | 34 | 0 | 0 | 9% | +| confirmation | 68 | 0 | 0 | 4% | +| fresh | 34 | 0 | 0 | 9% | +| all | 136 | 0 | 0 | 2% | + + + +## What it shows + +1. **The specific rule meets the criteria.** On the fresh runs it calls all multipath of 3 m and above, and none of + the other runs: no control, no step, no ramp, no bias. +2. **The sensitive rule does not.** It calls multipath from 1 m, but it also calls half of the 5 m steps and a few + ramps and bias steps, so 14% of the other fresh runs are called multipath. +3. **The cost of the specific rule is sensitivity.** Multipath of 2 m is called in about two thirds of runs, 1.5 m + in a few, and 1 m and below in none. The detector sees multipath of about 3 m and above, which is several times the + 0.8 m noise of the simulated GNSS. +4. **No control run in any set is called multipath.** None of 136 control runs, which bounds the false-alarm rate at + about 2% with 95% confidence. + +## Limits + +Injected white noise on simulated GNSS, 20 s windows, two datasets, and the calibrated bias-walk setting. Real +multipath is correlated over seconds and comes with reflections that bias the position, so a real multipath run will +have a higher autocorrelation than the injected noise and may fall above 0.8. That is not tested here. Only the +specific rule was chosen on clean data, and its floor of about 3 m is a measurement of this injection, not of a +receiver. diff --git a/mkdocs.yml b/mkdocs.yml index 49fb9b4..66779d4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -107,6 +107,7 @@ nav: - Slow-ramp spoofing: slow_ramp.md - A second source against a ramp: vision_ramp.md - Telling the causes apart: fault_classification.md + - A multipath signature: multipath.md - EuRoC, recorded IMU: euroc.md - TUM VI, recorded IMU: tumvi.md - Performance: PERFORMANCE.md @@ -145,6 +146,7 @@ nav: - "0019 Slow-ramp spoofing": adr/0019-slow-ramp-spoofing.md - "0020 Second source against a ramp": adr/0020-second-source-against-a-ramp.md - "0021 Fault classification": adr/0021-fault-classification-from-innovations.md + - "0022 Multipath signature": adr/0022-multipath-signature.md - Project: - Scope and responsible use: scope.md - Status and limits: status.md diff --git a/src/navkit/multipath_study.py b/src/navkit/multipath_study.py index 9815909..29eb564 100644 --- a/src/navkit/multipath_study.py +++ b/src/navkit/multipath_study.py @@ -198,8 +198,8 @@ def criteria_markdown(floor: list[dict[str, Any]], fresh: list[dict[str, Any]]) res = criteria_results(fresh) lines = [ f"| Rule | Recall, multipath of 3 m and above (needs {_pct(CRITERIA['recall_at_3m_and_above'])}) | " - f"Called multipath, all other runs including controls (needs at most {_pct(CRITERIA['false_call_rate_other'])}) " - "| Criteria met |", + "Called multipath, all other runs including controls " + f"(needs at most {_pct(CRITERIA['false_call_rate_other'])}) | Criteria met |", "|---|---:|---:|---|", ] for rule in mp.RULES: @@ -214,11 +214,16 @@ def false_alarm_markdown(floor: list[dict[str, Any]], fresh: list[dict[str, Any] "confirmation": [r for r in confirm_rows() if r["kind"] == "none"], "fresh": [r for r in fresh if r["kind"] == "none"], } - lines = ["| Control runs | Runs | Sensitive rule calls | Specific rule calls |", "|---|---:|---:|---:|"] + lines = [ + "| Control runs | Runs | Sensitive rule calls | Specific rule calls " + "| 95% upper bound on the specific rule's rate |", + "|---|---:|---:|---:|---:|", + ] everything = [r for sel in ctrl.values() for r in sel] for label, sel in (*ctrl.items(), ("all", everything)): s, p = (sum(called(r, rule) for r in sel) for rule in mp.RULES) - lines.append(f"| {label} | {len(sel)} | {s} | {p} |") + bound = _pct(upper_bound(len(sel))) if p == 0 else "see the count" + lines.append(f"| {label} | {len(sel)} | {s} | {p} | {bound} |") return "\n".join(lines) diff --git a/tests/test_multipath.py b/tests/test_multipath.py new file mode 100644 index 0000000..6161f46 --- /dev/null +++ b/tests/test_multipath.py @@ -0,0 +1,132 @@ +"""The multipath signature function, and the study page against its committed CSVs.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from navkit import euroc_eval as ee +from navkit import multipath_study as ms +from navkit.fault_classification import features +from navkit.fdir import multipath as mp + +ROOT = Path(__file__).resolve().parent.parent +PAGE = ROOT / "docs" / "multipath.md" + + +@pytest.fixture(scope="module") +def floor(): + return ms.read_csv(ROOT / "docs" / "data" / "multipath_detector.csv") + + +@pytest.fixture(scope="module") +def fresh(): + return ms.read_csv(ROOT / "docs" / "data" / "multipath_fresh.csv") + + +def _window(scale=1.0, offset=0.0, n=100, seed=0): + rng = np.random.default_rng(seed) + s = np.eye(3) + return [(35.0 + i * 0.2, rng.standard_normal(3) * scale + np.array([offset, 0.0, 0.0]), s) for i in range(n)] + + +def test_white_noise_of_large_size_is_multipath(): + sig = mp.signature(_window(scale=5.0)) + assert sig is not None and sig.multipath and sig.specific and sig.sensitive + + +def test_healthy_noise_is_not_multipath(): + sig = mp.signature(_window(scale=1.0)) + assert sig is not None and not sig.sensitive and not sig.specific + + +def test_a_persistent_offset_is_not_multipath_however_large(): + sig = mp.signature(_window(scale=1.0, offset=10.0)) + assert sig is not None and sig.nis > mp.NIS_MIN_SPECIFIC and sig.ac1 > mp.AC1_MAX and not sig.multipath + + +def test_a_short_window_gives_no_call(): + assert mp.signature(_window(n=5)) is None + assert mp.is_multipath(_window(n=5)) is False + + +def test_the_two_rules_differ_in_the_nis_they_need(): + assert mp.call(3.0, 0.0, "sensitive") and not mp.call(3.0, 0.0, "specific") + assert mp.call(7.0, 0.0, "specific") and not mp.call(7.0, 0.9, "specific") + with pytest.raises(ValueError): + mp.call(3.0, 0.0, "bogus") + assert mp.is_multipath(_window(scale=2.0), "sensitive") and not mp.is_multipath(_window(scale=2.0), "specific") + + +def test_statistics_agree_with_the_classification_study_features(): + inn = _window(scale=3.0, seed=4) + bias = [(34.0, np.zeros(3), np.zeros(3)), (54.0, np.zeros(3), np.zeros(3))] + f = features(inn, bias, 35.0) + sig = mp.signature(inn) + assert f is not None and sig is not None + assert sig.nis == pytest.approx(f["nis"]) and sig.ac1 == pytest.approx(f["ac1"]) + + +def test_page_blocks_match_the_csvs(floor, fresh): + text = PAGE.read_text(encoding="utf-8") + for fn in ms.BLOCKS.values(): + assert fn(floor, fresh) in text + + +def test_the_criteria_were_met_by_the_specific_rule_only(fresh): + res = ms.criteria_results(fresh) + assert res["specific"]["met"] and not res["sensitive"]["met"] + assert res["specific"]["recall"] == 1.0 and res["specific"]["false_call_rate"] == 0.0 + assert res["sensitive"]["false_call_rate"] > ms.CRITERIA["false_call_rate_other"] + + +def test_the_criteria_are_the_ones_written_before_the_runs(): + assert ms.CRITERIA == {"recall_at_3m_and_above": 0.95, "false_call_rate_other": 0.02} + + +def test_no_control_run_in_any_set_is_called_multipath(floor, fresh): + controls = [r for r in floor + fresh + ms.confirm_rows() if r["kind"] == "none"] + assert len(controls) == 136 + assert not any(ms.called(r, rule) for r in controls for rule in mp.RULES) + + +def test_the_specific_rule_floor(floor, fresh): + def share(rows, level): + return ms._share([r for r in rows if r["kind"] == "multipath" and float(r["level"]) == level], "specific") + + assert share(floor, 0.5) == 0.0 and share(floor, 1.0) == 0.0 and share(floor, 1.5) < 0.1 + assert 0.6 < share(fresh, 2.0) < 0.7 + assert share(fresh, 3.0) == 1.0 and share(fresh, 5.0) == 1.0 + + +def test_the_sensitive_rule_calls_half_the_five_metre_steps(fresh): + sel = [r for r in fresh if r["kind"] == "step" and float(r["level"]) == 5.0] + assert ms._share(sel, "sensitive") == 0.5 and ms._share(sel, "specific") == 0.0 + + +def test_run_case_and_sweep_on_a_synthetic_sequence(tmp_path): + ee.write_fixture(tmp_path, "MH_01_easy", duration_s=80.0, seed=1) + seq = ee.load_sequence(tmp_path, "MH_01_easy") + noisy = ms.run_case(seq, "walk10", 0, "multipath", 30.0) + control = ms.run_case(seq, "walk10", 0, "none", 0.0) + assert noisy is not None and control is not None + assert noisy["nis"] > 5.0 * control["nis"] + got = ms.sweep_rows("euroc", tmp_path, (("none", 0.0), ("multipath", 30.0)), (1,)) + assert [r["kind"] for r in got] == ["none", "multipath"] + short = tmp_path / "s" + ee.write_fixture(short, "MH_01_easy", duration_s=40.0, seed=1) + assert ms.run_case(ee.load_sequence(short, "MH_01_easy"), "walk10", 0, "none", 0.0) is None + assert ms.floor_cases()[0] == ("none", 0.0) + + +def test_csv_round_trip_cli_and_missing_markers(floor, tmp_path, capsys): + out = tmp_path / "again.csv" + ms.write_csv(out, floor) + assert ms.read_csv(out) == [{k: str(v) for k, v in r.items()} for r in floor] + assert ms.main(["--markdown"]) == 0 + assert "Specific rule" in capsys.readouterr().out + page = tmp_path / "p.md" + page.write_text("no markers\n", encoding="utf-8") + assert ms.main(["--page", str(page)]) == 1