From 9bb618f15504869c7be4218c062d309a6849d32a Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:39:04 +0200 Subject: [PATCH 1/5] feat(urbannav): the aided run reports Wilson intervals and NEES sample sizes (#131) The aided run (a sequence without an outage) now also reports the gate's alpha and the rejections expected at it, the Wilson 95% interval of the rejection rate on the raw count and on the effective count (ADR-0029), and the effective sample size of the horizontal and attitude NEES series with the upper-bound flag. A constant over-the-level series has no estimate and counts every fix, which the new fix_rejection_ess_defined flag says. Aided rows only: the Tunnel-1 quantities and rows are untouched. Co-Authored-By: Claude Sonnet 5.5 --- src/navkit/urbannav_run.py | 82 ++++++++++++++++++++++++++++++-- tests/test_urbannav_synthetic.py | 82 ++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 4 deletions(-) diff --git a/src/navkit/urbannav_run.py b/src/navkit/urbannav_run.py index c5aeca3..1e756e5 100644 --- a/src/navkit/urbannav_run.py +++ b/src/navkit/urbannav_run.py @@ -43,6 +43,7 @@ from . import urbannav_data as ud from .estimators.eskf import ErrorStateKalmanFilter, EskfConfig from .euroc_eval import RunOptions, run_sequence +from .eval.calibration import effective_samples, wilson_interval from .fdir.gating import chi2_threshold from .io.imu import DEFAULT_NOISE, ImuNoiseModel from .recorded import EurocSequence, SequenceError @@ -312,12 +313,42 @@ def accel_only_error_m(seq: EurocSequence, span: tuple[float, float]) -> float: FIX_PERCENTILES = (50, 95, 99) +def rejection_intervals(rejected: int, offered: int, nis: np.ndarray, gate_level: float) -> dict[str, Any]: + """The Wilson intervals of the rejection rate, on the raw count and on the effective count (ADR-0029). + + The raw interval takes the ``offered`` fixes as independent. The effective one replaces their number by the + effective sample size of the 0/1 series "the fix's NIS is over the gate's level" and the count by + ``rejected * ess / offered``, which is how ``fdir_false_alarm`` pools a rate. A series with no fix over the level + (or all of them over it) is constant and has no autocorrelation: it counts as ``offered`` independent samples, + the generous reading, so the effective interval is then the raw one, and ``fix_rejection_ess_defined`` is 0. + """ + ess, _ = effective_samples((np.asarray(nis, dtype=float) > gate_level).astype(float)) + defined = math.isfinite(ess) + if not defined: + ess = float(offered) + lo, hi = wilson_interval(rejected, offered) + eff_lo, eff_hi = wilson_interval(rejected * ess / offered if offered else 0.0, ess) + return { + "fix_gate_alpha": GATE_ALPHA, + "fix_rejections_expected": offered * GATE_ALPHA, + "fix_rejection_wilson_lo": lo, + "fix_rejection_wilson_hi": hi, + "fix_rejection_ess": ess, + "fix_rejection_ess_defined": int(defined), + "fix_rejection_eff_wilson_lo": eff_lo, + "fix_rejection_eff_wilson_hi": eff_hi, + } + + def aided_summary( detail: dict[str, Any], stats: dict[str, Any], offered: int, fix_difference_m: np.ndarray, fix_sigma_h_m: np.ndarray, + nis: np.ndarray | None = None, + gate_level: float | None = None, + nees_h: np.ndarray | None = None, ) -> dict[str, Any]: """The quantities of the aided run, over the whole recording. Aggregates only. @@ -328,6 +359,10 @@ def aided_summary( * ``fix_*``: the receiver's fixes themselves against the reference, without the filter: percentiles of the horizontal difference, and the mean of (horizontal difference / receiver sigma) squared, which is 2 when the receiver's sigma is right and the errors are Gaussian. + * ``fix_rejection_*``, with ``nis`` and ``gate_level``: the Wilson intervals of :func:`rejection_intervals`. + * ``*_nees_ess``, with ``nees_h`` for the horizontal one: the effective sample size of each NEES series, and + whether it reached the largest window (then it is an upper bound, ADR-0029). The means are over every IMU step, + so they are far from independent samples, and the effective size is what says how many there are. """ est, ref, cov = detail["est"], detail["reference"], detail["position_cov"] err = est.positions - ref.positions @@ -339,12 +374,18 @@ def aided_summary( "fix_rejection_rate": float(stats.get("gnss_updates_rejected", 0)) / offered if offered else math.nan, "fix_nees_mean": float(np.mean((fix_difference_m / fix_sigma_h_m) ** 2)), } + if nis is not None and gate_level is not None: + out.update(rejection_intervals(int(stats.get("gnss_updates_rejected", 0)), offered, nis, gate_level)) + if nees_h is not None: + out["hor_nees_ess"], truncated = effective_samples(nees_h) + out["hor_nees_ess_truncated"] = int(truncated) rot_cov = detail.get("rotation_cov") if rot_cov is not None: dtheta = rot_log_batch(np.einsum("nji,njk->nik", est.rotations, ref.rotations)) - out["att_nees_mean"] = float( - np.mean(np.einsum("ni,ni->n", dtheta, np.linalg.solve(rot_cov, dtheta[..., None])[..., 0])) - ) + nees_att = np.einsum("ni,ni->n", dtheta, np.linalg.solve(rot_cov, dtheta[..., None])[..., 0]) + out["att_nees_mean"] = float(np.mean(nees_att)) + out["att_nees_ess"], truncated = effective_samples(nees_att) + out["att_nees_ess_truncated"] = int(truncated) for q in FIX_PERCENTILES: out[f"fix_err_p{q}_m"] = float(np.percentile(fix_difference_m, q)) return out @@ -445,7 +486,16 @@ def run_once( out["hor_err_median_after_m"] = float(np.median(eh[after])) else: out.update( - aided_summary(detail, record["stats"], int(nis.size), info["fix_difference_m"], info["fix_sigma_h_m"]) + aided_summary( + detail, + record["stats"], + int(nis.size), + info["fix_difference_m"], + info["fix_sigma_h_m"], + nis=nis, + gate_level=level, + nees_h=nees_h, + ) ) return out @@ -500,10 +550,34 @@ def run_once( ("fixes_used", "Fixes the filter used", "count"), ("gnss_updates_rejected", "Fixes the filter rejected", "count"), ("fix_rejection_rate", "Share of the offered fixes the gate rejected", ""), + ("fix_gate_alpha", "The gate's alpha (the FDIR default)", ""), + ( + "fix_rejections_expected", + "Rejections expected at that alpha if every fix were consistent (offered x alpha)", + "count", + ), + ("fix_rejection_wilson_lo", "Wilson 95% interval of the rejection rate on the raw count, lower end", ""), + ("fix_rejection_wilson_hi", "Wilson 95% interval of the rejection rate on the raw count, upper end", ""), + ( + "fix_rejection_ess", + "Effective number of fixes behind the rate (effective sample size of the over-the-level series)", + "count", + ), + ( + "fix_rejection_ess_defined", + "That size could be estimated (1 yes, 0 no: the series is constant, so it counts every fix)", + "flag", + ), + ("fix_rejection_eff_wilson_lo", "Wilson 95% interval on the effective count, lower end", ""), + ("fix_rejection_eff_wilson_hi", "Wilson 95% interval on the effective count, upper end", ""), ("gnss_channel_faulted", "GNSS channel declared faulty by the end", "count"), ("gnss_reaccepted", "Times a faulty channel was taken back", "count"), ("hor_nees_mean", "Horizontal mean NEES over the whole run (2 degrees of freedom, expected 2)", ""), + ("hor_nees_ess", "Effective sample size of the horizontal NEES series (it has one value per IMU step)", "count"), + ("hor_nees_ess_truncated", "That size is an upper bound (1 yes, 0 no)", "flag"), ("att_nees_mean", "Attitude mean NEES over the whole run (3 degrees of freedom, expected 3)", ""), + ("att_nees_ess", "Effective sample size of the attitude NEES series", "count"), + ("att_nees_ess_truncated", "That size is an upper bound (1 yes, 0 no)", "flag"), ( "hor_coverage_2sigma", f"Pooled 2-sigma per-axis coverage, east and north (a Gaussian gives {COVERAGE_2SIGMA_GAUSSIAN:.4f})", diff --git a/tests/test_urbannav_synthetic.py b/tests/test_urbannav_synthetic.py index 8e78ba9..90a0c51 100644 --- a/tests/test_urbannav_synthetic.py +++ b/tests/test_urbannav_synthetic.py @@ -487,6 +487,88 @@ def test_the_aided_summary_against_known_answers(error_sigmas, att_error_rad, co assert out["fix_err_p50_m"] == pytest.approx(2.5) and out["aided"] == 1 +def _known_detail(n: int = 50) -> dict[str, Any]: + ref = _trajectory(np.zeros((n, 3)), np.tile(np.eye(3), (n, 1, 1))) + return { + "est": ref, + "reference": ref, + "position_cov": np.tile(np.diag([4.0] * 3), (n, 1, 1)), + "rotation_cov": np.tile(np.eye(3) * 0.05**2, (n, 1, 1)), + } + + +def test_the_rejection_rate_has_wilson_intervals_on_the_raw_and_the_effective_count(): + from navkit.eval.calibration import wilson_interval + + level = ur.chi2_threshold(3, ur.GATE_ALPHA) + rng = np.random.default_rng(5) + nis = rng.uniform(0.0, 0.9 * level, 400) # independent, and none over the level + nis[[100, 101, 102, 103]] = 2.0 * level # four in a row over it + out = ur.aided_summary( + _known_detail(), + {"gnss_updates_used": 396, "gnss_updates_rejected": 4}, + 400, + np.ones(4), + np.ones(4), + nis=nis, + gate_level=level, + ) + assert out["fix_gate_alpha"] == ur.GATE_ALPHA + assert out["fix_rejections_expected"] == pytest.approx(400 * ur.GATE_ALPHA) + lo, hi = wilson_interval(4, 400) + assert (out["fix_rejection_wilson_lo"], out["fix_rejection_wilson_hi"]) == (lo, hi) + # four rejections in a row are one cluster, not four: the effective count is smaller and the interval wider + assert out["fix_rejection_ess_defined"] == 1 and 1.0 <= out["fix_rejection_ess"] < 400.0 + eff_k = 4 * out["fix_rejection_ess"] / 400 + elo, ehi = wilson_interval(eff_k, out["fix_rejection_ess"]) + assert (out["fix_rejection_eff_wilson_lo"], out["fix_rejection_eff_wilson_hi"]) == (elo, ehi) + assert ehi - elo > hi - lo + + +def test_with_no_rejection_the_effective_interval_is_the_raw_one_and_says_so(): + from navkit.eval.calibration import wilson_interval + + level = ur.chi2_threshold(3, ur.GATE_ALPHA) + nis = np.linspace(0.1, 0.5 * level, 300) + out = ur.aided_summary( + _known_detail(), + {"gnss_updates_used": 300, "gnss_updates_rejected": 0}, + 300, + np.ones(4), + np.ones(4), + nis=nis, + gate_level=level, + ) + assert out["fix_rejection_ess_defined"] == 0 and out["fix_rejection_ess"] == 300.0 + assert (out["fix_rejection_wilson_lo"], out["fix_rejection_wilson_hi"]) == wilson_interval(0, 300) + assert (out["fix_rejection_eff_wilson_lo"], out["fix_rejection_eff_wilson_hi"]) == wilson_interval(0, 300) + + +def test_the_nees_means_carry_their_effective_sample_sizes(): + from navkit.eval.calibration import effective_samples + + n = 400 + rng = np.random.default_rng(2) + slow = np.cumsum(rng.normal(size=n)) ** 2 + 1.0 # strongly autocorrelated + out = ur.aided_summary( + _known_detail(n), {"gnss_updates_used": 4, "gnss_updates_rejected": 0}, 4, np.ones(4), np.ones(4), nees_h=slow + ) + n_eff, truncated = effective_samples(slow) + assert out["hor_nees_ess"] == pytest.approx(n_eff) and out["hor_nees_ess_truncated"] == int(truncated) + assert 1.0 <= out["hor_nees_ess"] < n + # the attitude series here is constant (zero error), which has no estimate: it is NaN and not a made-up number + assert math.isnan(out["att_nees_ess"]) and out["att_nees_ess_truncated"] == 0 + + +def test_the_aided_run_reports_the_intervals_and_sample_sizes_it_documents(aided): + for v, r in aided.items(): + assert 0.0 <= r["fix_rejection_wilson_lo"] <= r["fix_rejection_rate"] <= r["fix_rejection_wilson_hi"] <= 1.0, v + assert r["fix_rejection_eff_wilson_lo"] <= r["fix_rejection_rate"] <= r["fix_rejection_eff_wilson_hi"], v + assert 1.0 <= r["fix_rejection_ess"] <= r["fixes_total"], v + assert r["fix_rejections_expected"] == pytest.approx(r["fixes_total"] * ur.GATE_ALPHA), v + assert r["hor_nees_ess"] >= 1.0 and r["att_nees_ess"] >= 1.0, v + + def test_the_coverage_expectation_is_the_gaussian_two_sigma_share(): assert abs(ur.COVERAGE_2SIGMA_GAUSSIAN - 0.9545) < 5e-5 From 346d35a6e5ed7dd78e096af2a47c6c5aecdee68c Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:37:53 +0200 Subject: [PATCH 2/5] feat(urbannav): Deep-Urban-1, the aided run on real urban GNSS (#131) Adds the Deep-Urban-1 rows (aided run, declared and per-fix variants, five seeds) to the run and data CSVs through the merge writer, and the Tunnel-1 lines of both files are byte-identical to before. The aided run now also reports counts of far fixes, the mean NIS and the gate level, the claimed sigma against the receiver's, and the NEES by third of the run. New: `navkit urbannav fixerrors` and docs/data/urbannav_fix_errors.csv, the time correlation of the receiver's own fix errors against the reference for both sequences. The page gets the Deep-Urban-1 findings, each tied to the CSVs by a test, an INTERPRETATION that says what it does not prove, and no typed number. ADR-0026 gets a dated context section (no Decision text). Data-gated tests for the real files skip without the data. The byte-prefix tests now use a new sequence name, so they still prove that the committed rows are untouched. Co-Authored-By: Claude Sonnet 5.5 --- CONSTRAINTS.md | 4 +- CONTRIBUTING.md | 2 +- ROADMAP.md | 2 +- .../0026-urbannav-as-a-real-gnss-dataset.md | 26 +- docs/data/metrics.json | 2 +- docs/data/urbannav_data.csv | 15 + docs/data/urbannav_fix_errors.csv | 51 ++ docs/data/urbannav_run.csv | 470 ++++++++++++++++++ docs/product_management/05_work_breakdown.md | 4 +- docs/urbannav.md | 263 +++++++++- src/navkit/cli.py | 15 +- src/navkit/urbannav_fix_errors.py | 201 ++++++++ src/navkit/urbannav_run.py | 29 ++ tests/test_urbannav_check.py | 22 +- tests/test_urbannav_data.py | 50 ++ tests/test_urbannav_fix_errors.py | 92 ++++ tests/test_urbannav_run.py | 191 ++++++- tests/test_urbannav_synthetic.py | 52 +- 18 files changed, 1439 insertions(+), 52 deletions(-) create mode 100644 docs/data/urbannav_fix_errors.csv create mode 100644 src/navkit/urbannav_fix_errors.py create mode 100644 tests/test_urbannav_fix_errors.py diff --git a/CONSTRAINTS.md b/CONSTRAINTS.md index 0d26e50..7677223 100644 --- a/CONSTRAINTS.md +++ b/CONSTRAINTS.md @@ -4,7 +4,7 @@ Invariants this project holds itself to. Every entry is either currently true or is a stated blocker; nothing here is aspirational. Ratchets are measured, not estimated, and the command to re-measure each one is given. -Last measured: 2320 tests collected, all of which pass except the deliberate +Last measured: 2355 tests collected, all of which pass except the deliberate xfails and the skips that `pytest -rs` lists, and 94.3% line coverage (10033/10638 executable lines) on **CPython 3.13**, measured with `python scripts/coverage_report.py`. The figures come from `docs/data/metrics.json`, which `scripts/metrics.py` checks in CI. Coverage is quoted from that script alone: @@ -91,7 +91,7 @@ the floors themselves are enforced by `scripts/coverage_report.py --ratchet`. | Ratchet | Floor | Current (CPython 3.13) | Re-measure with | |---|---:|---:|---| -| Tests collected | 405 | 2320 | `pytest`, `scripts/metrics.py --check` | +| Tests collected | 405 | 2355 | `pytest`, `scripts/metrics.py --check` | | Tests passing | 300 | all collected, except the deliberate xfails and the listed skips | `pytest -rs` | | Line coverage | 75% | 94.3% | `scripts/coverage_report.py`, `scripts/metrics.py --check-coverage` | | `io/trajectory.py` coverage | 85% | 95.0% | as above | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 744aa00..c17527e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,7 +52,7 @@ mypy src --ignore-missing-imports `ruff format src tests scripts` applies formatting; the `--check` form is the gate. -The suite collects 2320 tests and all of them pass on a fresh clone, except the deliberate +The suite collects 2355 tests and all of them pass on a fresh clone, except the deliberate xfails and the skips. `pytest -rs` lists the skips: two need TUM VI reference data that is not vendored (see below), and the doc-table check runs once `python scripts/run_benchmark.py` has written `results/benchmark.json`. The count and the other numbers the docs quote are in diff --git a/ROADMAP.md b/ROADMAP.md index ac8a020..774ee67 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,7 +5,7 @@ Status markers: `[x]` done, `[~]` in progress, `[ ]` not started, `[!]` blocked. Checked items record what was measured when the item closed. They are not kept current: the current values are in the README tables, `docs/results.md` and `CONSTRAINTS.md` (B1). -Measured baseline: 2320 tests collected and 94.3% line coverage on CPython +Measured baseline: 2355 tests collected and 94.3% line coverage on CPython 3.13 (`docs/data/metrics.json`, from `scripts/metrics.py` and `scripts/coverage_report.py`). Every number below re-measures against that, not against a remembered value. diff --git a/docs/adr/0026-urbannav-as-a-real-gnss-dataset.md b/docs/adr/0026-urbannav-as-a-real-gnss-dataset.md index c929470..07e6989 100644 --- a/docs/adr/0026-urbannav-as-a-real-gnss-dataset.md +++ b/docs/adr/0026-urbannav-as-a-real-gnss-dataset.md @@ -64,8 +64,9 @@ and is labelled as not blind. Both variants agree on the outage. One sequence, o ## What this does not settle -- The run is one sequence, one receiver and five start perturbations, not five independent tests. Deep-Urban-1, the - planned unseen check, has not been fetched or run. +- The run is one sequence, one receiver and five start perturbations, not five independent tests. Deep-Urban-1 has + since been run as an aided run, and its section of the page says what that does and does not show (see the dated + section below). - The reference inside the outage is an inertial estimate of unknown error. - The start attitude needs the rotation between the NovAtel and Xsens units, which is not in the files. - The F9P height is biased differently on the two sides of the outage, so a run would use the horizontal axes or declare @@ -124,3 +125,24 @@ not an outage sequence. The run for it is the aided run over the whole sequence: no oracle diagnostics (they act only inside an outage and are reported as not applicable), and quantities that do not need an outage: the normalised errors, the pooled 2-sigma coverage per axis, the gate's rejection rate and the fixes' own errors against the reference. Outages imposed on this sequence would be a separate experiment, with its own record. + +## Added 2026-10-24: the Deep-Urban-1 aided run + +This section is dated context. It does not change the Decision section above. + +Option A of the section above (the text reference only, no INSPVAX reader) was used, as the maintainer chose. The +receiver file reports a valid fix at every epoch, so the outage rule finds no outage, and the rule was not changed. +The run is the aided run over the whole sequence, with the declared and per-fix variants, five seeds that perturb +only the start state, and the oracle diagnostics not applicable. The rows are in `docs/data/urbannav_run.csv`, the +data rows in `docs/data/urbannav_data.csv`, and the new `docs/data/urbannav_fix_errors.csv` holds the time behaviour +of the receiver's fix errors for both sequences. Tunnel-1 rows in the first two files are byte-identical to before. + +What the run found, in words, with every number on [the UrbanNav page](../urbannav.md): the gate rejects none of the +offered fixes although some are far from the reference; the filter is overconfident against the reference in +position and attitude, and the excess is present in every third of the run; the receiver's fix errors are correlated +over tens of seconds on this sequence, which the filter does not model. The last point is an interpretation. The +filter was not run with a correlated-error model, and the reference is a post-processed estimate whose own error is +not known, so some of the correlated difference could be in the reference. + +What it does not settle: it is one sequence, one receiver and one recording, and the seeds are not independent tests. +It says nothing about the filter through an outage on this sequence. Imposed outages would be a separate experiment. diff --git a/docs/data/metrics.json b/docs/data/metrics.json index de00f1f..8962211 100644 --- a/docs/data/metrics.json +++ b/docs/data/metrics.json @@ -6,5 +6,5 @@ "coverage_python": "3.13", "outage_visual_window_coverage_percent": 5.0, "outage_visual_window_nees": 2783.35, - "tests_collected": 2320 + "tests_collected": 2355 } diff --git a/docs/data/urbannav_data.csv b/docs/data/urbannav_data.csv index 7d68547..628b15f 100644 --- a/docs/data/urbannav_data.csv +++ b/docs/data/urbannav_data.csv @@ -23,3 +23,18 @@ tunnel1,fix_error_first_valid_after_outage,52.76,m tunnel1,fix_error_max_first_valid_after_outage,52.76,m tunnel1,fix_vertical_bias_before_outage,-7.63,m tunnel1,fix_vertical_bias_after_outage,1.164,m +deepurban1,imu_samples,614782,count +deepurban1,imu_span,1537,s +deepurban1,imu_rate_median,400.5,Hz +deepurban1,imu_longest_gap,9.376,ms +deepurban1,reference_epochs,1539,count +deepurban1,reference_spacing_median,1,s +deepurban1,reference_quality_codes,5,count +deepurban1,clock_offset,0.01,s +deepurban1,clock_offset_grid_step,0.01,s +deepurban1,gnss_epochs,1539,count +deepurban1,gnss_denied_epochs,0,count +deepurban1,gnss_outages,0,count +deepurban1,fix_horizontal_error_median,2.361,m +deepurban1,fix_horizontal_error_p95,7.497,m +deepurban1,fix_horizontal_error_max,13.8,m diff --git a/docs/data/urbannav_fix_errors.csv b/docs/data/urbannav_fix_errors.csv new file mode 100644 index 0000000..3c0e496 --- /dev/null +++ b/docs/data/urbannav_fix_errors.csv @@ -0,0 +1,51 @@ +sequence,quantity,value,unit +tunnel1,fix_error_fixes,250,count +tunnel1,fix_error_lag_tolerance,0.5,s +tunnel1,fix_error_offset_east,-1.234,m +tunnel1,fix_error_offset_north,1.95,m +tunnel1,fix_error_rms_about_offset,10.87,m +tunnel1,fix_error_pairs_lag1s,248,count +tunnel1,fix_error_corr_east_lag1s,0.9405, +tunnel1,fix_error_corr_north_lag1s,0.9908, +tunnel1,fix_error_corr_pooled_lag1s,0.9704, +tunnel1,fix_error_pairs_lag10s,230,count +tunnel1,fix_error_corr_east_lag10s,0.07743, +tunnel1,fix_error_corr_north_lag10s,0.3562, +tunnel1,fix_error_corr_pooled_lag10s,0.2331, +tunnel1,fix_error_pairs_lag30s,190,count +tunnel1,fix_error_corr_east_lag30s,-0.05684, +tunnel1,fix_error_corr_north_lag30s,0.03377, +tunnel1,fix_error_corr_pooled_lag30s,-0.00567, +tunnel1,fix_error_pairs_lag60s,130,count +tunnel1,fix_error_corr_east_lag60s,-0.07971, +tunnel1,fix_error_corr_north_lag60s,-0.1313, +tunnel1,fix_error_corr_pooled_lag60s,-0.1038, +tunnel1,fix_error_ess_east,66.8,count +tunnel1,fix_error_ess_east_truncated,0,flag +tunnel1,fix_error_ess_north,18.33,count +tunnel1,fix_error_ess_north_truncated,0,flag +deepurban1,fix_error_fixes,1539,count +deepurban1,fix_error_lag_tolerance,0.5,s +deepurban1,fix_error_offset_east,0.9633,m +deepurban1,fix_error_offset_north,-1.233,m +deepurban1,fix_error_rms_about_offset,3.508,m +deepurban1,fix_error_pairs_lag1s,1538,count +deepurban1,fix_error_corr_east_lag1s,0.9983, +deepurban1,fix_error_corr_north_lag1s,0.9985, +deepurban1,fix_error_corr_pooled_lag1s,0.9984, +deepurban1,fix_error_pairs_lag10s,1529,count +deepurban1,fix_error_corr_east_lag10s,0.9455, +deepurban1,fix_error_corr_north_lag10s,0.95, +deepurban1,fix_error_corr_pooled_lag10s,0.9477, +deepurban1,fix_error_pairs_lag30s,1509,count +deepurban1,fix_error_corr_east_lag30s,0.7442, +deepurban1,fix_error_corr_north_lag30s,0.777, +deepurban1,fix_error_corr_pooled_lag30s,0.7599, +deepurban1,fix_error_pairs_lag60s,1479,count +deepurban1,fix_error_corr_east_lag60s,0.4217, +deepurban1,fix_error_corr_north_lag60s,0.5265, +deepurban1,fix_error_corr_pooled_lag60s,0.4719, +deepurban1,fix_error_ess_east,16.46,count +deepurban1,fix_error_ess_east_truncated,0,flag +deepurban1,fix_error_ess_north,17.87,count +deepurban1,fix_error_ess_north_truncated,0,flag diff --git a/docs/data/urbannav_run.csv b/docs/data/urbannav_run.csv index 09c5a16..3798c4e 100644 --- a/docs/data/urbannav_run.csv +++ b/docs/data/urbannav_run.csv @@ -924,3 +924,473 @@ tunnel1,oracle_attitude,4,gnss_reaccepted,0,count tunnel1,oracle_attitude,4,fixes_over_gate,7,count tunnel1,oracle_attitude,4,wrong_fixes,13,count tunnel1,oracle_attitude,4,wrong_fixes_over_gate,7,count +deepurban1,declared,0,fixes_total,1537,count +deepurban1,declared,0,fixes_used,1537,count +deepurban1,declared,0,gnss_updates_rejected,0,count +deepurban1,declared,0,fix_rejection_rate,0, +deepurban1,declared,0,fix_gate_alpha,0.001, +deepurban1,declared,0,fix_gate_level,16.27, +deepurban1,declared,0,nis_mean,0.5061, +deepurban1,declared,0,fix_rejections_expected,1.537,count +deepurban1,declared,0,fix_rejection_wilson_lo,0, +deepurban1,declared,0,fix_rejection_wilson_hi,0.002493, +deepurban1,declared,0,fix_rejection_ess,1537,count +deepurban1,declared,0,fix_rejection_ess_defined,0,flag +deepurban1,declared,0,fix_rejection_eff_wilson_lo,0, +deepurban1,declared,0,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,declared,0,gnss_channel_faulted,0,count +deepurban1,declared,0,gnss_reaccepted,0,count +deepurban1,declared,0,hor_nees_mean,13.84, +deepurban1,declared,0,hor_nees_third1,14.12, +deepurban1,declared,0,hor_nees_third2,22.44, +deepurban1,declared,0,hor_nees_third3,4.949, +deepurban1,declared,0,hor_nees_ess,18.03,count +deepurban1,declared,0,hor_nees_ess_truncated,0,flag +deepurban1,declared,0,att_nees_mean,14.12, +deepurban1,declared,0,att_nees_third1,20.27, +deepurban1,declared,0,att_nees_third2,12.89, +deepurban1,declared,0,att_nees_third3,9.183, +deepurban1,declared,0,att_nees_ess,62.38,count +deepurban1,declared,0,att_nees_ess_truncated,0,flag +deepurban1,declared,0,hor_coverage_2sigma,0.606, +deepurban1,declared,0,hor_inside_95,0.4459, +deepurban1,declared,0,hor_rmse_m,4.354,m +deepurban1,declared,0,ver_rmse_m,11.44,m +deepurban1,declared,0,ate_rmse_m,12.24,m +deepurban1,declared,0,fix_err_p50_m,2.361,m +deepurban1,declared,0,fix_err_p95_m,7.497,m +deepurban1,declared,0,fix_err_p99_m,12.97,m +deepurban1,declared,0,fixes_over_3sigma,166,count +deepurban1,declared,0,receiver_sigma_median_m,2.164,m +deepurban1,declared,0,claimed_sigma_median_m,1.173,m +deepurban1,declared,0,fix_nees_mean,3.285, +deepurban1,declared,0,fixes_over_gate,0,count +deepurban1,declared,0,wrong_fixes,48,count +deepurban1,declared,0,wrong_fixes_over_gate,0,count +deepurban1,declared,0,epochs_not_valid,0,count +deepurban1,declared,0,fixes_variance_unusable,0,count +deepurban1,declared,0,fixes_variance_below_floor,0,count +deepurban1,declared,0,start_bias_at_rest,0,flag +deepurban1,declared,1,fixes_total,1537,count +deepurban1,declared,1,fixes_used,1537,count +deepurban1,declared,1,gnss_updates_rejected,0,count +deepurban1,declared,1,fix_rejection_rate,0, +deepurban1,declared,1,fix_gate_alpha,0.001, +deepurban1,declared,1,fix_gate_level,16.27, +deepurban1,declared,1,nis_mean,0.496, +deepurban1,declared,1,fix_rejections_expected,1.537,count +deepurban1,declared,1,fix_rejection_wilson_lo,0, +deepurban1,declared,1,fix_rejection_wilson_hi,0.002493, +deepurban1,declared,1,fix_rejection_ess,1537,count +deepurban1,declared,1,fix_rejection_ess_defined,0,flag +deepurban1,declared,1,fix_rejection_eff_wilson_lo,0, +deepurban1,declared,1,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,declared,1,gnss_channel_faulted,0,count +deepurban1,declared,1,gnss_reaccepted,0,count +deepurban1,declared,1,hor_nees_mean,13.82, +deepurban1,declared,1,hor_nees_third1,14.07, +deepurban1,declared,1,hor_nees_third2,22.43, +deepurban1,declared,1,hor_nees_third3,4.951, +deepurban1,declared,1,hor_nees_ess,18.05,count +deepurban1,declared,1,hor_nees_ess_truncated,0,flag +deepurban1,declared,1,att_nees_mean,13.46, +deepurban1,declared,1,att_nees_third1,18.28, +deepurban1,declared,1,att_nees_third2,12.88, +deepurban1,declared,1,att_nees_third3,9.205, +deepurban1,declared,1,att_nees_ess,163.6,count +deepurban1,declared,1,att_nees_ess_truncated,0,flag +deepurban1,declared,1,hor_coverage_2sigma,0.6079, +deepurban1,declared,1,hor_inside_95,0.4488, +deepurban1,declared,1,hor_rmse_m,4.348,m +deepurban1,declared,1,ver_rmse_m,10.19,m +deepurban1,declared,1,ate_rmse_m,11.08,m +deepurban1,declared,1,fix_err_p50_m,2.361,m +deepurban1,declared,1,fix_err_p95_m,7.497,m +deepurban1,declared,1,fix_err_p99_m,12.97,m +deepurban1,declared,1,fixes_over_3sigma,166,count +deepurban1,declared,1,receiver_sigma_median_m,2.164,m +deepurban1,declared,1,claimed_sigma_median_m,1.173,m +deepurban1,declared,1,fix_nees_mean,3.285, +deepurban1,declared,1,fixes_over_gate,0,count +deepurban1,declared,1,wrong_fixes,48,count +deepurban1,declared,1,wrong_fixes_over_gate,0,count +deepurban1,declared,1,epochs_not_valid,0,count +deepurban1,declared,1,fixes_variance_unusable,0,count +deepurban1,declared,1,fixes_variance_below_floor,0,count +deepurban1,declared,1,start_bias_at_rest,0,flag +deepurban1,declared,2,fixes_total,1537,count +deepurban1,declared,2,fixes_used,1537,count +deepurban1,declared,2,gnss_updates_rejected,0,count +deepurban1,declared,2,fix_rejection_rate,0, +deepurban1,declared,2,fix_gate_alpha,0.001, +deepurban1,declared,2,fix_gate_level,16.27, +deepurban1,declared,2,nis_mean,0.5026, +deepurban1,declared,2,fix_rejections_expected,1.537,count +deepurban1,declared,2,fix_rejection_wilson_lo,0, +deepurban1,declared,2,fix_rejection_wilson_hi,0.002493, +deepurban1,declared,2,fix_rejection_ess,1537,count +deepurban1,declared,2,fix_rejection_ess_defined,0,flag +deepurban1,declared,2,fix_rejection_eff_wilson_lo,0, +deepurban1,declared,2,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,declared,2,gnss_channel_faulted,0,count +deepurban1,declared,2,gnss_reaccepted,0,count +deepurban1,declared,2,hor_nees_mean,13.84, +deepurban1,declared,2,hor_nees_third1,14.13, +deepurban1,declared,2,hor_nees_third2,22.43, +deepurban1,declared,2,hor_nees_third3,4.952, +deepurban1,declared,2,hor_nees_ess,18.06,count +deepurban1,declared,2,hor_nees_ess_truncated,0,flag +deepurban1,declared,2,att_nees_mean,13.64, +deepurban1,declared,2,att_nees_third1,18.81, +deepurban1,declared,2,att_nees_third2,12.88, +deepurban1,declared,2,att_nees_third3,9.216, +deepurban1,declared,2,att_nees_ess,160.7,count +deepurban1,declared,2,att_nees_ess_truncated,0,flag +deepurban1,declared,2,hor_coverage_2sigma,0.6075, +deepurban1,declared,2,hor_inside_95,0.4512, +deepurban1,declared,2,hor_rmse_m,4.351,m +deepurban1,declared,2,ver_rmse_m,10.56,m +deepurban1,declared,2,ate_rmse_m,11.43,m +deepurban1,declared,2,fix_err_p50_m,2.361,m +deepurban1,declared,2,fix_err_p95_m,7.497,m +deepurban1,declared,2,fix_err_p99_m,12.97,m +deepurban1,declared,2,fixes_over_3sigma,166,count +deepurban1,declared,2,receiver_sigma_median_m,2.164,m +deepurban1,declared,2,claimed_sigma_median_m,1.173,m +deepurban1,declared,2,fix_nees_mean,3.285, +deepurban1,declared,2,fixes_over_gate,0,count +deepurban1,declared,2,wrong_fixes,48,count +deepurban1,declared,2,wrong_fixes_over_gate,0,count +deepurban1,declared,2,epochs_not_valid,0,count +deepurban1,declared,2,fixes_variance_unusable,0,count +deepurban1,declared,2,fixes_variance_below_floor,0,count +deepurban1,declared,2,start_bias_at_rest,0,flag +deepurban1,declared,3,fixes_total,1537,count +deepurban1,declared,3,fixes_used,1537,count +deepurban1,declared,3,gnss_updates_rejected,0,count +deepurban1,declared,3,fix_rejection_rate,0, +deepurban1,declared,3,fix_gate_alpha,0.001, +deepurban1,declared,3,fix_gate_level,16.27, +deepurban1,declared,3,nis_mean,0.4989, +deepurban1,declared,3,fix_rejections_expected,1.537,count +deepurban1,declared,3,fix_rejection_wilson_lo,0, +deepurban1,declared,3,fix_rejection_wilson_hi,0.002493, +deepurban1,declared,3,fix_rejection_ess,1537,count +deepurban1,declared,3,fix_rejection_ess_defined,0,flag +deepurban1,declared,3,fix_rejection_eff_wilson_lo,0, +deepurban1,declared,3,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,declared,3,gnss_channel_faulted,0,count +deepurban1,declared,3,gnss_reaccepted,0,count +deepurban1,declared,3,hor_nees_mean,13.82, +deepurban1,declared,3,hor_nees_third1,14.09, +deepurban1,declared,3,hor_nees_third2,22.44, +deepurban1,declared,3,hor_nees_third3,4.951, +deepurban1,declared,3,hor_nees_ess,18.05,count +deepurban1,declared,3,hor_nees_ess_truncated,0,flag +deepurban1,declared,3,att_nees_mean,13.5, +deepurban1,declared,3,att_nees_third1,18.43, +deepurban1,declared,3,att_nees_third2,12.88, +deepurban1,declared,3,att_nees_third3,9.199, +deepurban1,declared,3,att_nees_ess,159,count +deepurban1,declared,3,att_nees_ess_truncated,0,flag +deepurban1,declared,3,hor_coverage_2sigma,0.6078, +deepurban1,declared,3,hor_inside_95,0.4522, +deepurban1,declared,3,hor_rmse_m,4.352,m +deepurban1,declared,3,ver_rmse_m,10.25,m +deepurban1,declared,3,ate_rmse_m,11.14,m +deepurban1,declared,3,fix_err_p50_m,2.361,m +deepurban1,declared,3,fix_err_p95_m,7.497,m +deepurban1,declared,3,fix_err_p99_m,12.97,m +deepurban1,declared,3,fixes_over_3sigma,166,count +deepurban1,declared,3,receiver_sigma_median_m,2.164,m +deepurban1,declared,3,claimed_sigma_median_m,1.173,m +deepurban1,declared,3,fix_nees_mean,3.285, +deepurban1,declared,3,fixes_over_gate,0,count +deepurban1,declared,3,wrong_fixes,48,count +deepurban1,declared,3,wrong_fixes_over_gate,0,count +deepurban1,declared,3,epochs_not_valid,0,count +deepurban1,declared,3,fixes_variance_unusable,0,count +deepurban1,declared,3,fixes_variance_below_floor,0,count +deepurban1,declared,3,start_bias_at_rest,0,flag +deepurban1,declared,4,fixes_total,1537,count +deepurban1,declared,4,fixes_used,1537,count +deepurban1,declared,4,gnss_updates_rejected,0,count +deepurban1,declared,4,fix_rejection_rate,0, +deepurban1,declared,4,fix_gate_alpha,0.001, +deepurban1,declared,4,fix_gate_level,16.27, +deepurban1,declared,4,nis_mean,0.4979, +deepurban1,declared,4,fix_rejections_expected,1.537,count +deepurban1,declared,4,fix_rejection_wilson_lo,0, +deepurban1,declared,4,fix_rejection_wilson_hi,0.002493, +deepurban1,declared,4,fix_rejection_ess,1537,count +deepurban1,declared,4,fix_rejection_ess_defined,0,flag +deepurban1,declared,4,fix_rejection_eff_wilson_lo,0, +deepurban1,declared,4,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,declared,4,gnss_channel_faulted,0,count +deepurban1,declared,4,gnss_reaccepted,0,count +deepurban1,declared,4,hor_nees_mean,13.82, +deepurban1,declared,4,hor_nees_third1,14.07, +deepurban1,declared,4,hor_nees_third2,22.44, +deepurban1,declared,4,hor_nees_third3,4.951, +deepurban1,declared,4,hor_nees_ess,18.05,count +deepurban1,declared,4,hor_nees_ess_truncated,0,flag +deepurban1,declared,4,att_nees_mean,13.49, +deepurban1,declared,4,att_nees_third1,18.4, +deepurban1,declared,4,att_nees_third2,12.88, +deepurban1,declared,4,att_nees_third3,9.199, +deepurban1,declared,4,att_nees_ess,158.5,count +deepurban1,declared,4,att_nees_ess_truncated,0,flag +deepurban1,declared,4,hor_coverage_2sigma,0.6081, +deepurban1,declared,4,hor_inside_95,0.4538, +deepurban1,declared,4,hor_rmse_m,4.352,m +deepurban1,declared,4,ver_rmse_m,10.4,m +deepurban1,declared,4,ate_rmse_m,11.28,m +deepurban1,declared,4,fix_err_p50_m,2.361,m +deepurban1,declared,4,fix_err_p95_m,7.497,m +deepurban1,declared,4,fix_err_p99_m,12.97,m +deepurban1,declared,4,fixes_over_3sigma,166,count +deepurban1,declared,4,receiver_sigma_median_m,2.164,m +deepurban1,declared,4,claimed_sigma_median_m,1.173,m +deepurban1,declared,4,fix_nees_mean,3.285, +deepurban1,declared,4,fixes_over_gate,0,count +deepurban1,declared,4,wrong_fixes,48,count +deepurban1,declared,4,wrong_fixes_over_gate,0,count +deepurban1,declared,4,epochs_not_valid,0,count +deepurban1,declared,4,fixes_variance_unusable,0,count +deepurban1,declared,4,fixes_variance_below_floor,0,count +deepurban1,declared,4,start_bias_at_rest,0,flag +deepurban1,per_fix,0,fixes_total,1537,count +deepurban1,per_fix,0,fixes_used,1537,count +deepurban1,per_fix,0,gnss_updates_rejected,0,count +deepurban1,per_fix,0,fix_rejection_rate,0, +deepurban1,per_fix,0,fix_gate_alpha,0.001, +deepurban1,per_fix,0,fix_gate_level,16.27, +deepurban1,per_fix,0,nis_mean,0.5249, +deepurban1,per_fix,0,fix_rejections_expected,1.537,count +deepurban1,per_fix,0,fix_rejection_wilson_lo,0, +deepurban1,per_fix,0,fix_rejection_wilson_hi,0.002493, +deepurban1,per_fix,0,fix_rejection_ess,1537,count +deepurban1,per_fix,0,fix_rejection_ess_defined,0,flag +deepurban1,per_fix,0,fix_rejection_eff_wilson_lo,0, +deepurban1,per_fix,0,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,per_fix,0,gnss_channel_faulted,0,count +deepurban1,per_fix,0,gnss_reaccepted,0,count +deepurban1,per_fix,0,hor_nees_mean,15.08, +deepurban1,per_fix,0,hor_nees_third1,18.97, +deepurban1,per_fix,0,hor_nees_third2,20.51, +deepurban1,per_fix,0,hor_nees_third3,5.772, +deepurban1,per_fix,0,hor_nees_ess,13.55,count +deepurban1,per_fix,0,hor_nees_ess_truncated,0,flag +deepurban1,per_fix,0,att_nees_mean,13.33, +deepurban1,per_fix,0,att_nees_third1,18.9, +deepurban1,per_fix,0,att_nees_third2,11.64, +deepurban1,per_fix,0,att_nees_third3,9.448, +deepurban1,per_fix,0,att_nees_ess,178.6,count +deepurban1,per_fix,0,att_nees_ess_truncated,0,flag +deepurban1,per_fix,0,hor_coverage_2sigma,0.5734, +deepurban1,per_fix,0,hor_inside_95,0.4242, +deepurban1,per_fix,0,hor_rmse_m,4.302,m +deepurban1,per_fix,0,ver_rmse_m,11.38,m +deepurban1,per_fix,0,ate_rmse_m,12.17,m +deepurban1,per_fix,0,fix_err_p50_m,2.361,m +deepurban1,per_fix,0,fix_err_p95_m,7.497,m +deepurban1,per_fix,0,fix_err_p99_m,12.97,m +deepurban1,per_fix,0,fixes_over_3sigma,166,count +deepurban1,per_fix,0,receiver_sigma_median_m,2.164,m +deepurban1,per_fix,0,claimed_sigma_median_m,1.137,m +deepurban1,per_fix,0,fix_nees_mean,3.285, +deepurban1,per_fix,0,fixes_over_gate,0,count +deepurban1,per_fix,0,wrong_fixes,48,count +deepurban1,per_fix,0,wrong_fixes_over_gate,0,count +deepurban1,per_fix,0,epochs_not_valid,0,count +deepurban1,per_fix,0,fixes_variance_unusable,0,count +deepurban1,per_fix,0,fixes_variance_below_floor,0,count +deepurban1,per_fix,0,start_bias_at_rest,0,flag +deepurban1,per_fix,1,fixes_total,1537,count +deepurban1,per_fix,1,fixes_used,1537,count +deepurban1,per_fix,1,gnss_updates_rejected,0,count +deepurban1,per_fix,1,fix_rejection_rate,0, +deepurban1,per_fix,1,fix_gate_alpha,0.001, +deepurban1,per_fix,1,fix_gate_level,16.27, +deepurban1,per_fix,1,nis_mean,0.5132, +deepurban1,per_fix,1,fix_rejections_expected,1.537,count +deepurban1,per_fix,1,fix_rejection_wilson_lo,0, +deepurban1,per_fix,1,fix_rejection_wilson_hi,0.002493, +deepurban1,per_fix,1,fix_rejection_ess,1537,count +deepurban1,per_fix,1,fix_rejection_ess_defined,0,flag +deepurban1,per_fix,1,fix_rejection_eff_wilson_lo,0, +deepurban1,per_fix,1,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,per_fix,1,gnss_channel_faulted,0,count +deepurban1,per_fix,1,gnss_reaccepted,0,count +deepurban1,per_fix,1,hor_nees_mean,15.07, +deepurban1,per_fix,1,hor_nees_third1,18.94, +deepurban1,per_fix,1,hor_nees_third2,20.51, +deepurban1,per_fix,1,hor_nees_third3,5.774, +deepurban1,per_fix,1,hor_nees_ess,13.6,count +deepurban1,per_fix,1,hor_nees_ess_truncated,0,flag +deepurban1,per_fix,1,att_nees_mean,13.07, +deepurban1,per_fix,1,att_nees_third1,18.03, +deepurban1,per_fix,1,att_nees_third2,11.71, +deepurban1,per_fix,1,att_nees_third3,9.479, +deepurban1,per_fix,1,att_nees_ess,195.6,count +deepurban1,per_fix,1,att_nees_ess_truncated,0,flag +deepurban1,per_fix,1,hor_coverage_2sigma,0.5719, +deepurban1,per_fix,1,hor_inside_95,0.4243, +deepurban1,per_fix,1,hor_rmse_m,4.301,m +deepurban1,per_fix,1,ver_rmse_m,10.14,m +deepurban1,per_fix,1,ate_rmse_m,11.01,m +deepurban1,per_fix,1,fix_err_p50_m,2.361,m +deepurban1,per_fix,1,fix_err_p95_m,7.497,m +deepurban1,per_fix,1,fix_err_p99_m,12.97,m +deepurban1,per_fix,1,fixes_over_3sigma,166,count +deepurban1,per_fix,1,receiver_sigma_median_m,2.164,m +deepurban1,per_fix,1,claimed_sigma_median_m,1.137,m +deepurban1,per_fix,1,fix_nees_mean,3.285, +deepurban1,per_fix,1,fixes_over_gate,0,count +deepurban1,per_fix,1,wrong_fixes,48,count +deepurban1,per_fix,1,wrong_fixes_over_gate,0,count +deepurban1,per_fix,1,epochs_not_valid,0,count +deepurban1,per_fix,1,fixes_variance_unusable,0,count +deepurban1,per_fix,1,fixes_variance_below_floor,0,count +deepurban1,per_fix,1,start_bias_at_rest,0,flag +deepurban1,per_fix,2,fixes_total,1537,count +deepurban1,per_fix,2,fixes_used,1537,count +deepurban1,per_fix,2,gnss_updates_rejected,0,count +deepurban1,per_fix,2,fix_rejection_rate,0, +deepurban1,per_fix,2,fix_gate_alpha,0.001, +deepurban1,per_fix,2,fix_gate_level,16.27, +deepurban1,per_fix,2,nis_mean,0.5206, +deepurban1,per_fix,2,fix_rejections_expected,1.537,count +deepurban1,per_fix,2,fix_rejection_wilson_lo,0, +deepurban1,per_fix,2,fix_rejection_wilson_hi,0.002493, +deepurban1,per_fix,2,fix_rejection_ess,1537,count +deepurban1,per_fix,2,fix_rejection_ess_defined,0,flag +deepurban1,per_fix,2,fix_rejection_eff_wilson_lo,0, +deepurban1,per_fix,2,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,per_fix,2,gnss_channel_faulted,0,count +deepurban1,per_fix,2,gnss_reaccepted,0,count +deepurban1,per_fix,2,hor_nees_mean,15.08, +deepurban1,per_fix,2,hor_nees_third1,18.97, +deepurban1,per_fix,2,hor_nees_third2,20.5, +deepurban1,per_fix,2,hor_nees_third3,5.775, +deepurban1,per_fix,2,hor_nees_ess,13.59,count +deepurban1,per_fix,2,hor_nees_ess_truncated,0,flag +deepurban1,per_fix,2,att_nees_mean,13.26, +deepurban1,per_fix,2,att_nees_third1,18.54, +deepurban1,per_fix,2,att_nees_third2,11.73, +deepurban1,per_fix,2,att_nees_third3,9.49, +deepurban1,per_fix,2,att_nees_ess,188.2,count +deepurban1,per_fix,2,att_nees_ess_truncated,0,flag +deepurban1,per_fix,2,hor_coverage_2sigma,0.5714, +deepurban1,per_fix,2,hor_inside_95,0.4256, +deepurban1,per_fix,2,hor_rmse_m,4.302,m +deepurban1,per_fix,2,ver_rmse_m,10.52,m +deepurban1,per_fix,2,ate_rmse_m,11.36,m +deepurban1,per_fix,2,fix_err_p50_m,2.361,m +deepurban1,per_fix,2,fix_err_p95_m,7.497,m +deepurban1,per_fix,2,fix_err_p99_m,12.97,m +deepurban1,per_fix,2,fixes_over_3sigma,166,count +deepurban1,per_fix,2,receiver_sigma_median_m,2.164,m +deepurban1,per_fix,2,claimed_sigma_median_m,1.137,m +deepurban1,per_fix,2,fix_nees_mean,3.285, +deepurban1,per_fix,2,fixes_over_gate,0,count +deepurban1,per_fix,2,wrong_fixes,48,count +deepurban1,per_fix,2,wrong_fixes_over_gate,0,count +deepurban1,per_fix,2,epochs_not_valid,0,count +deepurban1,per_fix,2,fixes_variance_unusable,0,count +deepurban1,per_fix,2,fixes_variance_below_floor,0,count +deepurban1,per_fix,2,start_bias_at_rest,0,flag +deepurban1,per_fix,3,fixes_total,1537,count +deepurban1,per_fix,3,fixes_used,1537,count +deepurban1,per_fix,3,gnss_updates_rejected,0,count +deepurban1,per_fix,3,fix_rejection_rate,0, +deepurban1,per_fix,3,fix_gate_alpha,0.001, +deepurban1,per_fix,3,fix_gate_level,16.27, +deepurban1,per_fix,3,nis_mean,0.5164, +deepurban1,per_fix,3,fix_rejections_expected,1.537,count +deepurban1,per_fix,3,fix_rejection_wilson_lo,0, +deepurban1,per_fix,3,fix_rejection_wilson_hi,0.002493, +deepurban1,per_fix,3,fix_rejection_ess,1537,count +deepurban1,per_fix,3,fix_rejection_ess_defined,0,flag +deepurban1,per_fix,3,fix_rejection_eff_wilson_lo,0, +deepurban1,per_fix,3,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,per_fix,3,gnss_channel_faulted,0,count +deepurban1,per_fix,3,gnss_reaccepted,0,count +deepurban1,per_fix,3,hor_nees_mean,15.07, +deepurban1,per_fix,3,hor_nees_third1,18.94, +deepurban1,per_fix,3,hor_nees_third2,20.51, +deepurban1,per_fix,3,hor_nees_third3,5.774, +deepurban1,per_fix,3,hor_nees_ess,13.58,count +deepurban1,per_fix,3,hor_nees_ess_truncated,0,flag +deepurban1,per_fix,3,att_nees_mean,13.07, +deepurban1,per_fix,3,att_nees_third1,18.04, +deepurban1,per_fix,3,att_nees_third2,11.69, +deepurban1,per_fix,3,att_nees_third3,9.471, +deepurban1,per_fix,3,att_nees_ess,195.8,count +deepurban1,per_fix,3,att_nees_ess_truncated,0,flag +deepurban1,per_fix,3,hor_coverage_2sigma,0.5714, +deepurban1,per_fix,3,hor_inside_95,0.4223, +deepurban1,per_fix,3,hor_rmse_m,4.302,m +deepurban1,per_fix,3,ver_rmse_m,10.19,m +deepurban1,per_fix,3,ate_rmse_m,11.06,m +deepurban1,per_fix,3,fix_err_p50_m,2.361,m +deepurban1,per_fix,3,fix_err_p95_m,7.497,m +deepurban1,per_fix,3,fix_err_p99_m,12.97,m +deepurban1,per_fix,3,fixes_over_3sigma,166,count +deepurban1,per_fix,3,receiver_sigma_median_m,2.164,m +deepurban1,per_fix,3,claimed_sigma_median_m,1.137,m +deepurban1,per_fix,3,fix_nees_mean,3.285, +deepurban1,per_fix,3,fixes_over_gate,0,count +deepurban1,per_fix,3,wrong_fixes,48,count +deepurban1,per_fix,3,wrong_fixes_over_gate,0,count +deepurban1,per_fix,3,epochs_not_valid,0,count +deepurban1,per_fix,3,fixes_variance_unusable,0,count +deepurban1,per_fix,3,fixes_variance_below_floor,0,count +deepurban1,per_fix,3,start_bias_at_rest,0,flag +deepurban1,per_fix,4,fixes_total,1537,count +deepurban1,per_fix,4,fixes_used,1537,count +deepurban1,per_fix,4,gnss_updates_rejected,0,count +deepurban1,per_fix,4,fix_rejection_rate,0, +deepurban1,per_fix,4,fix_gate_alpha,0.001, +deepurban1,per_fix,4,fix_gate_level,16.27, +deepurban1,per_fix,4,nis_mean,0.5152, +deepurban1,per_fix,4,fix_rejections_expected,1.537,count +deepurban1,per_fix,4,fix_rejection_wilson_lo,0, +deepurban1,per_fix,4,fix_rejection_wilson_hi,0.002493, +deepurban1,per_fix,4,fix_rejection_ess,1537,count +deepurban1,per_fix,4,fix_rejection_ess_defined,0,flag +deepurban1,per_fix,4,fix_rejection_eff_wilson_lo,0, +deepurban1,per_fix,4,fix_rejection_eff_wilson_hi,0.002493, +deepurban1,per_fix,4,gnss_channel_faulted,0,count +deepurban1,per_fix,4,gnss_reaccepted,0,count +deepurban1,per_fix,4,hor_nees_mean,15.07, +deepurban1,per_fix,4,hor_nees_third1,18.93, +deepurban1,per_fix,4,hor_nees_third2,20.51, +deepurban1,per_fix,4,hor_nees_third3,5.774, +deepurban1,per_fix,4,hor_nees_ess,13.59,count +deepurban1,per_fix,4,hor_nees_ess_truncated,0,flag +deepurban1,per_fix,4,att_nees_mean,13.05, +deepurban1,per_fix,4,att_nees_third1,18, +deepurban1,per_fix,4,att_nees_third2,11.69, +deepurban1,per_fix,4,att_nees_third3,9.471, +deepurban1,per_fix,4,att_nees_ess,195.9,count +deepurban1,per_fix,4,att_nees_ess_truncated,0,flag +deepurban1,per_fix,4,hor_coverage_2sigma,0.5713, +deepurban1,per_fix,4,hor_inside_95,0.422, +deepurban1,per_fix,4,hor_rmse_m,4.302,m +deepurban1,per_fix,4,ver_rmse_m,10.34,m +deepurban1,per_fix,4,ate_rmse_m,11.2,m +deepurban1,per_fix,4,fix_err_p50_m,2.361,m +deepurban1,per_fix,4,fix_err_p95_m,7.497,m +deepurban1,per_fix,4,fix_err_p99_m,12.97,m +deepurban1,per_fix,4,fixes_over_3sigma,166,count +deepurban1,per_fix,4,receiver_sigma_median_m,2.164,m +deepurban1,per_fix,4,claimed_sigma_median_m,1.137,m +deepurban1,per_fix,4,fix_nees_mean,3.285, +deepurban1,per_fix,4,fixes_over_gate,0,count +deepurban1,per_fix,4,wrong_fixes,48,count +deepurban1,per_fix,4,wrong_fixes_over_gate,0,count +deepurban1,per_fix,4,epochs_not_valid,0,count +deepurban1,per_fix,4,fixes_variance_unusable,0,count +deepurban1,per_fix,4,fixes_variance_below_floor,0,count +deepurban1,per_fix,4,start_bias_at_rest,0,flag diff --git a/docs/product_management/05_work_breakdown.md b/docs/product_management/05_work_breakdown.md index 3e1d244..7c51e9f 100644 --- a/docs/product_management/05_work_breakdown.md +++ b/docs/product_management/05_work_breakdown.md @@ -27,7 +27,7 @@ TUM VI data path was M. No hours are quoted, because none were recorded. | WB-6 | Remaining P5-09 and P5-12 | Baseline records and the pull-request clean-up | None | S | Stated in the roadmap item. | | WB-7 | UrbanNav | A licence answer, then a third dataset | Issue 94 | M | The licence permits the use, or the dataset is dropped. | -WB-1 is done: the [vision-ramp study](../vision_ramp.md) puts the floor at about 2 m/s for independent visual errors. WB-2 has a feasibility result: the [fault classification study](../fault_classification.md) separates multipath, calls spoofing in about three quarters of runs and has a control false-alarm rate of 35% on unseen runs, so the exit test is not met and the package stays open. WB-6 is done: the baseline records are dated snapshots by decision, and no housekeeping item is open. WB-4 is done: option 3 was taken and B4 is resolved (R5 in `CONSTRAINTS.md`). WB-7 is unblocked on the licence question, and a first run on Tunnel-1 exists: see section 5 and the [UrbanNav page](../urbannav.md). +WB-1 is done: the [vision-ramp study](../vision_ramp.md) puts the floor at about 2 m/s for independent visual errors. WB-2 has a feasibility result: the [fault classification study](../fault_classification.md) separates multipath, calls spoofing in about three quarters of runs and has a control false-alarm rate of 35% on unseen runs, so the exit test is not met and the package stays open. WB-6 is done: the baseline records are dated snapshots by decision, and no housekeeping item is open. WB-4 is done: option 3 was taken and B4 is resolved (R5 in `CONSTRAINTS.md`). WB-7 is unblocked on the licence question, and a first run on Tunnel-1 and an aided run on Deep-Urban-1 exist: see section 5 and the [UrbanNav page](../urbannav.md). ## 3. B4: the published TUM VI estimate @@ -95,4 +95,4 @@ at 1 Hz and the comparison uses linear interpolation of its position. 5 is done: was fixed before any filter ran. 6 is a draft, with the decision section left to the maintainer. The exit test in the table is met on the licence, as step 1 confirms. A first run exists on Tunnel-1 (see the -[UrbanNav page](../urbannav.md)). The package stays open until the unseen check on Deep-Urban-1 has been run. +[UrbanNav page](../urbannav.md)). Deep-Urban-1 has been run as an aided run (it has no outage by the rule), which the page describes. Imposed outages on it would be a separate experiment. diff --git a/docs/urbannav.md b/docs/urbannav.md index 6c6db6a..e0ace7e 100644 --- a/docs/urbannav.md +++ b/docs/urbannav.md @@ -3,7 +3,8 @@ The first dataset here with a real GNSS receiver and a real GNSS outage. The reader is `navkit.urbannav_data`, the measurements below come from `navkit urbannav check`, and the decision behind the dataset is [ADR-0026](adr/0026-urbannav-as-a-real-gnss-dataset.md). This page says what the files contain, then what the filter -did on the Tunnel-1 recording with the receiver's own fixes and its real outage. +did on the Tunnel-1 recording with the receiver's own fixes and its real outage, then what it did on Deep-Urban-1, +where the receiver reports no outage. ## Read this before quoting anything @@ -65,6 +66,21 @@ did on the Tunnel-1 recording with the receiver's own fixes and its real outage. | tunnel1 | Largest difference in the first valid fixes after the outage | 52.76 | m | | tunnel1 | Mean fix height minus reference height, before the outage | -7.63 | m | | tunnel1 | Mean fix height minus reference height, after the outage | 1.164 | m | +| deepurban1 | IMU samples | 614782 | count | +| deepurban1 | IMU time span | 1537 | s | +| deepurban1 | IMU rate, median | 400.5 | Hz | +| deepurban1 | IMU longest gap | 9.376 | ms | +| deepurban1 | Reference epochs | 1539 | count | +| deepurban1 | Reference spacing, median | 1 | s | +| deepurban1 | Distinct reference quality codes | 5 | count | +| deepurban1 | Reference-to-IMU clock offset | 0.01 | s | +| deepurban1 | Offset search step | 0.01 | s | +| deepurban1 | F9P fixes (GGA sentences) | 1539 | count | +| deepurban1 | Epochs with no fix | 0 | count | +| deepurban1 | Outages by the rule | 0 | count | +| deepurban1 | Fix horizontal difference from the reference, median | 2.361 | m | +| deepurban1 | Fix horizontal difference, 95th percentile | 7.497 | m | +| deepurban1 | Fix horizontal difference, largest | 13.8 | m | @@ -265,7 +281,8 @@ Every statement is about one sequence and one receiver, and points at rows of th bias inferred from the data over the stretch before the outage, but starting the filter from the inferred biases in a scratch run (no code in the repository reproduces it) still ended with a kilometre-scale error, so the growth is not explained. -- Deep-Urban-1, the planned unseen check, has not been fetched or run. +- It does not say how the filter does on a sequence where the receiver reports no outage. The Deep-Urban-1 section + below is that aided run, and it is not an outage test. ## What it shows about the data @@ -278,35 +295,88 @@ Every statement is about one sequence and one receiver, and points at rows of th ## Deep-Urban-1: real urban GNSS, no outage -**Status: not run for this page.** The tables in this section are generated from the committed CSV. It has no rows for -this sequence yet, and they say so. Until it has rows, nothing on this page is a result about Deep-Urban-1. +**Status: run for this page.** The tables in this section are generated from the committed CSVs. They are one +sequence, one receiver and one recording. They are not field performance. - **One sequence, one receiver, one recording.** The seeds perturb only the start state, as for Tunnel-1. - **The receiver reports no outage here.** The outage rule is the one used for Tunnel-1, and it was not changed or tuned to find one (issue #95). So this is not an outage test. It is the **aided run**: the filter is offered the - receiver's own fixes over the whole recording, and the question is how it handles them. -- **The settings are the ones declared above, unchanged.** Both variants are fixed before this sequence is run. Neither - is tuned on it. The per-fix variant was added after the first Tunnel-1 result, so on Tunnel-1 it is not blind. Here it - is declared in advance. + receiver's own fixes over the whole recording, and the question is how it handles them. The filter is offered the + fixes that fall inside the IMU recording, which is two fewer than the receiver wrote. +- **The settings are the ones declared above, unchanged.** Both variants were fixed before this sequence was run. + Neither was tuned on it. The per-fix variant was added after the first Tunnel-1 result, so on Tunnel-1 it is not + blind. Here it was declared in advance. - **The oracle diagnostics are not applicable.** They replace part of the attitude inside an outage, and there is none. The block below says so. - **The reference is the same kind of file as for Tunnel-1:** the post-processed NovAtel SPAN-CPT solution, read from - the text file and not from the bag. Between tall buildings its own error is unknown, and every score here is a score + the text file and not from the bag (the ADR records this choice). Between tall buildings its own error is not + known. Its quality column has several codes, and what each means has not been confirmed. Every score here is a score against an estimate. +- **The height is not relied on.** The vertical noise is declared very large, so the vertical error is large and the + page makes no claim about it. The absolute trajectory error is in the CSV and not in the tables, because it is a + three-dimensional number that the vertical error dominates. The tables quote the horizontal error. What the section tests: 1. **Whether the filter is right about how wrong it is on real urban GNSS errors.** The rows are the horizontal and - attitude normalised errors over the whole run, the pooled 2-sigma coverage per axis, and the fixes' own normalised - error against the receiver's reported sigma. -2. **How the gate behaves on real multipath fixes.** The rows are the share of fixes it rejects, whether the channel - is declared faulty, and whether it is taken back. + attitude normalised errors over the whole run and by third of the run, the pooled 2-sigma coverage per axis, the + sigma the filter claims against the receiver's own, and the fixes' own normalised error against the receiver's + reported sigma. +2. **How the gate behaves on real multipath fixes.** The rows are the fixes it rejects, whether the channel is + declared faulty, and the NIS against the gate's level. **As declared**, one scalar GNSS sigma for every fix: -No rows for Deep-Urban-1 are in the committed CSV, so it has not been run for this page. The table appears here when the CSV has its rows. +| Quantity | seed 0 | seed 1 | seed 2 | seed 3 | seed 4 | Unit | +|---|---:|---:|---:|---:|---:|---| +| Valid fixes offered to the filter | 1537 | 1537 | 1537 | 1537 | 1537 | count | +| Fixes the filter used | 1537 | 1537 | 1537 | 1537 | 1537 | count | +| Fixes the filter rejected | 0 | 0 | 0 | 0 | 0 | count | +| Share of the offered fixes the gate rejected | 0 | 0 | 0 | 0 | 0 | | +| The gate's alpha (the FDIR default) | 0.001 | 0.001 | 0.001 | 0.001 | 0.001 | | +| The gate's level on the NIS (3 degrees of freedom) | 16.27 | 16.27 | 16.27 | 16.27 | 16.27 | | +| Mean NIS of the offered fixes | 0.5061 | 0.496 | 0.5026 | 0.4989 | 0.4979 | | +| Rejections expected at that alpha if every fix were consistent (offered x alpha) | 1.537 | 1.537 | 1.537 | 1.537 | 1.537 | count | +| Wilson 95% interval of the rejection rate on the raw count, lower end | 0 | 0 | 0 | 0 | 0 | | +| Wilson 95% interval of the rejection rate on the raw count, upper end | 0.002493 | 0.002493 | 0.002493 | 0.002493 | 0.002493 | | +| Effective number of fixes behind the rate (effective sample size of the over-the-level series) | 1537 | 1537 | 1537 | 1537 | 1537 | count | +| That size could be estimated (1 yes, 0 no: the series is constant, so it counts every fix) | 0 | 0 | 0 | 0 | 0 | flag | +| Wilson 95% interval on the effective count, lower end | 0 | 0 | 0 | 0 | 0 | | +| Wilson 95% interval on the effective count, upper end | 0.002493 | 0.002493 | 0.002493 | 0.002493 | 0.002493 | | +| GNSS channel declared faulty by the end | 0 | 0 | 0 | 0 | 0 | count | +| Times a faulty channel was taken back | 0 | 0 | 0 | 0 | 0 | count | +| Horizontal mean NEES over the whole run (2 degrees of freedom, expected 2) | 13.84 | 13.82 | 13.84 | 13.82 | 13.82 | | +| Horizontal mean NEES, first third of the run | 14.12 | 14.07 | 14.13 | 14.09 | 14.07 | | +| Horizontal mean NEES, middle third of the run | 22.44 | 22.43 | 22.43 | 22.44 | 22.44 | | +| Horizontal mean NEES, last third of the run | 4.949 | 4.951 | 4.952 | 4.951 | 4.951 | | +| Effective sample size of the horizontal NEES series (it has one value per IMU step) | 18.03 | 18.05 | 18.06 | 18.05 | 18.05 | count | +| That size is an upper bound (1 yes, 0 no) | 0 | 0 | 0 | 0 | 0 | flag | +| Attitude mean NEES over the whole run (3 degrees of freedom, expected 3) | 14.12 | 13.46 | 13.64 | 13.5 | 13.49 | | +| Attitude mean NEES, first third of the run | 20.27 | 18.28 | 18.81 | 18.43 | 18.4 | | +| Attitude mean NEES, middle third of the run | 12.89 | 12.88 | 12.88 | 12.88 | 12.88 | | +| Attitude mean NEES, last third of the run | 9.183 | 9.205 | 9.216 | 9.199 | 9.199 | | +| Effective sample size of the attitude NEES series | 62.38 | 163.6 | 160.7 | 159 | 158.5 | count | +| That size is an upper bound (1 yes, 0 no) | 0 | 0 | 0 | 0 | 0 | flag | +| Pooled 2-sigma per-axis coverage, east and north (a Gaussian gives 0.9545) | 0.606 | 0.6079 | 0.6075 | 0.6078 | 0.6081 | | +| Share of epochs inside the horizontal 95% ellipse (expected 0.95) | 0.4459 | 0.4488 | 0.4512 | 0.4522 | 0.4538 | | +| Horizontal position RMSE over the recording | 4.354 | 4.348 | 4.351 | 4.352 | 4.352 | m | +| Vertical RMSE (the height is almost unused by design) | 11.44 | 10.19 | 10.56 | 10.25 | 10.4 | m | +| Fix horizontal difference from the reference, median | 2.361 | 2.361 | 2.361 | 2.361 | 2.361 | m | +| Fix horizontal difference, 95th percentile | 7.497 | 7.497 | 7.497 | 7.497 | 7.497 | m | +| Fix horizontal difference, 99th percentile | 12.97 | 12.97 | 12.97 | 12.97 | 12.97 | m | +| Fixes more than 3 of the receiver's own per-axis sigmas from the reference | 166 | 166 | 166 | 166 | 166 | count | +| Receiver's own reported per-axis sigma, median | 2.164 | 2.164 | 2.164 | 2.164 | 2.164 | m | +| Per-axis position sigma the filter claims, median over the run | 1.173 | 1.173 | 1.173 | 1.173 | 1.173 | m | +| Fixes alone: mean of (horizontal difference / receiver sigma) squared (expected 2) | 3.285 | 3.285 | 3.285 | 3.285 | 3.285 | | +| Fixes whose NIS is over the 0.1% chi-square level | 0 | 0 | 0 | 0 | 0 | count | +| Valid fixes over 10 m from the reference | 48 | 48 | 48 | 48 | 48 | count | +| Of those, over the 0.1% chi-square level | 0 | 0 | 0 | 0 | 0 | count | +| Receiver epochs the reader did not pass on (no fix or no position) | 0 | 0 | 0 | 0 | 0 | count | +| Fixes dropped because a reported variance was negative, NaN or infinite | 0 | 0 | 0 | 0 | 0 | count | +| Fixes with a reported sigma below the filter's floor | 0 | 0 | 0 | 0 | 0 | count | +| The start-bias stretch was judged at rest (1 yes, 0 no) | 0 | 0 | 0 | 0 | 0 | flag | @@ -314,7 +384,54 @@ No rows for Deep-Urban-1 are in the committed CSV, so it has not been run for th -No rows for Deep-Urban-1 are in the committed CSV, so it has not been run for this page. The table appears here when the CSV has its rows. +| Quantity | seed 0 | seed 1 | seed 2 | seed 3 | seed 4 | Unit | +|---|---:|---:|---:|---:|---:|---| +| Valid fixes offered to the filter | 1537 | 1537 | 1537 | 1537 | 1537 | count | +| Fixes the filter used | 1537 | 1537 | 1537 | 1537 | 1537 | count | +| Fixes the filter rejected | 0 | 0 | 0 | 0 | 0 | count | +| Share of the offered fixes the gate rejected | 0 | 0 | 0 | 0 | 0 | | +| The gate's alpha (the FDIR default) | 0.001 | 0.001 | 0.001 | 0.001 | 0.001 | | +| The gate's level on the NIS (3 degrees of freedom) | 16.27 | 16.27 | 16.27 | 16.27 | 16.27 | | +| Mean NIS of the offered fixes | 0.5249 | 0.5132 | 0.5206 | 0.5164 | 0.5152 | | +| Rejections expected at that alpha if every fix were consistent (offered x alpha) | 1.537 | 1.537 | 1.537 | 1.537 | 1.537 | count | +| Wilson 95% interval of the rejection rate on the raw count, lower end | 0 | 0 | 0 | 0 | 0 | | +| Wilson 95% interval of the rejection rate on the raw count, upper end | 0.002493 | 0.002493 | 0.002493 | 0.002493 | 0.002493 | | +| Effective number of fixes behind the rate (effective sample size of the over-the-level series) | 1537 | 1537 | 1537 | 1537 | 1537 | count | +| That size could be estimated (1 yes, 0 no: the series is constant, so it counts every fix) | 0 | 0 | 0 | 0 | 0 | flag | +| Wilson 95% interval on the effective count, lower end | 0 | 0 | 0 | 0 | 0 | | +| Wilson 95% interval on the effective count, upper end | 0.002493 | 0.002493 | 0.002493 | 0.002493 | 0.002493 | | +| GNSS channel declared faulty by the end | 0 | 0 | 0 | 0 | 0 | count | +| Times a faulty channel was taken back | 0 | 0 | 0 | 0 | 0 | count | +| Horizontal mean NEES over the whole run (2 degrees of freedom, expected 2) | 15.08 | 15.07 | 15.08 | 15.07 | 15.07 | | +| Horizontal mean NEES, first third of the run | 18.97 | 18.94 | 18.97 | 18.94 | 18.93 | | +| Horizontal mean NEES, middle third of the run | 20.51 | 20.51 | 20.5 | 20.51 | 20.51 | | +| Horizontal mean NEES, last third of the run | 5.772 | 5.774 | 5.775 | 5.774 | 5.774 | | +| Effective sample size of the horizontal NEES series (it has one value per IMU step) | 13.55 | 13.6 | 13.59 | 13.58 | 13.59 | count | +| That size is an upper bound (1 yes, 0 no) | 0 | 0 | 0 | 0 | 0 | flag | +| Attitude mean NEES over the whole run (3 degrees of freedom, expected 3) | 13.33 | 13.07 | 13.26 | 13.07 | 13.05 | | +| Attitude mean NEES, first third of the run | 18.9 | 18.03 | 18.54 | 18.04 | 18 | | +| Attitude mean NEES, middle third of the run | 11.64 | 11.71 | 11.73 | 11.69 | 11.69 | | +| Attitude mean NEES, last third of the run | 9.448 | 9.479 | 9.49 | 9.471 | 9.471 | | +| Effective sample size of the attitude NEES series | 178.6 | 195.6 | 188.2 | 195.8 | 195.9 | count | +| That size is an upper bound (1 yes, 0 no) | 0 | 0 | 0 | 0 | 0 | flag | +| Pooled 2-sigma per-axis coverage, east and north (a Gaussian gives 0.9545) | 0.5734 | 0.5719 | 0.5714 | 0.5714 | 0.5713 | | +| Share of epochs inside the horizontal 95% ellipse (expected 0.95) | 0.4242 | 0.4243 | 0.4256 | 0.4223 | 0.422 | | +| Horizontal position RMSE over the recording | 4.302 | 4.301 | 4.302 | 4.302 | 4.302 | m | +| Vertical RMSE (the height is almost unused by design) | 11.38 | 10.14 | 10.52 | 10.19 | 10.34 | m | +| Fix horizontal difference from the reference, median | 2.361 | 2.361 | 2.361 | 2.361 | 2.361 | m | +| Fix horizontal difference, 95th percentile | 7.497 | 7.497 | 7.497 | 7.497 | 7.497 | m | +| Fix horizontal difference, 99th percentile | 12.97 | 12.97 | 12.97 | 12.97 | 12.97 | m | +| Fixes more than 3 of the receiver's own per-axis sigmas from the reference | 166 | 166 | 166 | 166 | 166 | count | +| Receiver's own reported per-axis sigma, median | 2.164 | 2.164 | 2.164 | 2.164 | 2.164 | m | +| Per-axis position sigma the filter claims, median over the run | 1.137 | 1.137 | 1.137 | 1.137 | 1.137 | m | +| Fixes alone: mean of (horizontal difference / receiver sigma) squared (expected 2) | 3.285 | 3.285 | 3.285 | 3.285 | 3.285 | | +| Fixes whose NIS is over the 0.1% chi-square level | 0 | 0 | 0 | 0 | 0 | count | +| Valid fixes over 10 m from the reference | 48 | 48 | 48 | 48 | 48 | count | +| Of those, over the 0.1% chi-square level | 0 | 0 | 0 | 0 | 0 | count | +| Receiver epochs the reader did not pass on (no fix or no position) | 0 | 0 | 0 | 0 | 0 | count | +| Fixes dropped because a reported variance was negative, NaN or infinite | 0 | 0 | 0 | 0 | 0 | count | +| Fixes with a reported sigma below the filter's floor | 0 | 0 | 0 | 0 | 0 | count | +| The start-bias stretch was judged at rest (1 yes, 0 no) | 0 | 0 | 0 | 0 | 0 | flag | @@ -322,19 +439,123 @@ No rows for Deep-Urban-1 are in the committed CSV, so it has not been run for th -No rows for Deep-Urban-1 are in the committed CSV, so it has not been run for this page. The table appears here when the CSV has its rows. +Not applicable: the receiver reports no outage on this sequence by the outage rule, and an oracle diagnostic acts only inside an outage. The oracle variants are not run. -What this section will not show, whatever the rows say: +The effective count behind the rejection rate has one convention. When no fix is over the gate's level the series of +yes and no answers is constant and has no autocorrelation to estimate, so it counts every fix as an independent +sample. The effective interval is then the raw one, and the row "That size could be estimated" is the flag for no. + +**The receiver's fix errors in time.** This table has no filter in it. It is the receiver's fixes against the +reference, for both sequences, with the mean offset taken out. The correlation at a lag is over the pairs of fixes +that far apart, and the effective sample size is per axis. Tunnel-1 has far fewer fixes, and its fixes are joined +across the outage, so its numbers are rougher. + + + +| Sequence | Quantity | Value | Unit | +|---|---|---:|---| +| tunnel1 | Valid fixes | 250 | count | +| tunnel1 | Pairs for a lag: time difference within this of the lag | 0.5 | s | +| tunnel1 | Mean offset of the fixes from the reference, east | -1.234 | m | +| tunnel1 | Mean offset of the fixes from the reference, north | 1.95 | m | +| tunnel1 | Horizontal root mean square of the difference about that offset | 10.87 | m | +| tunnel1 | Pairs of fixes 1 s apart | 248 | count | +| tunnel1 | Correlation of the fix difference 1 s apart, east | 0.9405 | | +| tunnel1 | Correlation of the fix difference 1 s apart, north | 0.9908 | | +| tunnel1 | Correlation of the fix difference 1 s apart, pooled | 0.9704 | | +| tunnel1 | Pairs of fixes 10 s apart | 230 | count | +| tunnel1 | Correlation of the fix difference 10 s apart, east | 0.07743 | | +| tunnel1 | Correlation of the fix difference 10 s apart, north | 0.3562 | | +| tunnel1 | Correlation of the fix difference 10 s apart, pooled | 0.2331 | | +| tunnel1 | Pairs of fixes 30 s apart | 190 | count | +| tunnel1 | Correlation of the fix difference 30 s apart, east | -0.05684 | | +| tunnel1 | Correlation of the fix difference 30 s apart, north | 0.03377 | | +| tunnel1 | Correlation of the fix difference 30 s apart, pooled | -0.00567 | | +| tunnel1 | Pairs of fixes 60 s apart | 130 | count | +| tunnel1 | Correlation of the fix difference 60 s apart, east | -0.07971 | | +| tunnel1 | Correlation of the fix difference 60 s apart, north | -0.1313 | | +| tunnel1 | Correlation of the fix difference 60 s apart, pooled | -0.1038 | | +| tunnel1 | Effective sample size of the fix difference, east | 66.8 | count | +| tunnel1 | That size is an upper bound, east (1 yes, 0 no) | 0 | flag | +| tunnel1 | Effective sample size of the fix difference, north | 18.33 | count | +| tunnel1 | That size is an upper bound, north (1 yes, 0 no) | 0 | flag | +| deepurban1 | Valid fixes | 1539 | count | +| deepurban1 | Pairs for a lag: time difference within this of the lag | 0.5 | s | +| deepurban1 | Mean offset of the fixes from the reference, east | 0.9633 | m | +| deepurban1 | Mean offset of the fixes from the reference, north | -1.233 | m | +| deepurban1 | Horizontal root mean square of the difference about that offset | 3.508 | m | +| deepurban1 | Pairs of fixes 1 s apart | 1538 | count | +| deepurban1 | Correlation of the fix difference 1 s apart, east | 0.9983 | | +| deepurban1 | Correlation of the fix difference 1 s apart, north | 0.9985 | | +| deepurban1 | Correlation of the fix difference 1 s apart, pooled | 0.9984 | | +| deepurban1 | Pairs of fixes 10 s apart | 1529 | count | +| deepurban1 | Correlation of the fix difference 10 s apart, east | 0.9455 | | +| deepurban1 | Correlation of the fix difference 10 s apart, north | 0.95 | | +| deepurban1 | Correlation of the fix difference 10 s apart, pooled | 0.9477 | | +| deepurban1 | Pairs of fixes 30 s apart | 1509 | count | +| deepurban1 | Correlation of the fix difference 30 s apart, east | 0.7442 | | +| deepurban1 | Correlation of the fix difference 30 s apart, north | 0.777 | | +| deepurban1 | Correlation of the fix difference 30 s apart, pooled | 0.7599 | | +| deepurban1 | Pairs of fixes 60 s apart | 1479 | count | +| deepurban1 | Correlation of the fix difference 60 s apart, east | 0.4217 | | +| deepurban1 | Correlation of the fix difference 60 s apart, north | 0.5265 | | +| deepurban1 | Correlation of the fix difference 60 s apart, pooled | 0.4719 | | +| deepurban1 | Effective sample size of the fix difference, east | 16.46 | count | +| deepurban1 | That size is an upper bound, east (1 yes, 0 no) | 0 | flag | +| deepurban1 | Effective sample size of the fix difference, north | 17.87 | count | +| deepurban1 | That size is an upper bound, north (1 yes, 0 no) | 0 | flag | + + + +### What the section shows -- It does not say how the filter does through an outage. Imposed outages are a separate experiment. +Every statement is about one sequence and one receiver, and points at rows of the generated tables above. + +1. **The gate rejects none of the offered fixes.** Some fixes are more than three of the receiver's own sigmas from + the reference, and some are more than ten metres from it (the two rows of counts of far fixes). The mean NIS stays far below the gate's level (rows "Mean NIS" and "The gate's + level"). The rejection rate is zero, and the gate's alpha lies inside its Wilson interval, so the rows do not + show the gate failing its own level. They show that it rejects none of these fixes. The channel is never declared + faulty. +2. **The filter is overconfident against this reference, in position and in attitude.** The horizontal and the + attitude normalised errors are far above their expected values, and the pooled 2-sigma coverage is far below the + Gaussian value. The per-axis sigma the filter claims is below the receiver's own median sigma. This holds in both + variants and on every seed. +3. **The excess is not a start-up transient.** The horizontal and the attitude normalised errors are above their + expected values in each third of the run, in both variants and on every seed. This does not say what causes it, + and the first, middle and last thirds differ from each other. +4. **The means have few independent samples behind them.** The normalised-error series has one value per IMU step, and + its effective sample size is a small fraction of its length (rows "Effective sample size"). The means say the + filter is overconfident. They do not give a precise value. +5. **The receiver's own sigma is optimistic against the reference.** The mean of the fixes' squared difference over + their reported sigma is above its expected value. This is a statement about the receiver and the reference. The + reference's own error is not known. +6. **Each variant sees the same fixes.** The fix statistics are identical in the two variants, because they are + statistics of the fixes and not of the filter. +7. **The seeds agree**, because the start perturbation is small against the effect. They are not independent tests. + +**INTERPRETATION, not a result of this page.** The filter treats the receiver's fixes as independent measurements. +The table of fix errors above shows that on this sequence the receiver's errors are correlated over tens of seconds: +the correlation is high at the longer lags, and the effective sample size per axis is a small fraction of the number +of fixes. Averaging correlated errors does not reduce the error the way the claimed covariance assumes, and the +innovations stay small because the estimate follows the fixes. This is consistent with the rows above. It does not +prove the cause, because the filter was not run with a correlated-error model here. The reference's own error is not +known, and some of the correlated difference could be in the reference and not in the receiver. + +### What the section does not show + +- It does not say how the filter does through an outage. Imposed outages are a separate experiment, and this section + did not run one. - It does not carry over to another receiver, another city or another day. -- The antenna offset from the IMU is not corrected, and the height is almost unused by design. +- It does not say whether a correlated-error model would repair the overconfidence. That was not run. +- It makes no claim about the height, and the antenna offset from the IMU is not corrected. +- It does not say how accurate the receiver is. The reference is a post-processed estimate with several quality codes, + and its error is not known. - It is not field performance. ## What follows The attitude is the next thing to understand: a run that starts from a better alignment, or that holds the gyroscope -bias, would show how much of this is the start and how much is the sensor. The planned unseen check on Deep-Urban-1 -needs its NMEA file. See [ADR-0026](adr/0026-urbannav-as-a-real-gnss-dataset.md). +bias, would show how much of this is the start and how much is the sensor. A filter model for correlated receiver +errors, which this page does not run, would be the next thing to try on Deep-Urban-1. See [ADR-0026](adr/0026-urbannav-as-a-real-gnss-dataset.md). diff --git a/src/navkit/cli.py b/src/navkit/cli.py index 855a8e3..1f24530 100644 --- a/src/navkit/cli.py +++ b/src/navkit/cli.py @@ -50,6 +50,7 @@ separability, slow_ramp, urbannav_check, + urbannav_fix_errors, urbannav_run, vision_ramp, ) @@ -130,15 +131,21 @@ def _gate(argv: list[str] | None) -> int: def _urbannav(argv: list[str] | None) -> int: - """Route ``navkit urbannav fetch`` and ``navkit urbannav check``.""" + """Route ``navkit urbannav fetch``, ``check``, ``run`` and ``fixerrors``.""" args = list(argv or []) - routes = {"fetch": urbannav_fetch.main, "check": urbannav_check.main, "run": urbannav_run.main} + routes = { + "fetch": urbannav_fetch.main, + "check": urbannav_check.main, + "run": urbannav_run.main, + "fixerrors": urbannav_fix_errors.main, + } if not args or args[0] in ("-h", "--help") or args[0] not in routes: - usage = "usage: navkit urbannav {fetch,check,run} [options]" + usage = "usage: navkit urbannav {fetch,check,run,fixerrors} [options]" notes = ( " fetch download the reference solution and IMU of UrbanNav sequences (MIT, cite Hsu et al. 2023)", " check measure what the fetched files contain and fill the UrbanNav data page", " run run the filter on a fetched sequence with the receiver's own fixes and outage", + " fixerrors measure how the receiver's fix errors behave in time against the reference", ) print("\n".join((usage, *notes)), file=sys.stderr) return 0 if args and args[0] in ("-h", "--help") else 2 @@ -157,7 +164,7 @@ def _urbannav(argv: list[str] | None) -> int: "tumvi": (_tumvi, "fetch TUM VI room sequences (CC BY 4.0); run them with `navkit euroc run --dataset tumvi`"), "urbannav": ( _urbannav, - "fetch UrbanNav reference and IMU files (MIT); real GNSS outages, not yet run by the filter", + "fetch UrbanNav reference and IMU files (MIT) and run the filter on the receiver's own fixes", ), "euroc": (euroc_cli.main, "fetch EuRoC sequences and run the filter on them (recorded IMU, simulated GNSS)"), } diff --git a/src/navkit/urbannav_fix_errors.py b/src/navkit/urbannav_fix_errors.py new file mode 100644 index 0000000..af4d5d4 --- /dev/null +++ b/src/navkit/urbannav_fix_errors.py @@ -0,0 +1,201 @@ +"""How the receiver's own fix errors behave in time, against the reference, written to a CSV the page is built from. + + navkit urbannav fixerrors --sequence tunnel1 --csv docs/data/urbannav_fix_errors.csv --page docs/urbannav.md + navkit urbannav fixerrors --sequence deepurban1 --csv docs/data/urbannav_fix_errors.csv --page docs/urbannav.md + +The question is whether the receiver's fix errors are independent from one second to the next, as a filter that takes +each fix as a separate measurement assumes. No filter is involved: these are the fixes and the reference only. + +* **The error** of a fix is its east and north position minus the reference interpolated to the fix time. The mean over + the sequence (the offset) is removed first, so what is measured is the change of the error and not a constant bias. +* **The correlation at a lag** is the sum of products of the centred errors over all pairs of fixes whose time + difference is within :data:`LAG_TOLERANCE_S` of the lag, divided by the square root of the two sums of squares of + the same pairs. It is given per axis and pooled over both. It is the usual sample correlation, so a few large errors + dominate it in a short sequence. +* **The effective sample size** per axis is ``navkit.eval.statistics.autocorrelation_estimate`` on the centred series + in time order, one value per valid fix. A sequence with an outage has its fixes joined across the gap, which is not + a regular series, and the number is then rough. The ``truncated`` flag says whether the window rule was met. + +The reference is an estimate whose own error is not known, so some of a correlated difference could be in the +reference and not in the receiver. These are differences between two solutions, as in ``urbannav_check``. +""" + +from __future__ import annotations + +import argparse +import csv +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import numpy as np + +from . import urbannav_data as ud +from .eval.statistics import autocorrelation_estimate +from .recorded import SequenceError + +CSV_COLUMNS = ("sequence", "quantity", "value", "unit") +DEFAULT_ROOT = "data/raw/urbannav" +LAGS_S = (1, 10, 30, 60) +#: Pairs of fixes count for a lag when their time difference is within this many seconds of it (the epochs are 1 Hz). +LAG_TOLERANCE_S = 0.5 +AXES = ("east", "north") + + +def fix_errors(root: str | Path, seq_name: str) -> tuple[np.ndarray, np.ndarray]: + """``(t, error)``: the valid fixes' times on the IMU clock and their east and north error, shape ``(n, 2)``.""" + seq = ud.load_sequence(root, seq_name) + fixes = ud.load_gga(root, seq_name) + valid = (fixes.quality > 0) & np.isfinite(fixes.lat_deg) & np.isfinite(fixes.lon_deg) + enu, _ = ud.llh_to_enu(fixes.lat_deg[valid], fixes.lon_deg[valid], fixes.height_m[valid], origin=seq.origin_llh) + t = fixes.utc_s[valid] - seq.epoch_unix_s + ref = np.column_stack([np.interp(t, seq.truth_t, seq.truth_enu[:, k]) for k in range(2)]) + return t, enu[:, :2] - ref + + +def lag_pairs(t: np.ndarray, lag_s: float, tolerance_s: float = LAG_TOLERANCE_S) -> tuple[np.ndarray, np.ndarray]: + """Indices ``(i, j)`` of the pairs whose time difference ``t[j] - t[i]`` is within ``tolerance_s`` of ``lag_s``.""" + dt = t[None, :] - t[:, None] + i, j = np.nonzero(np.abs(dt - lag_s) < tolerance_s) + return i, j + + +def pair_correlation(centred: np.ndarray, i: np.ndarray, j: np.ndarray) -> float: + """Sample correlation over the pairs ``(i, j)`` of a centred series (one axis, or both axes pooled).""" + a, b = centred[i], centred[j] + denom = float(np.sqrt(np.sum(a**2) * np.sum(b**2))) + return float(np.sum(a * b) / denom) if denom > 0.0 else float("nan") + + +def rows_for(seq_name: str, t: np.ndarray, err: np.ndarray) -> list[dict[str, Any]]: + """The quantities of one sequence, as CSV rows, from the fix times and their ``(n, 2)`` east and north errors.""" + offset = err.mean(axis=0) + centred = err - offset + rows: list[dict[str, Any]] = [] + + def add(quantity: str, value: float | int, unit: str) -> None: + shown = value if isinstance(value, int) else f"{value:.4g}" + rows.append({"sequence": seq_name, "quantity": quantity, "value": shown, "unit": unit}) + + add("fix_error_fixes", int(len(t)), "count") + add("fix_error_lag_tolerance", LAG_TOLERANCE_S, "s") + add("fix_error_offset_east", float(offset[0]), "m") + add("fix_error_offset_north", float(offset[1]), "m") + add("fix_error_rms_about_offset", float(np.sqrt(np.mean(np.sum(centred**2, axis=1)))), "m") + for lag in LAGS_S: + i, j = lag_pairs(t, lag) + add(f"fix_error_pairs_lag{lag}s", int(len(i)), "count") + if len(i) >= 2: + for k, axis in enumerate(AXES): + add(f"fix_error_corr_{axis}_lag{lag}s", pair_correlation(centred[:, k], i, j), "") + add(f"fix_error_corr_pooled_lag{lag}s", pair_correlation(centred, i, j), "") + for k, axis in enumerate(AXES): + est = autocorrelation_estimate(centred[:, k]) + add(f"fix_error_ess_{axis}", est.effective_sample_size, "count") + add(f"fix_error_ess_{axis}_truncated", int(est.truncated), "flag") + return rows + + +def measure(root: str | Path, seq_name: str) -> list[dict[str, Any]]: + t, err = fix_errors(root, seq_name) + try: + return rows_for(seq_name, t, err) + except ValueError as exc: # too few fixes, or a constant error series + raise SequenceError(f"{seq_name}: {exc}") from exc + + +# ---------------------------------------------------------------------- output + +LABELS = { + "fix_error_fixes": "Valid fixes", + "fix_error_lag_tolerance": "Pairs for a lag: time difference within this of the lag", + "fix_error_offset_east": "Mean offset of the fixes from the reference, east", + "fix_error_offset_north": "Mean offset of the fixes from the reference, north", + "fix_error_rms_about_offset": "Horizontal root mean square of the difference about that offset", + **{f"fix_error_pairs_lag{n}s": f"Pairs of fixes {n} s apart" for n in LAGS_S}, + **{ + f"fix_error_corr_{a}_lag{n}s": f"Correlation of the fix difference {n} s apart, {a}" + for n in LAGS_S + for a in (*AXES, "pooled") + }, + **{f"fix_error_ess_{a}": f"Effective sample size of the fix difference, {a}" for a in AXES}, + **{f"fix_error_ess_{a}_truncated": f"That size is an upper bound, {a} (1 yes, 0 no)" for a in AXES}, +} + + +def table_markdown(rows: list[dict[str, Any]]) -> str: + lines = ["| Sequence | Quantity | Value | Unit |", "|---|---|---:|---|"] + for r in rows: + lines.append(f"| {r['sequence']} | {LABELS.get(r['quantity'], r['quantity'])} | {r['value']} | {r['unit']} |") + return "\n".join(lines) + + +BLOCKS: dict[str, Callable[[list[dict[str, Any]]], str]] = {"urbannav-fix-errors": table_markdown} + + +def write_csv(path: str | Path, rows: list[dict[str, Any]]) -> None: + """Write ``rows`` to ``path``, replacing the rows of those sequences and keeping the rows of any other.""" + ud.write_merged_csv(path, CSV_COLUMNS, rows) + + +def rows_from_csv(path: str | Path) -> list[dict[str, Any]]: + with Path(path).open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def write_block(path: str | Path, name: str, block: str) -> None: + page = Path(path) + text = page.read_text(encoding="utf-8") + start, end = f"", f"" + if start not in text or end not in text or text.index(start) > text.index(end): + raise ValueError(f"{page} needs the lines {start} and {end}, in that order") + head, rest = text.split(start, 1) + _, tail = rest.split(end, 1) + page.write_text(f"{head}{start}\n\n{block}\n\n{end}{tail}", encoding="utf-8") + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="navkit urbannav fixerrors", description="Measure how the receiver's fix errors behave in time." + ) + p.add_argument("--root", default=DEFAULT_ROOT) + p.add_argument("--sequence", "-s", default="tunnel1") + p.add_argument("--csv", default=None, help="write the quantities here; other sequences' rows in the file are kept") + p.add_argument("--from-csv", default=None, help="rebuild the table from a saved CSV; reads no dataset files") + p.add_argument("--page", default=None, help="fill the urbannav-fix-errors block (needs --csv or --from-csv)") + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.page and not (args.csv or args.from_csv): + print("error: --page needs --csv or --from-csv, so that the block is filled from one file", file=sys.stderr) + return 2 + try: + rows = rows_from_csv(args.from_csv) if args.from_csv else measure(args.root, args.sequence) + except SequenceError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if args.csv: + try: + write_csv(args.csv, rows) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(f"wrote {args.csv}", file=sys.stderr) + if args.page: + page_rows = rows_from_csv(args.csv or args.from_csv) + try: + for name, fn in BLOCKS.items(): + write_block(args.page, name, fn(page_rows)) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if not (args.csv or args.page): + print(table_markdown(rows)) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/src/navkit/urbannav_run.py b/src/navkit/urbannav_run.py index 1e756e5..8acfe92 100644 --- a/src/navkit/urbannav_run.py +++ b/src/navkit/urbannav_run.py @@ -360,6 +360,10 @@ def aided_summary( horizontal difference, and the mean of (horizontal difference / receiver sigma) squared, which is 2 when the receiver's sigma is right and the errors are Gaussian. * ``fix_rejection_*``, with ``nis`` and ``gate_level``: the Wilson intervals of :func:`rejection_intervals`. + * ``fixes_over_3sigma``, ``receiver_sigma_median_m``, ``claimed_sigma_median_m``: the fixes more than three of the + receiver's own per-axis sigmas from the reference, the receiver's median per-axis sigma, and the median per-axis + sigma the filter claims for its position. ``nis_mean`` and ``fix_gate_level`` come with ``nis``. + * ``*_nees_third1..3``: the mean of each NEES series over the first, middle and last third of the run. * ``*_nees_ess``, with ``nees_h`` for the horizontal one: the effective sample size of each NEES series, and whether it reached the largest window (then it is an upper bound, ADR-0029). The means are over every IMU step, so they are far from independent samples, and the effective size is what says how many there are. @@ -374,11 +378,18 @@ def aided_summary( "fix_rejection_rate": float(stats.get("gnss_updates_rejected", 0)) / offered if offered else math.nan, "fix_nees_mean": float(np.mean((fix_difference_m / fix_sigma_h_m) ** 2)), } + out["fixes_over_3sigma"] = int(np.sum(fix_difference_m > 3.0 * fix_sigma_h_m)) + out["receiver_sigma_median_m"] = float(np.median(fix_sigma_h_m)) + out["claimed_sigma_median_m"] = float(np.median(np.sqrt(0.5 * (cov[:, 0, 0] + cov[:, 1, 1])))) if nis is not None and gate_level is not None: + out["nis_mean"] = float(np.mean(nis)) + out["fix_gate_level"] = float(gate_level) out.update(rejection_intervals(int(stats.get("gnss_updates_rejected", 0)), offered, nis, gate_level)) if nees_h is not None: out["hor_nees_ess"], truncated = effective_samples(nees_h) out["hor_nees_ess_truncated"] = int(truncated) + for k, part in enumerate(np.array_split(nees_h, 3), start=1): + out[f"hor_nees_third{k}"] = float(np.mean(part)) rot_cov = detail.get("rotation_cov") if rot_cov is not None: dtheta = rot_log_batch(np.einsum("nji,njk->nik", est.rotations, ref.rotations)) @@ -386,6 +397,8 @@ def aided_summary( out["att_nees_mean"] = float(np.mean(nees_att)) out["att_nees_ess"], truncated = effective_samples(nees_att) out["att_nees_ess_truncated"] = int(truncated) + for k, part in enumerate(np.array_split(nees_att, 3), start=1): + out[f"att_nees_third{k}"] = float(np.mean(part)) for q in FIX_PERCENTILES: out[f"fix_err_p{q}_m"] = float(np.percentile(fix_difference_m, q)) return out @@ -551,6 +564,8 @@ def run_once( ("gnss_updates_rejected", "Fixes the filter rejected", "count"), ("fix_rejection_rate", "Share of the offered fixes the gate rejected", ""), ("fix_gate_alpha", "The gate's alpha (the FDIR default)", ""), + ("fix_gate_level", "The gate's level on the NIS (3 degrees of freedom)", ""), + ("nis_mean", "Mean NIS of the offered fixes", ""), ( "fix_rejections_expected", "Rejections expected at that alpha if every fix were consistent (offered x alpha)", @@ -573,9 +588,15 @@ def run_once( ("gnss_channel_faulted", "GNSS channel declared faulty by the end", "count"), ("gnss_reaccepted", "Times a faulty channel was taken back", "count"), ("hor_nees_mean", "Horizontal mean NEES over the whole run (2 degrees of freedom, expected 2)", ""), + ("hor_nees_third1", "Horizontal mean NEES, first third of the run", ""), + ("hor_nees_third2", "Horizontal mean NEES, middle third of the run", ""), + ("hor_nees_third3", "Horizontal mean NEES, last third of the run", ""), ("hor_nees_ess", "Effective sample size of the horizontal NEES series (it has one value per IMU step)", "count"), ("hor_nees_ess_truncated", "That size is an upper bound (1 yes, 0 no)", "flag"), ("att_nees_mean", "Attitude mean NEES over the whole run (3 degrees of freedom, expected 3)", ""), + ("att_nees_third1", "Attitude mean NEES, first third of the run", ""), + ("att_nees_third2", "Attitude mean NEES, middle third of the run", ""), + ("att_nees_third3", "Attitude mean NEES, last third of the run", ""), ("att_nees_ess", "Effective sample size of the attitude NEES series", "count"), ("att_nees_ess_truncated", "That size is an upper bound (1 yes, 0 no)", "flag"), ( @@ -590,6 +611,9 @@ def run_once( ("fix_err_p50_m", "Fix horizontal difference from the reference, median", "m"), ("fix_err_p95_m", "Fix horizontal difference, 95th percentile", "m"), ("fix_err_p99_m", "Fix horizontal difference, 99th percentile", "m"), + ("fixes_over_3sigma", "Fixes more than 3 of the receiver's own per-axis sigmas from the reference", "count"), + ("receiver_sigma_median_m", "Receiver's own reported per-axis sigma, median", "m"), + ("claimed_sigma_median_m", "Per-axis position sigma the filter claims, median over the run", "m"), ( "fix_nees_mean", "Fixes alone: mean of (horizontal difference / receiver sigma) squared (expected 2)", @@ -605,6 +629,9 @@ def run_once( ) #: Keys only the aided run has. A sequence with rows for any of them is an aided sequence. AIDED_ONLY_KEYS = frozenset(k for k, _, _ in AIDED_QUANTITIES) - frozenset(k for k, _, _ in QUANTITIES) +#: Rows of the aided run that are in the CSV and not in the page tables: the absolute trajectory error is a +#: three-dimensional number that the unused height dominates, so the page quotes the horizontal RMSE. +NOT_ON_PAGE = frozenset({"ate_rmse_m"}) CSV_COLUMNS = ("sequence", "variant", "seed", "quantity", "value", "unit") @@ -666,6 +693,8 @@ def run_markdown(rows: list[dict[str, Any]], variant: str = "declared", sequence head = "| Quantity | " + " | ".join(f"seed {s}" for s in seeds) + " | Unit |" lines = [head, "|---|" + "---:|" * len(seeds) + "---|"] for key, label, unit in quantities: + if key in NOT_ON_PAGE and quantities is AIDED_QUANTITIES: + continue if any((s, key) in by for s in seeds): lines.append(f"| {label} | " + " | ".join(str(by.get((s, key), "")) for s in seeds) + f" | {unit} |") return "\n".join(lines) diff --git a/tests/test_urbannav_check.py b/tests/test_urbannav_check.py index 1457dcf..0583c4d 100644 --- a/tests/test_urbannav_check.py +++ b/tests/test_urbannav_check.py @@ -137,13 +137,29 @@ def _stub(sequence): def test_writing_another_sequence_keeps_the_committed_rows_byte_for_byte(tmp_path): copy = tmp_path / "copy.csv" copy.write_bytes(CSV.read_bytes()) - uc.write_csv(copy, _stub("deepurban1")) + uc.write_csv(copy, _stub("stubseq")) # a sequence the file does not hold: added at the end assert copy.read_bytes().startswith(CSV.read_bytes()) - uc.write_csv(copy, _stub("deepurban1")) # again: replaced, not added - assert [r["sequence"] for r in uc.rows_from_csv(copy)].count("deepurban1") == 1 + uc.write_csv(copy, _stub("stubseq")) # again: replaced, not added + assert [r["sequence"] for r in uc.rows_from_csv(copy)].count("stubseq") == len(_stub("stubseq")) assert copy.read_bytes().startswith(CSV.read_bytes()) +def _lines_of(path, sequence): + return [x for x in path.read_bytes().splitlines(keepends=True) if x.startswith(f"{sequence},".encode())] + + +def test_rewriting_one_sequence_leaves_the_other_sequences_lines_byte_identical_and_in_place(tmp_path): + copy = tmp_path / "copy.csv" + copy.write_bytes(CSV.read_bytes()) + tunnel, deep = _lines_of(CSV, "tunnel1"), _lines_of(CSV, "deepurban1") + assert tunnel and deep # both sequences are in the committed file + uc.write_csv(copy, _stub("deepurban1")) + assert _lines_of(copy, "tunnel1") == tunnel + assert _lines_of(copy, "deepurban1") != deep # the rewritten sequence did change + uc.write_csv(copy, uc.rows_from_csv(CSV)) # writing back what the file holds reproduces the committed bytes + assert copy.read_bytes() == CSV.read_bytes() + + def test_a_csv_with_another_header_is_refused_and_left_alone(tmp_path): path = tmp_path / "x.csv" path.write_text("a,b\n1,2\n", encoding="utf-8") diff --git a/tests/test_urbannav_data.py b/tests/test_urbannav_data.py index 1cf6072..ae95251 100644 --- a/tests/test_urbannav_data.py +++ b/tests/test_urbannav_data.py @@ -458,6 +458,56 @@ def test_the_real_tunnel_is_one_outage_of_minutes_and_not_at_the_edges(real_fixe assert real.reference.utc_s[0] + 60.0 < start and end < real.reference.utc_s[-1] - 30.0 +# ----------------------------------------------- the real Deep-Urban-1 files (skipped without the data) + +DEEP = "deepurban1" + + +@pytest.fixture(scope="module") +def deep_fixes(): + if not (FETCHED / DEEP / ud.NMEA_FILES[DEEP]).is_file(): + pytest.skip("the deepurban1 NMEA file is not in data/raw/urbannav/deepurban1/") + return ud.load_gga(FETCHED, DEEP) + + +@pytest.fixture(scope="module") +def deep_real(): + if not all((FETCHED / DEEP / name).is_file() for name in uf.SOURCES[DEEP]): + pytest.skip("deepurban1 is not fetched (navkit urbannav fetch --sequence deepurban1)") + return ud.load_sequence(FETCHED, DEEP) + + +def test_the_real_deep_urban_receiver_has_a_valid_fix_at_every_epoch_and_the_rule_finds_no_outage(deep_fixes): + assert deep_fixes.skipped == 0 and len(deep_fixes) > 1000 + assert np.all(deep_fixes.quality == 2) and np.all(deep_fixes.num_sats == 12) + assert np.all(np.isfinite(deep_fixes.lat_deg)) and ud.outage_intervals(deep_fixes) == [] + + +def test_the_real_deep_urban_receiver_epochs_sit_on_the_reference_seconds(deep_fixes, deep_real): + assert len(deep_fixes) == len(deep_real.reference) + assert np.all(np.isin(deep_fixes.utc_s, deep_real.reference.utc_s)) + assert np.allclose(np.diff(deep_real.reference.utc_s), 1.0) + + +def test_the_real_deep_urban_imu_runs_at_about_400_hz_and_reads_gravity_on_z(deep_real): + assert 350.0 < 1.0 / np.median(np.diff(deep_real.imu.t)) < 450.0 + first = deep_real.imu.accel[deep_real.imu.t < 3.0].mean(axis=0) + assert 9.5 < first[2] < 10.1 and abs(first[0]) < 0.5 and abs(first[1]) < 0.5 + + +def test_the_real_deep_urban_attitude_convention_matches_the_positions_and_gravity(deep_real): + ref = deep_real.reference + r = ud.rpy_to_rotation(ref.rpy_deg[:, 0], ref.rpy_deg[:, 1], ref.rpy_deg[:, 2]) + v = np.einsum("nij,nj->ni", r, ref.vel_body)[1:-1] + e, t = deep_real.truth_enu, deep_real.truth_t + fd = (e[2:] - e[:-2]) / (t[2:] - t[:-2])[:, None] + moving = np.linalg.norm(fd[:, :2], axis=1) > 3.0 + assert moving.sum() > 100 and np.median(np.linalg.norm((v - fd)[moving], axis=1)) < 0.2 + k = int(np.argmin(np.abs(t - 1.5))) + up = r[k] @ deep_real.imu.accel[deep_real.imu.t < 3.0].mean(axis=0) + assert up[2] > 9.3 and abs(up[0]) < 0.6 and abs(up[1]) < 0.6 + + # ------------------------------------------------------------ GST and attitude diff --git a/tests/test_urbannav_fix_errors.py b/tests/test_urbannav_fix_errors.py new file mode 100644 index 0000000..f261e55 --- /dev/null +++ b/tests/test_urbannav_fix_errors.py @@ -0,0 +1,92 @@ +"""The fix-error aggregate: how the receiver's own fix errors behave in time (``navkit urbannav fixerrors``). + +The functions are checked against series whose answer is known. No dataset is read here. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from navkit import urbannav_fix_errors as fe + + +def _t(n: int) -> np.ndarray: + return np.arange(n, dtype=float) + + +def test_the_pairs_of_a_lag_are_the_fixes_that_far_apart_within_the_tolerance(): + t = _t(100) + i, j = fe.lag_pairs(t, 10) + assert len(i) == 90 and np.all(j - i == 10) + odd = np.arange(100) % 2 + assert len(fe.lag_pairs(t + 0.3 * odd, 11)[0]) == 89 # 10.7 and 11.3 s apart: within 0.5 of 11 + assert len(fe.lag_pairs(t + 0.7 * odd, 11)[0]) == 0 # 10.3 and 11.7 s apart: out + + +def test_a_gap_in_the_times_removes_the_pairs_across_it(): + t = np.concatenate([_t(50), _t(50) + 100.0]) + i, j = fe.lag_pairs(t, 10) + assert len(i) == 80 and np.all(t[j] - t[i] == 10.0) + + +def test_a_sine_of_period_twenty_seconds_is_anticorrelated_at_ten_and_correlated_at_twenty(): + t = _t(400) + x = np.sin(2 * np.pi * t / 20.0) + assert fe.pair_correlation(x, *fe.lag_pairs(t, 10)) == pytest.approx(-1.0, abs=1e-9) + assert fe.pair_correlation(x, *fe.lag_pairs(t, 20)) == pytest.approx(1.0, abs=1e-9) + + +def test_white_noise_has_a_small_correlation_at_every_lag_and_a_large_effective_sample_size(): + t = _t(1500) + err = np.random.default_rng(1).normal(size=(1500, 2)) + rows = {r["quantity"]: float(r["value"]) for r in fe.rows_for("x", t, err)} + for lag in fe.LAGS_S: + assert abs(rows[f"fix_error_corr_pooled_lag{lag}s"]) < 0.1 + assert rows["fix_error_ess_east"] > 500 and rows["fix_error_ess_north"] > 500 + + +def test_a_slowly_wandering_error_is_correlated_over_tens_of_seconds_and_has_few_effective_samples(): + t = _t(1500) + rng = np.random.default_rng(2) + walk = np.zeros(1500) + for k in range(1, 1500): # AR(1) with a correlation time of about 50 s + walk[k] = 0.98 * walk[k - 1] + rng.normal() + err = np.column_stack([walk, np.roll(walk, 7)]) + rows = {r["quantity"]: float(r["value"]) for r in fe.rows_for("x", t, err)} + assert rows["fix_error_corr_pooled_lag1s"] > 0.95 and rows["fix_error_corr_pooled_lag10s"] > 0.6 + assert rows["fix_error_corr_pooled_lag10s"] > rows["fix_error_corr_pooled_lag60s"] + assert rows["fix_error_ess_east"] < 100 + + +def test_a_constant_offset_is_reported_and_does_not_change_the_correlation(): + t = _t(300) + rng = np.random.default_rng(3) + base = np.cumsum(rng.normal(size=(300, 2)), axis=0) + a = {r["quantity"]: r["value"] for r in fe.rows_for("x", t, base)} + b = {r["quantity"]: r["value"] for r in fe.rows_for("x", t, base + np.array([5.0, -3.0]))} + assert float(b["fix_error_offset_east"]) == pytest.approx(float(a["fix_error_offset_east"]) + 5.0, abs=1e-2) + assert float(b["fix_error_offset_north"]) == pytest.approx(float(a["fix_error_offset_north"]) - 3.0, abs=1e-2) + for key in a: + if key.startswith(("fix_error_corr", "fix_error_rms")): + assert a[key] == b[key], key + + +def test_the_rms_about_the_offset_is_the_horizontal_root_mean_square_of_the_centred_error(): + t = _t(100) + err = np.column_stack([np.where(np.arange(100) % 2, 3.0, -3.0), np.where(np.arange(100) % 2, 4.0, -4.0)]) + rows = {r["quantity"]: float(r["value"]) for r in fe.rows_for("x", t, err)} + assert rows["fix_error_rms_about_offset"] == pytest.approx(5.0) + assert rows["fix_error_offset_east"] == pytest.approx(0.0, abs=1e-9) + + +def test_a_lag_longer_than_the_sequence_has_no_pairs_and_no_correlation_row(): + rows = {r["quantity"] for r in fe.rows_for("x", _t(40), np.random.default_rng(4).normal(size=(40, 2)))} + assert "fix_error_pairs_lag30s" in rows and "fix_error_corr_pooled_lag30s" in rows + assert "fix_error_pairs_lag60s" in rows and "fix_error_corr_pooled_lag60s" not in rows + + +def test_every_quantity_has_a_label_and_the_rows_have_the_documented_columns(): + rows = fe.rows_for("x", _t(200), np.random.default_rng(5).normal(size=(200, 2))) + assert all(set(r) == set(fe.CSV_COLUMNS) for r in rows) + assert all(r["quantity"] in fe.LABELS for r in rows) diff --git a/tests/test_urbannav_run.py b/tests/test_urbannav_run.py index 22ae36b..739cf9f 100644 --- a/tests/test_urbannav_run.py +++ b/tests/test_urbannav_run.py @@ -203,7 +203,8 @@ def test_the_run_reports_what_the_reader_dropped_and_the_variances_the_filter_wo def test_the_page_says_what_it_does_not_show(): text = PAGE.read_text(encoding="utf-8") - assert "Deep-Urban-1, the planned unseen check, has not been fetched or run" in text + assert "has not been fetched or run" not in text # Deep-Urban-1 has been run, as an aided run + assert "does not say how the filter does on a sequence where the receiver reports no outage" in text assert "not blind" in text and "not field performance" in text @@ -255,28 +256,30 @@ def test_the_deep_urban_section_has_its_generated_blocks_and_nothing_typed_in_th def test_the_deep_urban_prose_has_no_typed_numbers(): prose = _outside_blocks(_deep_section()) - for allowed in ("Deep-Urban-1", "Tunnel-1", "issue #95", "2-sigma"): + for allowed in ("Deep-Urban-1", "Tunnel-1", "issue #95", "2-sigma", "SPAN-CPT"): prose = prose.replace(allowed, "") prose = re.sub(r"^\s*\d+\. ", "", prose, flags=re.M) # the numbering of a list assert not re.search(r"\d", prose), re.findall(r".{20}\d.{20}", prose) -def test_until_it_is_run_the_page_says_deep_urban_1_has_not_been_run(rows): - if ur.rows_of(rows, DEEP): - pytest.skip("Deep-Urban-1 has rows in the CSV; the statements below are for the state without them") +def test_a_sequence_with_no_rows_is_said_not_to_have_been_run(): + assert "has not been run for this page" in ur.run_markdown([], "declared", DEEP) + assert "has not been run for this page" in ur.oracle_markdown([], DEEP) + assert "has not been run" not in ur.run_markdown(ur.rows_from_csv(CSV), "declared", DEEP) + + +def test_the_deep_urban_status_is_that_it_has_been_run(rows): section = _deep_section() - assert "Status: not run for this page" in section - assert section.count("has not been run for this page") == 3 # the three generated blocks - assert "Deep-Urban-1, the planned unseen check, has not been fetched or run" in PAGE.read_text(encoding="utf-8") + assert ur.rows_of(rows, DEEP) and "Status: run for this page" in section + assert "has not been run for this page" not in section + assert "Status: not run" not in PAGE.read_text(encoding="utf-8") -def test_when_deep_urban_1_has_rows_they_are_the_aided_run(rows): - """Switched on by the commit that adds the rows. Until then it skips, and the test above holds.""" +def test_deep_urban_1_has_rows_and_they_are_the_aided_run(rows): deep = ur.rows_of(rows, DEEP) - if not deep: - pytest.skip("no Deep-Urban-1 rows in the committed CSV yet") assert {r["variant"] for r in deep} == set(ur.VARIANTS) # no oracle rows assert ur.oracle_markdown(rows, DEEP) == ur.ORACLE_NOT_APPLICABLE + assert ur.ORACLE_NOT_APPLICABLE in PAGE.read_text(encoding="utf-8") for v in ur.VARIANTS: used, rejected = _q(rows, v, "fixes_used", DEEP), _q(rows, v, "gnss_updates_rejected", DEEP) assert np.all(used + rejected == _q(rows, v, "fixes_total", DEEP)) @@ -285,6 +288,170 @@ def test_when_deep_urban_1_has_rows_they_are_the_aided_run(rows): assert len(_q(rows, v, key, DEEP)) == len(ur.SEEDS), (v, key) +def test_the_absolute_trajectory_error_is_in_the_csv_and_not_in_the_page_tables(rows): + assert all(len(_q(rows, v, "ate_rmse_m", DEEP)) == len(ur.SEEDS) for v in ur.VARIANTS) + for v in ur.VARIANTS: + assert "Absolute trajectory error" not in ur.run_markdown(rows, v, DEEP) + assert "Horizontal position RMSE" in ur.run_markdown(rows, v, DEEP) + assert "Absolute trajectory error" not in PAGE.read_text(encoding="utf-8") + + +# -------------------------------------- the statements of the Deep-Urban-1 section, each tied to the CSVs + +DATA_CSV = ROOT / "docs" / "data" / "urbannav_data.csv" +FIX_ERRORS_CSV = ROOT / "docs" / "data" / "urbannav_fix_errors.csv" +HOR_NEES_EXPECTED, ATT_NEES_EXPECTED = 2.0, 3.0 + + +def _fix_errors(sequence, quantity): + from navkit import urbannav_fix_errors as fe + + (value,) = [ + float(r["value"]) + for r in fe.rows_from_csv(FIX_ERRORS_CSV) + if r["sequence"] == sequence and r["quantity"] == quantity + ] + return value + + +def _data(sequence, quantity): + from navkit import urbannav_check as uc + + (value,) = [ + float(r["value"]) for r in uc.rows_from_csv(DATA_CSV) if r["sequence"] == sequence and r["quantity"] == quantity + ] + return value + + +def test_statement_the_receiver_reports_no_outage(rows): + assert _data(DEEP, "gnss_outages") == 0 and _data(DEEP, "gnss_denied_epochs") == 0 + assert _data(DEEP, "gnss_epochs") == _data(DEEP, "reference_epochs") + assert _data(TUNNEL, "gnss_outages") == 1 # the control: the same rule does find the tunnel + assert "The receiver reports no outage here" in _deep_section() + + +def test_statement_the_gate_rejects_none_though_far_fixes_exist_and_the_nis_is_far_below_its_level(rows): + for v in ur.VARIANTS: + assert np.all(_q(rows, v, "gnss_updates_rejected", DEEP) == 0) + assert np.all(_q(rows, v, "gnss_channel_faulted", DEEP) == 0) + assert np.all(_q(rows, v, "fixes_over_3sigma", DEEP) > 0) and np.all(_q(rows, v, "wrong_fixes", DEEP) > 0) + assert np.all(_q(rows, v, "wrong_fixes_over_gate", DEEP) == 0) + assert np.all(_q(rows, v, "nis_mean", DEEP) < 0.25 * _q(rows, v, "fix_gate_level", DEEP)) + alpha = _q(rows, v, "fix_gate_alpha", DEEP) + assert np.all(_q(rows, v, "fix_rejection_wilson_lo", DEEP) <= alpha) + assert np.all(alpha <= _q(rows, v, "fix_rejection_wilson_hi", DEEP)) + assert np.all(_q(rows, v, "fix_gate_level", DEEP) == pytest.approx(chi2_threshold(3, ur.GATE_ALPHA), rel=1e-3)) + assert "The gate rejects none of the offered fixes" in _deep_section() + + +def test_statement_the_rejection_count_convention_for_a_constant_series(rows): + for v in ur.VARIANTS: + assert np.all(_q(rows, v, "fix_rejection_ess_defined", DEEP) == 0) + assert np.all(_q(rows, v, "fix_rejection_ess", DEEP) == _q(rows, v, "fixes_total", DEEP)) + assert np.all(_q(rows, v, "fix_rejection_eff_wilson_hi", DEEP) == _q(rows, v, "fix_rejection_wilson_hi", DEEP)) + assert "counts every fix as an independent" in " ".join(_deep_section().split()) + + +def test_statement_the_filter_is_overconfident_in_position_and_attitude(rows): + for v in ur.VARIANTS: + assert np.all(_q(rows, v, "hor_nees_mean", DEEP) > 3 * HOR_NEES_EXPECTED) + assert np.all(_q(rows, v, "att_nees_mean", DEEP) > 3 * ATT_NEES_EXPECTED) + assert np.all(_q(rows, v, "hor_coverage_2sigma", DEEP) < ur.COVERAGE_2SIGMA_GAUSSIAN - 0.25) + assert np.all(_q(rows, v, "claimed_sigma_median_m", DEEP) < _q(rows, v, "receiver_sigma_median_m", DEEP)) + assert "The filter is overconfident against this reference" in _deep_section() + + +def test_statement_the_excess_is_in_every_third_so_it_is_not_a_start_up_transient(rows): + for v in ur.VARIANTS: + for k in (1, 2, 3): + assert np.all(_q(rows, v, f"hor_nees_third{k}", DEEP) > 2 * HOR_NEES_EXPECTED), (v, k) + assert np.all(_q(rows, v, f"att_nees_third{k}", DEEP) > 2 * ATT_NEES_EXPECTED), (v, k) + mean = ( + _q(rows, v, "hor_nees_third1", DEEP) + + _q(rows, v, "hor_nees_third2", DEEP) + + _q(rows, v, "hor_nees_third3", DEEP) + ) / 3 + assert mean == pytest.approx(_q(rows, v, "hor_nees_mean", DEEP), rel=0.01) # the thirds are the run's parts + assert "not a start-up transient" in _deep_section() + + +def test_statement_the_means_have_few_independent_samples_behind_them(rows): + for v in ur.VARIANTS: + n = _q(rows, v, "fixes_total", DEEP) + assert np.all(_q(rows, v, "hor_nees_ess", DEEP) < 100) # of 614782 IMU steps + assert np.all(_q(rows, v, "hor_nees_ess", DEEP) < n) and np.all(_q(rows, v, "att_nees_ess", DEEP) < n) + assert "few independent samples" in _deep_section() + + +def test_statement_the_receivers_own_sigma_is_optimistic_and_both_variants_see_the_same_fixes(rows): + for key in ("fix_nees_mean", "fix_err_p50_m", "fix_err_p95_m", "fix_err_p99_m", "fixes_over_3sigma", "wrong_fixes"): + assert np.array_equal(_q(rows, "declared", key, DEEP), _q(rows, "per_fix", key, DEEP)), key + assert np.all(_q(rows, "declared", "fix_nees_mean", DEEP) > HOR_NEES_EXPECTED) + + +def test_statement_the_height_is_not_relied_on_and_its_error_is_large(rows): + assert float(ur.DECLARED["gnss_vertical_sigma"][0]) == 100.0 + for v in ur.VARIANTS: + assert np.all(_q(rows, v, "ver_rmse_m", DEEP) > 2 * _q(rows, v, "hor_rmse_m", DEEP)) + assert "The height is not relied on" in _deep_section() + + +def test_statement_the_reference_is_an_estimate_with_several_quality_codes(): + assert _data(DEEP, "reference_quality_codes") > 1 + section = " ".join(_deep_section().split()) + assert "post-processed estimate" in section and "its error is not known" in section + + +def test_interpretation_the_fix_errors_are_correlated_over_tens_of_seconds_and_few_samples_are_independent(): + n = _fix_errors(DEEP, "fix_error_fixes") + assert _fix_errors(DEEP, "fix_error_corr_pooled_lag10s") > 0.9 + assert _fix_errors(DEEP, "fix_error_corr_pooled_lag30s") > 0.5 + assert _fix_errors(DEEP, "fix_error_corr_pooled_lag60s") > 0.3 + for axis in ("east", "north"): + assert _fix_errors(DEEP, f"fix_error_ess_{axis}") < 0.05 * n + assert _fix_errors(DEEP, f"fix_error_ess_{axis}_truncated") == 0 + # the other sequence is less correlated at ten seconds: the correlation is not a property of every sequence + assert _fix_errors(TUNNEL, "fix_error_corr_pooled_lag10s") < 0.5 * _fix_errors(DEEP, "fix_error_corr_pooled_lag10s") + section = " ".join(_deep_section().split()) + for phrase in ( + "INTERPRETATION, not a result of this page", + "treats the receiver's fixes as independent measurements", + "correlated over tens of seconds", + "does not prove the cause", + "the filter was not run with a correlated-error model here", + "some of the correlated difference could be in the reference", + ): + assert phrase in section, phrase + + +def test_the_fix_error_block_is_what_the_code_and_its_csv_produce(): + from navkit import urbannav_fix_errors as fe + + rows = fe.rows_from_csv(FIX_ERRORS_CSV) + assert {r["sequence"] for r in rows} == {TUNNEL, DEEP} + text = PAGE.read_text(encoding="utf-8") + for fn in fe.BLOCKS.values(): + assert fn(rows) in text + + +def test_the_fix_error_csv_holds_aggregates_only_and_the_tunnel_data_rows_are_untouched(): + from navkit import urbannav_fix_errors as fe + + text = FIX_ERRORS_CSV.read_text(encoding="utf-8") + assert text.splitlines()[0] == ",".join(fe.CSV_COLUMNS) + assert all(r["quantity"] in fe.LABELS for r in fe.rows_from_csv(FIX_ERRORS_CSV)) + assert len(text.splitlines()) < 100 # a table of aggregates, not of fixes + # the committed Tunnel-1 lines of the two older CSVs, hashed (the values recorded before Deep-Urban-1 was added) + import hashlib + + def tunnel_hash(path): + lines = [x for x in path.read_bytes().splitlines(keepends=True) if x.startswith(b"tunnel1,")] + return hashlib.sha256(b"".join(lines)).hexdigest() + + assert tunnel_hash(CSV) == "de65bb1c4a9cee30aaaafc02f8046ab23b339d0bf7379d3a279433ec815783c3" + assert tunnel_hash(DATA_CSV) == "b9d3fc433e183e15c30c6a44a5c9a4c28b86cdc973fc0da8b439486a480a0b68" + + # ----------------------------------------------------- the reader's report and the oracles diff --git a/tests/test_urbannav_synthetic.py b/tests/test_urbannav_synthetic.py index 90a0c51..28fd462 100644 --- a/tests/test_urbannav_synthetic.py +++ b/tests/test_urbannav_synthetic.py @@ -569,6 +569,39 @@ def test_the_aided_run_reports_the_intervals_and_sample_sizes_it_documents(aided assert r["hor_nees_ess"] >= 1.0 and r["att_nees_ess"] >= 1.0, v +def test_the_aided_summary_counts_fixes_over_three_sigma_and_compares_the_sigmas(): + n = 60 + d = np.array([1.0, 5.9, 6.1, 20.0]) # receiver sigma 2: two of these are over 3 sigma + nis = np.array([0.5, 1.5, 2.5, 0.1]) + out = ur.aided_summary( + _known_detail(n), + {"gnss_updates_used": 4, "gnss_updates_rejected": 0}, + 4, + d, + np.full(4, 2.0), + nis=nis, + gate_level=16.0, + ) + assert out["fixes_over_3sigma"] == 2 + assert out["receiver_sigma_median_m"] == 2.0 and out["claimed_sigma_median_m"] == pytest.approx(2.0) # sqrt(4) + assert out["nis_mean"] == pytest.approx(1.15) and out["fix_gate_level"] == 16.0 + + +def test_the_nees_thirds_are_the_means_of_the_first_middle_and_last_part(): + n = 90 + rng = np.random.default_rng(7) + ref = _trajectory(np.zeros((n, 3)), np.tile(np.eye(3), (n, 1, 1))) + ramp = np.repeat([0.0, 1.0, 2.0], n // 3) # error of 0, 1 and 2 sigma in each axis, by third + est = _trajectory(np.column_stack([ramp * 2.0, ramp * 2.0, np.zeros(n)]), np.tile(np.eye(3), (n, 1, 1))) + detail = {**_known_detail(n), "est": est, "reference": ref} + nees = np.repeat([0.0, 2.0, 8.0], n // 3) + 0.0 * rng.normal(size=n) + out = ur.aided_summary( + detail, {"gnss_updates_used": 4, "gnss_updates_rejected": 0}, 4, np.ones(4), np.ones(4), nees_h=nees + ) + assert [out[f"hor_nees_third{k}"] for k in (1, 2, 3)] == [0.0, 2.0, 8.0] + assert [out[f"att_nees_third{k}"] for k in (1, 2, 3)] == [0.0, 0.0, 0.0] + + def test_the_coverage_expectation_is_the_gaussian_two_sigma_share(): assert abs(ur.COVERAGE_2SIGMA_GAUSSIAN - 0.9545) < 5e-5 @@ -673,11 +706,24 @@ def test_the_committed_csv_is_a_prefix_of_what_a_second_sequence_adds_to_it(tmp_ committed = Path(__file__).resolve().parent.parent / "docs" / "data" / "urbannav_run.csv" copy = tmp_path / "copy.csv" copy.write_bytes(committed.read_bytes()) - ur.write_csv(copy, _stub_rows(DEEP)) + ur.write_csv(copy, _stub_rows("stubseq")) # a sequence the file does not hold: added at the end assert copy.read_bytes().startswith(committed.read_bytes()) # writing back the rows the file already holds gives the same bytes: the writer is the one that wrote them - ur.write_csv(copy, ur.rows_from_csv(committed)) - assert copy.read_bytes().startswith(committed.read_bytes()) + again = tmp_path / "again.csv" + again.write_bytes(committed.read_bytes()) + ur.write_csv(again, ur.rows_from_csv(committed)) + assert again.read_bytes() == committed.read_bytes() + + +def test_rewriting_deep_urban_1_leaves_the_tunnel_1_lines_byte_identical(tmp_path): + committed = Path(__file__).resolve().parent.parent / "docs" / "data" / "urbannav_run.csv" + copy = tmp_path / "copy.csv" + copy.write_bytes(committed.read_bytes()) + tunnel = _csv_lines(committed, SEQ) + assert tunnel and _csv_lines(committed, DEEP) + ur.write_csv(copy, _stub_rows(DEEP)) + assert _csv_lines(copy, SEQ) == tunnel + assert _csv_lines(copy, DEEP) != _csv_lines(committed, DEEP) def test_a_csv_with_another_header_is_refused_and_not_overwritten(tmp_path): From ee434005e01766be3715b7704e318fe1d60dcf09 Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:18:48 +0200 Subject: [PATCH 3/5] chore: regenerate the coverage and test count markers CI measured 93.76% against the 94.31% recorded, a gap above the 0.5 points allowed. Regenerated with scripts/metrics.py --write --coverage after merging origin/main. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01L6ro5x4WRA4P7sQHLyPPGa --- CONSTRAINTS.md | 10 +++++----- CONTRIBUTING.md | 2 +- ROADMAP.md | 2 +- docs/data/metrics.json | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CONSTRAINTS.md b/CONSTRAINTS.md index fb52222..4163f30 100644 --- a/CONSTRAINTS.md +++ b/CONSTRAINTS.md @@ -4,9 +4,9 @@ Invariants this project holds itself to. Every entry is either currently true or is a stated blocker; nothing here is aspirational. Ratchets are measured, not estimated, and the command to re-measure each one is given. -Last measured: more than 2400 tests collected, all of which pass except the deliberate -xfails and the skips that `pytest -rs` lists, and 94.3% -line coverage (10033/10638 executable lines) on **CPython 3.13**, measured +Last measured: more than 2500 tests collected, all of which pass except the deliberate +xfails and the skips that `pytest -rs` lists, and 93.8% +line coverage (14787/15761 executable lines) on **CPython 3.13**, measured with `python scripts/coverage_report.py`. The figures come from `docs/data/metrics.json`, which `scripts/metrics.py` checks in CI. Coverage is quoted from that script alone: it is the project's dependency-free tracer (S1 forbids adding `pytest-cov`), and a number taken from a different tool is not comparable with the module figures @@ -91,9 +91,9 @@ the floors themselves are enforced by `scripts/coverage_report.py --ratchet`. | Ratchet | Floor | Current (CPython 3.13) | Re-measure with | |---|---:|---:|---| -| Tests collected | 405 | more than 2400 | `pytest`, `scripts/metrics.py --check` | +| Tests collected | 405 | more than 2500 | `pytest`, `scripts/metrics.py --check` | | Tests passing | 300 | all collected, except the deliberate xfails and the listed skips | `pytest -rs` | -| Line coverage | 75% | 94.3% | `scripts/coverage_report.py`, `scripts/metrics.py --check-coverage` | +| Line coverage | 75% | 93.8% | `scripts/coverage_report.py`, `scripts/metrics.py --check-coverage` | | `io/trajectory.py` coverage | 85% | 95.0% | as above | | `analysis/findings.py` coverage | 80% | 99.7% | as above | | `config.py` coverage | 80% | 98.4% | as above | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57d17a0..e153fd2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,7 +52,7 @@ mypy src --ignore-missing-imports `ruff format src tests scripts` applies formatting; the `--check` form is the gate. -The suite collects more than 2400 tests and all of them pass on a fresh clone, except the deliberate +The suite collects more than 2500 tests and all of them pass on a fresh clone, except the deliberate xfails and the skips. `pytest -rs` lists the skips: two need TUM VI reference data that is not vendored (see below), and the doc-table check runs once `python scripts/run_benchmark.py` has written `results/benchmark.json`. The count and the other numbers the docs quote are in diff --git a/ROADMAP.md b/ROADMAP.md index c7771c8..193e13c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,7 +5,7 @@ Status markers: `[x]` done, `[~]` in progress, `[ ]` not started, `[!]` blocked. Checked items record what was measured when the item closed. They are not kept current: the current values are in the README tables, `docs/results.md` and `CONSTRAINTS.md` (B1). -Measured baseline: more than 2400 tests collected and 94.3% line coverage on CPython +Measured baseline: more than 2500 tests collected and 93.8% line coverage on CPython 3.13 (`docs/data/metrics.json`, from `scripts/metrics.py` and `scripts/coverage_report.py`). Every number below re-measures against that, not against a remembered value. diff --git a/docs/data/metrics.json b/docs/data/metrics.json index 2184bbb..a7d3543 100644 --- a/docs/data/metrics.json +++ b/docs/data/metrics.json @@ -1,10 +1,10 @@ { "adr_count": 36, - "coverage_lines_hit": 10033, - "coverage_lines_total": 10638, - "coverage_percent": 94.31, + "coverage_lines_hit": 14787, + "coverage_lines_total": 15761, + "coverage_percent": 93.82, "coverage_python": "3.13", "outage_visual_window_coverage_percent": 5.0, "outage_visual_window_nees": 2783.35, - "tests_collected_floor": 2400 + "tests_collected_floor": 2500 } From cd4617deaa66c6d3b8c66c2fadb3858d00d5cfec Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:53:15 +0200 Subject: [PATCH 4/5] test(urbannav): exercise the check and fix-error studies on the synthetic sequence CI has no real dataset, so urbannav_check and urbannav_fix_errors were at 73% and 58% line coverage, and total coverage fell 0.55 points below metrics.json. These tests run measure(), the CSV and page-block paths and main() on the generated fixture. Both modules are now above 99%. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01QnVDRzD2rMdsXzjGpA5up9 --- tests/test_urbannav_studies_synthetic.py | 111 +++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/test_urbannav_studies_synthetic.py diff --git a/tests/test_urbannav_studies_synthetic.py b/tests/test_urbannav_studies_synthetic.py new file mode 100644 index 0000000..37b20b4 --- /dev/null +++ b/tests/test_urbannav_studies_synthetic.py @@ -0,0 +1,111 @@ +"""The two data-section studies (``urbannav_check``, ``urbannav_fix_errors``) end to end on the made-up sequence. + +The fixture comes from ``test_urbannav_synthetic`` (S2: generated from code, no data file committed). These tests do +not skip when the real dataset is absent, so CI exercises the measuring code and the command line that the real +data would. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from navkit import urbannav_check as uc +from navkit import urbannav_fix_errors as fe +from navkit.recorded import SequenceError +from test_urbannav_synthetic import SEQ, write_tunnel + +PAGE = "x\n\n\ny\n" + + +@pytest.fixture(scope="module") +def tunnel(tmp_path_factory: pytest.TempPathFactory) -> Path: + return write_tunnel(tmp_path_factory.mktemp("studies")) + + +@pytest.fixture(scope="module") +def check_rows(tunnel: Path) -> list[dict]: + return uc.measure(tunnel, SEQ) + + +@pytest.fixture(scope="module") +def fix_rows(tunnel: Path) -> list[dict]: + return fe.measure(tunnel, SEQ) + + +def test_the_check_reports_the_documented_quantities_for_the_made_up_sequence(check_rows): + q = {r["quantity"]: r for r in check_rows} + assert {"imu_samples", "clock_offset", "gnss_outages", "gnss_denied_epochs"} <= set(q) + assert q["gnss_outages"]["value"] == 1 + assert all(r["sequence"] == SEQ for r in check_rows) + assert all(r["quantity"] in uc.LABELS for r in check_rows) + + +def test_the_fix_errors_are_measured_against_the_reference_for_every_lag(fix_rows): + q = {r["quantity"]: r for r in fix_rows} + assert int(q["fix_error_fixes"]["value"]) > 0 + for lag in fe.LAGS_S: + assert f"fix_error_pairs_lag{lag}s" in q + assert all(r["quantity"] in fe.LABELS for r in fix_rows) + + +def test_the_fix_error_series_has_one_row_of_east_north_per_valid_fix(tunnel): + t, err = fe.fix_errors(tunnel, SEQ) + assert err.shape == (len(t), 2) + assert (t[1:] > t[:-1]).all() + + +@pytest.mark.parametrize("mod, name", [(uc, "urbannav-data"), (fe, "urbannav-fix-errors")]) +def test_a_csv_round_trips_and_fills_its_page_block(mod, name, check_rows, fix_rows, tmp_path): + rows = check_rows if mod is uc else fix_rows + csv_path, page = tmp_path / "r.csv", tmp_path / "p.md" + mod.write_csv(csv_path, rows) + back = mod.rows_from_csv(csv_path) + assert [r["quantity"] for r in back] == [r["quantity"] for r in rows] + page.write_text(PAGE.format(name=name), encoding="utf-8") + mod.write_block(page, name, mod.table_markdown(back)) + assert mod.table_markdown(back) in page.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("mod, name", [(uc, "urbannav-data"), (fe, "urbannav-fix-errors")]) +def test_main_measures_writes_the_csv_and_fills_the_page(mod, name, tunnel, tmp_path): + csv_path, page = tmp_path / "r.csv", tmp_path / "p.md" + page.write_text(PAGE.format(name=name), encoding="utf-8") + argv = ["--root", str(tunnel), "--sequence", SEQ, "--csv", str(csv_path), "--page", str(page)] + assert mod.main(argv) == 0 + assert csv_path.exists() and "| Sequence |" in page.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("mod", [uc, fe]) +def test_main_prints_the_table_when_given_no_output(mod, tunnel, capsys): + assert mod.main(["--root", str(tunnel), "--sequence", SEQ]) == 0 + assert "| Sequence |" in capsys.readouterr().out + + +def test_fix_errors_main_needs_a_csv_to_fill_a_page(tmp_path, capsys): + assert fe.main(["--page", str(tmp_path / "p.md")]) == 2 + assert "--page needs" in capsys.readouterr().err + + +def test_fix_errors_main_reports_missing_data_and_a_page_without_markers(tunnel, tmp_path): + assert fe.main(["--root", str(tmp_path), "--sequence", SEQ]) == 1 + page = tmp_path / "p.md" + page.write_text("no markers\n", encoding="utf-8") + argv = ["--root", str(tunnel), "--sequence", SEQ, "--csv", str(tmp_path / "r.csv"), "--page", str(page)] + assert fe.main(argv) == 1 + + +def test_fix_errors_main_refuses_a_csv_with_another_header(tunnel, tmp_path): + bad = tmp_path / "bad.csv" + bad.write_text("a,b\n1,2\n", encoding="utf-8") + assert fe.main(["--root", str(tunnel), "--sequence", SEQ, "--csv", str(bad)]) == 1 + assert bad.read_text(encoding="utf-8") == "a,b\n1,2\n" + + +def test_too_few_fixes_is_a_sequence_error(monkeypatch, tunnel): + import numpy as np + + monkeypatch.setattr(fe, "fix_errors", lambda root, seq: (np.arange(3.0), np.zeros((3, 2)))) + with pytest.raises(SequenceError): + fe.measure(tunnel, SEQ) From db8b20d354cf86296acc77cc32645ebbef033dfa Mon Sep 17 00:00:00 2001 From: Daniel Telschow <48818540+Telschow@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:09:37 +0200 Subject: [PATCH 5/5] docs: regenerate the coverage markers Generated by scripts/metrics.py --write --coverage after the new study tests. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01QnVDRzD2rMdsXzjGpA5up9 --- CONSTRAINTS.md | 6 +++--- ROADMAP.md | 2 +- docs/data/metrics.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONSTRAINTS.md b/CONSTRAINTS.md index 4163f30..1221bb6 100644 --- a/CONSTRAINTS.md +++ b/CONSTRAINTS.md @@ -5,8 +5,8 @@ or is a stated blocker; nothing here is aspirational. Ratchets are measured, not estimated, and the command to re-measure each one is given. Last measured: more than 2500 tests collected, all of which pass except the deliberate -xfails and the skips that `pytest -rs` lists, and 93.8% -line coverage (14787/15761 executable lines) on **CPython 3.13**, measured +xfails and the skips that `pytest -rs` lists, and 94.5% +line coverage (14889/15761 executable lines) on **CPython 3.13**, measured with `python scripts/coverage_report.py`. The figures come from `docs/data/metrics.json`, which `scripts/metrics.py` checks in CI. Coverage is quoted from that script alone: it is the project's dependency-free tracer (S1 forbids adding `pytest-cov`), and a number taken from a different tool is not comparable with the module figures @@ -93,7 +93,7 @@ the floors themselves are enforced by `scripts/coverage_report.py --ratchet`. |---|---:|---:|---| | Tests collected | 405 | more than 2500 | `pytest`, `scripts/metrics.py --check` | | Tests passing | 300 | all collected, except the deliberate xfails and the listed skips | `pytest -rs` | -| Line coverage | 75% | 93.8% | `scripts/coverage_report.py`, `scripts/metrics.py --check-coverage` | +| Line coverage | 75% | 94.5% | `scripts/coverage_report.py`, `scripts/metrics.py --check-coverage` | | `io/trajectory.py` coverage | 85% | 95.0% | as above | | `analysis/findings.py` coverage | 80% | 99.7% | as above | | `config.py` coverage | 80% | 98.4% | as above | diff --git a/ROADMAP.md b/ROADMAP.md index 193e13c..aa2bc22 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,7 +5,7 @@ Status markers: `[x]` done, `[~]` in progress, `[ ]` not started, `[!]` blocked. Checked items record what was measured when the item closed. They are not kept current: the current values are in the README tables, `docs/results.md` and `CONSTRAINTS.md` (B1). -Measured baseline: more than 2500 tests collected and 93.8% line coverage on CPython +Measured baseline: more than 2500 tests collected and 94.5% line coverage on CPython 3.13 (`docs/data/metrics.json`, from `scripts/metrics.py` and `scripts/coverage_report.py`). Every number below re-measures against that, not against a remembered value. diff --git a/docs/data/metrics.json b/docs/data/metrics.json index a7d3543..84f3d27 100644 --- a/docs/data/metrics.json +++ b/docs/data/metrics.json @@ -1,8 +1,8 @@ { "adr_count": 36, - "coverage_lines_hit": 14787, + "coverage_lines_hit": 14889, "coverage_lines_total": 15761, - "coverage_percent": 93.82, + "coverage_percent": 94.47, "coverage_python": "3.13", "outage_visual_window_coverage_percent": 5.0, "outage_visual_window_nees": 2783.35,