From b792a3bbe20304313d8e47ba22dc1d35855105e5 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:10:29 +0200 Subject: [PATCH 01/18] Back to the beta channel after 0.10.0 Co-Authored-By: Claude Opus 5.5 --- visionstate/config.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/visionstate/config.yaml b/visionstate/config.yaml index 4a40245..d3463cd 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ -name: VisionState -version: "0.10.0" +name: VisionState (beta) +version: "0.10.0b7" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState @@ -16,7 +16,7 @@ ingress_port: 8099 # a request into memory and refuses one over 16 MB. ingress_stream: true panel_icon: mdi:eye-check-outline -panel_title: VisionState +panel_title: VisionState β homeassistant_api: true hassio_api: true services: @@ -39,3 +39,5 @@ schema: mqtt_port: port? mqtt_username: str? mqtt_password: password? +environment: + VISIONSTATE_MEDIA: /media/visionstate_beta From 612fa6a2b3bfda2857c25c94ece1f4e055cc339d Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Fri, 23 Oct 2026 08:33:27 +0200 Subject: [PATCH 02/18] Readings export: where the region is in each image; a new wheel reader plan The export's margin around the region is cut off at the frame's edges, so the region was not always centred in the exported image. readings.json (format 4) now gives each image's "box", and tools/wheelreader cuts the cells from it. docs/WHEEL_READER_PLAN.md is rewritten from the new exports: a benchmark, wheels-v2 with cells that see their neighbours (robust to a region that does not fit), the weakest-wheel confidence, an embedding output and a wheel model per sensor. Co-Authored-By: Claude Opus 5.5 --- docs/SCOPE.md | 9 +- docs/WHEEL_READER_PLAN.md | 565 ++++++++++------------ tools/wheelreader/common.py | 6 +- tools/wheelreader/items.py | 2 +- visionstate/backend/tests/test_reading.py | 7 +- visionstate/backend/visionstate/bundle.py | 16 +- 6 files changed, 274 insertions(+), 331 deletions(-) diff --git a/docs/SCOPE.md b/docs/SCOPE.md index e46fb47..6fb9635 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -365,7 +365,7 @@ times its busiest shower) rejected right readings after a 35-minute update. when the size is back, or when the user says the region is right (`POST /sensors/{id}/accept-picture`) or draws a new one (both clear the size). Held rather than paused, so a camera that comes back at its old size needs nothing. A camera that moved without - changing size is not caught by this: that is C3 in `docs/WHEEL_READER_PLAN.md`. + changing size is not caught by this: that is *camera drift* in `docs/WHEEL_READER_PLAN.md` §8. **Wheel reader** (default for mechanical counters, `counter_reader: "wheels"`, registry `wheel_readers` in `readers.json`, model `wheels-v1`, bundled, 2.2 MB, ~2 ms for 8 wheels on one @@ -687,9 +687,10 @@ automatically. checked, accepted or rejected (newest first, `"answer": "unchecked"`; never used as labels without a check, see `docs/WHEEL_READER_PLAN.md`), whose frame is kept — each only the region it was read in plus `export_margin` (the region is kept with every stored reading since 0.7.1b2, - `probs["roi"]`; older ones are cut with today's region, `"region": "current"`). One ZIP of at + `probs["roi"]`; older ones are cut with today's region, `"region": "current"`; `"box"`: the + region inside the image in pixels, as the margin is cut off at the frame's edges). One ZIP of at most `READING["export_max_mb"]` (GitHub takes 25 MB per file; about 1,000 readings); the - readings that did not fit are counted (`left_out`). `readings.json` (`"format": 3`; 2 left out + readings that did not fit are counted (`left_out`). `readings.json` (`"format": 4`; 3 had no `"box"`, 2 left out the rejected readings nobody answered, so a sensor's checks could not be replayed: read text, value, rejection reason, confidence, answer, right value, day only and seconds since the oldest reading; per reading the reader id and app version (kept with every stored reading since 0.7.1b5, null before), whether the picture was greyscale (IR), whether @@ -801,7 +802,7 @@ is part of backups and needs no restart. | **Counter limits** ✅ | A counter's limit grows with the time it was away (learned, a fixed rate or per reading), Review moves a counter on, readings held when the camera's picture changes size, training images in the app's storage (backups hold them) | 0.9.1 (betas 0.9.1b1–b6) | | **Reviewing readings** ✅ | A counter's value sent at its first reading, decimals for its rate, a rejected reading that repeats waits in Review once, Review shows a reading's region large and one sensor at a time | 0.9.2 (betas 0.9.2b1–b4) | | **Backup of everything** ✅ | One file with everything (parts to leave out, camera passwords only when ticked), a restore that replaces it and keeps the entities, backups of every database version kept in the tests; uploads over 16 MB through Ingress; much less memory (no ONNX Runtime arena, classifier without scikit-learn) | 0.10.0 (betas 0.10.0b1–b7) | -| **Better counter readings** | Benchmark, the last value as an expectation, calibrated confidence, extra decimal, self-calibrating regions, wheels-v2, learning per sensor — status per phase in [`WHEEL_READER_PLAN.md`](WHEEL_READER_PLAN.md) | planned | +| **Better counter readings** | Benchmark, wheels-v2 (cells with context, weakest-wheel confidence), more shared meters, later the last value as an expectation — status per step in [`WHEEL_READER_PLAN.md`](WHEEL_READER_PLAN.md) | planned | **Open ideas** (not scheduled): more on the History page (export of a filtered selection, a summary per hour or day, "seen in the frame" as a filter, saved filters, bulk actions); merge diff --git a/docs/WHEEL_READER_PLAN.md b/docs/WHEEL_READER_PLAN.md index a1cc093..eecf054 100644 --- a/docs/WHEEL_READER_PLAN.md +++ b/docs/WHEEL_READER_PLAN.md @@ -1,196 +1,212 @@ # Wheel reader: plan for better counter readings -The design for improving how VisionState reads **mechanical counters** (rolling digit wheels on -water and gas meters); what is done and what is next is in the table in §9. It applies to reading sensors with the *counter* display and the wheel reader -(`counter_reader: "wheels"`). The general design is in [`SCOPE.md`](SCOPE.md), and the model and its -training are in [`tools/wheelreader`](../tools/wheelreader/README.md). - -## 1. Where we are (wheels-v1) - -- **How it reads.** The region is split into one cell per wheel. A small CNN (565 k parameters, - 2.2 MB, about 2 ms for 8 wheels on one CPU thread) gives each wheel's position as a distribution - over 100 bins (0.0–9.9). `readers.wheel_value` then finds the most likely value whose wheels fit - together, from right to left, with one common shift of up to ±2 bins. The last wheel is rounded, - and the confidence is the mean probability per wheel. -- **Training data and results:** [`tools/wheelreader/README.md`](../tools/wheelreader/README.md). -- **Known weaknesses, with evidence:** - - *±1 on the last digit* when the last wheel is about half way. The rounding moves with the - per-frame shift (pedro32: 81 exact + 12 at +1). - - *Only three real meter types tested.* Gas meters, IR or night images and angled cameras are untested. - - *Confidence is only moderately informative.* A threshold that rejects 4 % of the reads catches - about half of the wrong ones (Dryad test). - - *Wheels at rest sit off their digit, each its own way*, but the decoder allows only one common - shift plus ±`wheel_slack_bins`. On oleost1 two black wheels sat at 0.3–0.4 past their digit all - day while the others sat at 0.1–0.2. The decoder put them outside the slack, where their - probability dropped to 0.02–0.08 although the digit was plain. That one wheel pulled the mean - under 0.70, so 22 of the 29 rejected readings were right. A low threshold is not the fix: at 0.56, - 3–4 of that day's 7 misreads would have passed, one of them too high (585 instead of 565). It can - also flip the digit. In one sharp, well-lit frame the black 7 had 0.49 of its mass at 7.2–7.3 - and 0.21 at 0.2. Only 7.2 fell inside the slack, so the decoder read 032.776 instead of 732.776 - at confidence 0.84. Only "went down" caught it. See A2. - - *Red wheels half way while the water stands still.* All of oleost1's misreads came from the last - wheel stopped exactly half way (6½). The neighbouring red 9 or 6 was then read as 8, and the last - wheel as 2, 8 or 9. The same hard frame then comes back for as long as nobody draws water. - - *Every frame is read on its own.* The last value is only used afterwards, by hard checks - (`went down`, `max_step`, `readers.settling`). - - *Geometry is assumed:* equal-width cells and a fronto-parallel view. - - *A wrong label can hide in the training data* (pedro32/0120 is labelled 628596; the wheels show 7–8). +How VisionState is to read **mechanical counters** (rolling digit wheels on water and gas meters) +better: what the shared data shows, what was tried, and what comes next, with its status in §9. +It applies to reading sensors with the *counter* display and the wheel reader +(`counter_reader: "wheels"`). The general design is in [`SCOPE.md`](SCOPE.md); the model and how +to train it are in [`tools/wheelreader`](../tools/wheelreader/README.md). + +## 1. What the data shows + +**How wheels-v1 reads.** The region is split into one equal cell per wheel. A small CNN (565 k +parameters, 2.2 MB, about 2 ms for 8 wheels on one CPU thread) gives each wheel's position as a +distribution over 100 bins (0.0–9.9). `readers.wheel_value` finds the most likely value whose +wheels fit together, with one common shift of up to ±2 bins; the confidence is the geometric mean +of the decoded bins. + +**The benchmark** (§3) holds about 6,000 readings of four real meters and the Dryad test photos: +the maintainer's water meter (8 wheels, 3 red, ESP32-CAM; 8 days), pedromfa's (7 wheels, ESP32; +8 days, shared in issue #40), two bolausson meters and Dryad's 5- and 6-digit meters. Results of +wheels-v1, region as drawn: + +| Set | Right (±1 on the last wheel) | Too high | Note | +|---|---|---|---| +| pedromfa, 2 periods | 99.0–99.7 % | 4 + 1 | the replay (§3) of the first period: 3 too high published, 5.3 h blocked | +| maintainer, 19–22 Oct | 98.0 % | 1 | | +| maintainer, 14–18 Oct | 86.6 % | 9 | 6.7 h blocked over the 8 days (5 stretches) | +| bolausson cold / warm | 100 % / 100 % | 0 | | +| Dryad 5 digits / 6 digits | 99.4 % / 90.2 % | 6 / 26 | right way up | + +What goes wrong, in order of harm: + +1. **A region that does not fit the wheels.** v1 reads the middle of equal cells, so a region a + little wider, narrower or off makes the outer cells show half a wheel. On pedromfa's frames + (99–100 % as drawn), with the region's right edge half a cell short v1 read 43–69 % right; + with half a cell more on each side, 0–94 % depending on the training run; 10 % taller added up + to 9 too-high readings. The maintainer's region cut the last wheel, which caused most of the + misreads of 14–18 Oct. +2. **The last (red) wheel half way, on a meter the model has not seen.** On the maintainer's meter + a red wheel between 5 and 6 was read as 7.8–8.0 (two steps too high) or as 2.0; the + second-to-last wheel half way between 1 and 2 was read as 5 (734.156 published for about + 734.120). When the water then stands still, the same hard frame comes back for hours, and a + too-high value blocks the sensor until the meter catches up. Three of the five blocked stretches + came from a too-high value. In the other two the value was right and the readings after it + misread lower. +3. **A confidence that hides one bad wheel.** The mean over wheels stays high when one wheel is + unclear. The weakest wheel's mass within ±0.4 of where the value puts it ("weakest wheel") is a + much better signal. Replayed through the counter checks at 0.70, it published 2 wrong instead + of 5 on the maintainer's meter (436 instead of 419 published), and 0 instead of 3 on + pedromfa's. With 1 % of the accepted readings wrong, 82 % instead of 42 % of the maintainer's + readings were accepted; risk–coverage (AURC) was better on Dryad and pedromfa #32 too. +4. **Dark, striped or noisy frames** (camera glitches) were read as noise; the checks stopped them + all, two above the threshold. +5. **Dryad 6-digit meters** (dirt, blur, fog) are read 83 % exactly. + +**Trained candidates** (same recipe, 25 epochs, three training runs each where it mattered; the +benchmark differs by 3–5 points on the maintainer's hard period and 2 on Dryad 6 between runs of +the *same* recipe, so smaller differences mean nothing): + +| Candidate | Result | +|---|---| +| **Cell with context**: each cell plus half a cell on each side, 64×48 input, drawn rows of wheels with the region off by up to 38 % of a cell | **robust to a region that does not fit**: 99–100 % on pedromfa's frames with the region half a cell short or wide, or 10 % taller (v1: 0–94 %); as drawn the same as v1 (within run-to-run spread); 958 k parameters, 3.7 MB, 4.7 ms for 8 wheels on one thread | +| The same, model twice as wide | no better | +| One cell, drawn cells with the wheel off-centre and neighbours' digits | no better than v1 | +| Shared exports as training data (labels: §5) | on the meter's own other period: small gain at most (14–18 Oct 83–89 % → 91 % with 19–22 Oct in training; the other way round no gain) | + +**Tried without training, and dropped:** reading 3–8 slightly shifted crops and averaging (a +small gain on one meter, none on the others); letting the app tune the region by what the model +finds most likely (§7: it made things worse). ## 2. Principles -1. **The base model stays general and stable.** Per-sensor tuning is a fallback on top of it, never - instead of it, and it can always be switched off. -2. **Every change is measured** on a fixed benchmark (§3) before it ships. Nothing that adds a - reading that is *too high* ships: a too-high value passes the counter checks and blocks the - sensor until the meter catches up. -3. **Only human answers are labels.** The model never trains on its own guesses (confirmation bias: - self-training on its own pseudo-labels makes wrong predictions stronger, especially when the - images differ from the training data). +1. **The base model stays general and stable.** Anything per sensor is a fallback on top of it, + never instead of it, and it can always be switched off. +2. **Every change is measured** on the benchmark (§3) before it ships, with at least three training + runs. Nothing that adds a reading that is *too high* ships: a too-high value passes the counter + checks and blocks the sensor until the meter catches up. +3. **Labels come from people or from other frames, never from the reading itself.** A reading's + label is a person's answer, or the interval between the readings next to it in time that the + sequence confirms (§5). A model never trains on its own value for a frame (confirmation bias: + self-training on its own pseudo-labels makes wrong predictions stronger). 4. **Licences:** training data must allow any use (CC0, or CC BY with attribution in the model card and a verified source). No NC/ND sets, no sets without a licence. 5. **No telemetry.** Data only arrives when a user exports it and shares it on purpose. -6. **The registry rules stay.** A changed model, or changed calibration of a model, gets a new +6. **The registry rules stay.** A new model, or a changed way to compute its confidence, gets a new registry id. Existing sensors keep their behaviour until the user picks the new one, as in 0.8.0 (counters made before the wheel reader kept the text reader). -## 3. Phase 0 — benchmark and metrics (the base for everything else) +## 3. The benchmark -**Benchmark** (in `VisionStateLocal/`, never in the repository; any image that can go into the -repository is CC0 and becomes a test asset): +Built in `VisionStateLocal/wheel/v2/` (data and scripts never go into the repository; a set's +images may only become test assets when they are CC0). To be moved into +[`tools/wheelreader`](../tools/wheelreader/README.md) with wheels-v2. | Set | Source | Use | |---|---|---| -| `pedro40` 0001–0017 | issue #40 (CC0) | held out (wheels-v1 was trained on it in the end: hold out the *next* export instead) | +| `ole2_a`, `ole2_b` | the maintainer's export of 2026-10-22 (CC0), split by date: 14–18 Oct / 19–22 Oct | test; `a` may train a candidate that is then tested on `b` only | +| `ped18_a`, `ped18_b` | pedromfa's export of 2026-10-18 (#40, CC0), up to / after 15 Oct | the same | +| `pedro40`, `pedro32`, `pedro_esp` | pedromfa's earlier exports (#32, #40) | wheels-v1 was trained on them | | `bol_cold`, `bol_warm` | bolausson (GPL-3.0) | test only, never training | -| `dryad_test` | Dryad (CC0) | test, right way up | -| `oleost1`, … | the maintainer's water meter (CC0 exports, one per few days from 2026-10-14; ESP32-CAM 1600×1200 with flash, 8 wheels, 3 red) | held out: test only, a meter with wheels sitting off their digits at rest | -| new shared exports | §5 | **split by meter**: one meter is either training or test, never both | - -**Metrics**, reported per set by `tools/wheelreader/evaluate.py`: - -- **exact** (last digit rounded), **±1**, **too high** (above +1), **too low**. -- **Risk–coverage:** sort the readings by confidence and plot the error rate of the accepted - readings against the share accepted. The default threshold is then picked for a target such as - "at most 1 % of accepted readings wrong", and its coverage (how many are accepted) is reported. - The area under the curve (AURC) gives one number to compare models and decoders. -- **Calibration:** expected calibration error (ECE). Readings are grouped by confidence, and in - each group the share that is right is compared with the mean confidence. -- **Sequence metrics** for exports that are a time series: published value right, number of - rejections, longest stretch blocked by a too-high value. - -**Gate for a new model or decoder:** no set gets more too-high readings, the exact rate on unseen -meters does not drop, and AURC on the benchmark is lower. - -**Label hygiene:** a reading whose label the model strongly disagrees with (very likely value ≠ -label, high confidence) is listed for a person to look at again. That is how pedro32/0120 was -found. The label is corrected or the image is dropped, never silently. - -## 4. Phase A — safer decoding (no new training) - -### A1. The last value as a soft expectation - -**Why in the decoder and not in the network:** the network sees one wheel. Teaching it time would -need sequences we do not have, and it could learn to repeat the last value instead of looking. -In the decoder the expectation can be tested, explained, set per sensor and used with every model. - -**Design:** - -- `wheel_value` gets an optional *prior* over the value V (in last-wheel units, with the fraction): - `score(V) = Σ wheels log p(position_i(V)) + log prior(V)`. -- The prior comes from the last **published** (debounced) value L at time t₀, with elapsed time Δt: - - V < L − settling tolerance: −∞ in practice (a very large penalty, so it can still be inspected); - - L ≤ V ≤ L + R·Δt + slack: flat, with a mild preference for small steps (an exponential - in V − L); - - above that, a steep penalty. -- **R (largest plausible rate):** the sensor's `max_step` per interval when it is set, otherwise - learned from its own history (for example 3× the 99th percentile of the accepted rate over 30 - days, at least a default in `settings.READING`). It is never learned from rejected readings. -- **Search:** the window [L, L + R·Δt + slack] is usually small (a few hundred last-wheel steps). - Every V in it is scored directly (wheels × candidates, microseconds), and also the free - optimum without the prior (today's DP). Result: - - the free optimum is inside the window: same as today, with a better confidence; - - outside the window, but the window's best explains the image almost as well (log-likelihood - ratio below a threshold): take the window's best (an unclear wheel settled by the expectation); - - outside and clearly better: **do not publish**. Mark the reading as "disagrees with the last - value", send it to review and count it. -- **Protection against lock-in** (a wrong value used as the expectation): - - only debounced, published values become L; - - N readings in a row that clearly disagree with L, *and agree with each other*, release the - prior. They are treated as a new start, which also covers a replaced or reset meter. This is - shown in the History as an event and on the Live tab. The counter limit decided otherwise for - now (readings that agree never override it, `docs/SCOPE.md`): a release would wait for Review; - - after a restart, L is the last value from the history (as today), but its age counts: an old L - gives a wide window. -- **UI and history:** a new reason, `disagrees_with_last` (lower-case key for the "problem" entity), - and the Live tab shows "read as … (expected … – …)". -- **Tests:** a pure unit test of the prior and the window decision (drawn positions); integration - tests that replay sequences (one blurry wheel settled by the prior; a meter that is replaced - releases the prior after N readings; a too-high value is never published). -- **Measure:** the sequence metrics on the exports, before and after. - -### A2. A confidence that means what it says - -- **Problem:** the mean per-wheel probability is not the chance that the whole reading is right. - Networks are also often over-confident (Guo et al. 2017). -- **Temperature scaling:** divide the logits by one temperature T, fitted on the benchmark's - held-out real images. This changes the confidence, not the reading. -- **Reading-level confidence:** use the decoder's alternatives: P(best V) / Σ P(top-k V) (a - normalised margin between the best value and its nearest rivals, typically ±1 on a wheel), not the - mean per wheel. The k-best come from the DP (keep the top few per state) or from the A1 window. -- **A wheel at rest counts by its whole digit**, not by the one bin the common shift puts it at: its - probability is the mass within about ±0.45 of its digit. The **weakest wheel** also counts on its - own, not only through the mean. On oleost1 (§1), with only the exported top three per wheel, the - right readings came out at 0.94–0.99 with their weakest wheel mostly ≥ 0.8. The misreads had a - weakest wheel of 0.14–0.43. "Weakest wheel under about 0.5 → unsure" would have passed nearly all - right readings and caught every misread, including the one that was published (last wheel read - as 2 at 0.44). This is to be measured with the full 100-bin output on the whole benchmark, set - against the value margin above. -- **Pick the default threshold** from the risk–coverage curve (§3). The UI text stays "at least - 70 % sure", but 70 % then really means 70 %. -- **Registry:** T belongs to the model, so a calibrated wheels-v1 is a new entry - (`wheels-v1c`, same file and SHA-256, plus `"temperature"`). Sensors keep their old behaviour until chosen - (registry rule; `confidence: "wheel-mean"` versus `"value-margin"`). - -### A3. The last wheel's fraction - -- The decoder already knows the last wheel is at, say, 9.8. A sensor option "**one more decimal from the last - wheel**" (wheel reader only) publishes V with one decimal more, for a smoother rate entity and - earlier leak detection. -- The settling rule (`readers.settling`) and the "went down" check work on V with a tolerance of a - few tenths, not one whole step. -- Off by default (entity precision and the Energy dashboard change), with a hint in the reading editor. - -## 5. Phase B — better data in - -### B1. The export ("Export readings") — done - -Built in 0.8.0; what it contains is in [`SCOPE.md`](SCOPE.md) §12. Decisions that differ from the -first plan (2026-10-10): -- the light is recorded per reading automatically (greyscale picture, and whether VisionState had - the sensor's light on) instead of a flag in the dialog; -- frames are **not** deduplicated in the app — a reading of the same frame is still a reading; the - training tools remove duplicate images (§B4); -- the unchecked readings, accepted and rejected, are **always** included (no option, no count - limit): the images are what is scarce, and with the rejected ones a sensor's checks can be - replayed in order (`seconds`). Their value is the reader's guess, so the tools use them as labels only after - a check (§B4); -- no spot-check button; `spot_rate` is the way to get accepted readings checked; -- one ZIP of at most 24 MB (GitHub's limit is 25 MB per file), checked readings first, then the - newest unchecked ones; what does not fit is counted in `left_out`. Splitting into parts was built - and dropped as more than we need now. Images are not scaled down. - -Privacy: only the region with a small margin, no camera, name or position, and the day only. The -user looks at the images before sharing (README.txt in the ZIP). - -### B2. Collecting - -- A pinned GitHub Discussion, "Share your meter", that explains why exports help, how they are - used (split by meter; credited in the model card) and that CC0 applies. -- A Discussions form (or an issue template) with the meter type, camera and light. -- Most wanted: gas meters (often white on black), electricity meters with a rolling counter, IR or - night frames, angled cameras and fogged glass. - -### B3. External data (checked 2026-10) +| `dryad5`, `dryad6` | Dryad test set (CC0); a third of the photos is upside down, the right way up is kept | test | +| new shared exports | §6 | **split by meter** (or by date for one meter) | + +**Labels:** + +- A person's answer (checked readings) gives the value. +- Every other reading gets the interval between the trusted readings before and after it in time. + Trusted readings are the checked ones, plus the accepted readings on the longest non-decreasing + chain of checked and accepted values (checked ones weigh most). +- Where a stretch was blocked (a run of "went down"), the frames were looked at and labelled by + eye. + +The export now says where the region is in each image (`"box"`). Before, a margin cut off at the +frame's edge put the cells in the wrong place in 94 % of the maintainer's images. + +**Metrics** per set: + +- right (within ±1 on the last wheel), exact, too high; +- risk–coverage: AURC, and the share accepted at 1 % wrong — for the mean and for the + weakest-wheel confidence; +- for the time series: a **replay** through *went down* and the change limit at threshold 0.70. + It counts published values, wrong ones and the hours a too-high value stood; +- **geometry**: the same frames read with the region's edges moved (half a cell short or wide, a + quarter cell off, 10 % up, down, taller, shorter). + +**Gate for a model or decoder:** over three training runs, no set gets more too-high readings and +none drops by more than the run-to-run spread. The geometry cases must not get worse. AURC must +not rise. + +## 4. wheels-v2: a cell with context + +**Model.** Each wheel's cell plus half a cell on each side (from the picture around the region, +not only the region; beyond the picture the edge is repeated), grey from the darkest channel as +v1, scaled to 64×48 with autocontrast. The CNN is v1's with a wider head, and the output is the +same 100 position bins, so `readers.wheel_decode` and the counter checks stay. The neighbours in +view let it find the wheel nearest the middle when the cell is off, and see a wheel that the +region cuts. A second output gives each cell's embedding (the layer before the bins), so +*learning per sensor* (§8) can later be built in the app without a new model. + +**Data:** + +- drawn rows of five wheels, cut as the app cuts a region that is off by up to 38 % of a cell, + with dividers, neighbours in other colours and printed text above or below the windows; the + photometric steps of `synth.py`; +- Dryad training crops (each photo's orientation as for v1); +- pedromfa #32/#40 and every shared export with labels as in §3, split by meter; +- still to add: red wheels half way (the maintainer's frames of 14–18 Oct are the cases), IR and + night frames, and fogged glass. + +**Confidence:** the weakest wheel (§1, point 3), part of the registry entry +(`"confidence": "weakest"`), with the threshold picked from the risk–coverage curve so that the +UI's "at least 70 % sure" means about that. v1 keeps its mean. + +**App:** + +- The cells are cut from the frame, not from the cropped region. `engine.models` passes the frame + and the region today's code crops, and the new cutting takes the context from the frame. +- New registry fields for the input and context (`cell_width` 64, a context share), and the + confidence kind. `readers.wheel_cells` stays for v1. +- "What the reader saw" shows the cells with their context dimmed. +- The wheel model becomes a per-sensor setting. Today every counter uses + `readers.DEFAULT_WHEEL_READER`. A counter stored without the setting is read as wheels-v1 (a + database step, as for `counter_reader`), a new counter gets v2, and the reading editor offers + the switch (as for the text reader → wheel reader in 0.8.0). The setting goes into sensor + exports too. +- Both models are bundled: v2 as the default (3.7 MB), and v1 because released entries never + change and counters still use it (2.2 MB). Bundled only means the file is in the image. A model + is loaded into memory when the first counter that uses it reads, and released when the last + one is deleted or switched. + +**Before release:** + +- three training runs pass the gate; +- the model is checked against real photos in the tests (registry rule: the drawn e2e counter + and the CC0 test images); +- timed on a real Raspberry Pi 4 (v1 2.3 ms and v2 4.7 ms for 8 wheels on one desktop thread; + the Pi is slower); +- a model card with the data and its licences. + +## 5. Labels from shared exports + +- An export holds the checked readings and all unchecked ones, accepted or rejected, with + `seconds` (`SCOPE.md` §12). From those, every reading gets an interval of values from its + neighbours (§3). Every wheel then gets the interval of positions it passes through, widened a + little for a wheel at rest. A wheel that may have turned more than half way gets no label (zero + loss). +- The intervals come from other frames, so a reading's own misread never teaches the model + (principle 3). The chain can still hold an accepted misread of a step or two. Such a misread + only widens the intervals around it. +- Label check after every training run: a reading whose label the model strongly disagrees with + (very likely value ≠ label, high confidence) is listed for a person to look at. That is how + pedro32/0120 was found (labelled 628596; the wheels show 7–8). The label is corrected or the + image dropped, never silently. +- Deduplicate a still meter's repeats when they dominate a set, and split by meter (or by date for + one meter). + +## 6. Data: the biggest limit + +Four real meters are not enough to measure "a meter the model has not seen" well. On the +maintainer's meter every candidate reads the hard period at 81–89 %, whatever its design. Wanted +most: + +- more meters, each exported after a week or two: gas meters (often white on black), electricity + meters with a rolling counter, red decimal wheels, IR or night frames, angled cameras, fogged + glass; +- sharing already works (*Quality* → *Export readings*, then a GitHub Discussion; `visionstate/DOCS.md` + says how). What is missing is asking for it: a pinned Discussion, "Share your meter", saying + which meters are wanted most and how exports are used (split by meter, credited in the model + card); +- external sets checked so far (2026-10): | Source | Licence | Verdict | |---|---|---| @@ -201,152 +217,64 @@ user looks at the images before sharing (README.txt in the ZIP). | UFPR-AMR, Copel-AMR (Laroca et al.) | no open licence found (given out on request for research) | not usable | | AI-on-the-edge data and models | no licence (data); non-commercial (project) | ideas only, no data or code | -### B4. Data hygiene - -- Deduplicate by frame. Split by meter (or by date for one meter, as for #40). -- **Unchecked readings** (`"answer": "unchecked"`) are the reader's own value, so they are never a - label as they are (principle 3). They become one after a check: a person looks at them, or the - sequence confirms them (a counter only goes up: a value between two confirmed neighbours that - fits the rate). Until the tools do that check, they skip them. -- A label check after every training run (§3). -- A last wheel stopped half way has two right answers. On oleost1 the same frame was marked right - as 596 on some readings and as 597 on others. The tools count either neighbour as right there and - keep such cells as interval labels, as the per-sensor labels in §8 do. -- Collect the half-way red-wheel cells from exports as their own case for wheels-v2 (§7). -- `VisionStateLocal/README.md` lists the source and licence of every set, as it does now. - -## 6. Phase C — geometry: let the app correct itself - -The goal: a region that is drawn a little off, or a camera that moves a little, does not hurt. - -### C1. A more robust model (training, in wheels-v2) - -Drawn cells with perspective, rotation of ±5°, cells of unequal width, a neighbouring wheel -showing at the side and wheels that sit a little off each other at rest. - -### C2. Calibration per sensor (no training) - -- In the background, every so often (for example after 50 readings, and after the region is - changed), the app takes the last K frames of the sensor and tries small changes to how the region - is read: - - vertical offset ±10 %; - - height scale ±10 %; - - rotation ±3°; - - cell borders moved ±10 % one by one, or as a linear stretch (cells of unequal width); - - a rest offset per wheel: where each wheel that is not turning sits relative to its digit, found - over the K frames (on oleost1 it was +0.3 for two wheels and +0.1 for the rest, the same all day). -- It keeps the change with the best total decode score over the K frames. Cost: K × changes × - about 2 ms; a coarse-to-fine search keeps it to a few seconds on a Pi. -- **The result** is stored with the sensor (`reading.calibration`, reset when the region or the - number of digits changes). - - A small change is used straight away and shown on the Live tab. - - A large change is offered: "The digits sit a little lower than the region. Adjust?", with a - before/after image. -- It **replaces the per-frame shift** in time: a fixed offset per sensor gives a stable last digit - and removes most ±1 readings (§1). -- Tests: drawn counters with a known offset, rotation and cell stretch, where the calibration - must find them again; e2e for the suggestion on all three devices. - -### C3. Camera drift - -For a camera that moves a little over days (a bracket that sags, a door being closed): estimate -the shift of the region against a reference frame by phase correlation (numpy FFT, sub-pixel, no -OpenCV) and move the region along. AI-on-the-edge does this with two reference marks the user -picks. Doing it automatically, on the area around the region, needs no user step. It must check -the correlation peak and skip frames where it is weak (light changes, IR switching). -A first real case was oleost1: an ESP32-CAM mount moved on its first day, the region was redrawn -twice, and the digits still sat at different heights in crops of the same size. -A camera that comes back with another picture size is already caught without this (readings held, -"camera changed", [`SCOPE.md`](SCOPE.md) §5); C3 is for a picture that moved at the same size. - -### C4. Perspective correction - -The manual 4-point correction stays planned for strongly angled cameras (see the earlier -decision: measure the read rate at real angles first, for example with the user's ESP32 camera, or -with `fake_camera` given a perspective parameter). C2 can later suggest the four points. Open -questions: reuse the 4-corner polygon region or a new corner tool (existing sensors must keep -working), the corner order and orientation, and dragging corners on a phone. - -## 7. Phase D — wheels-v2 - -- **Drawn cells:** whole counter rows drawn in perspective and then cut like the app does; - monochrome IR; flash reflections on glass; red wheels stopped half way; fog and drops; DIN-like narrow fonts (OFL); the failure - types from phase B. -- **Context:** the cell plus a strip of each neighbour (or a model that reads the whole row and - gives one position per wheel), so carries are seen together. Measure against the per-wheel model. -- **Test-time augmentation:** read 2–3 slightly shifted crops and average them (a few ms). A small - ensemble only if the benchmark shows a clear gain. -- **Dryad position labels:** fit positions for the Dryad crops with v1 (inside each crop's - nearest-digit interval, offline) and train v2 on the sharper labels. This is only for that public - training set, anchored by the exactly known drawn wheels, never for user data. -- **An embedding output** (the layer before the last) as a second ONNX output, for phase E. -- **Limits:** under 5 MB and under 10 ms per reading on a Raspberry Pi 4. Measure on a real Pi - (CI's QEMU check only shows that it runs). -- **Release:** new id `wheels-v2`, the benchmark gate (§3), and a model card with the data and its licences. - -## 8. Phase E — learning per sensor - -- **What already exists:** the review queue asks "read correctly / misread, what was it?", and the - answer is stored with the frame and the region it was read in (`prediction.read_ok`, - `correct_value`, `probs.roi`). The reader does not learn from it today. -- **Design (like object sensors' taught boxes):** - - From every reading checked by hand, the wheels at rest give exact labels (digit d at d.0), and - the last wheel gives a ±0.5 interval. - - Their embeddings (v2's second output) are kept per sensor. - - A new wheel's position probabilities are mixed with a nearest-neighbour vote among that - sensor's own wheels. The weight grows with the number of examples and with how well it did on - the sensor's own checked readings. - - The base model is never changed. The adaptation can be dropped and rebuilt at any time, and - it is part of the sensor's data (backup and export). -- **Safety net:** a leave-one-out check on the sensor's checked readings. The adaptation is used - only when it beats the base model there. Otherwise it is off, and the Quality tab says so - ("adapted from 23 checked readings · 21 → 23 right"). -- **Asking at the right moment:** a reading where the base model and the adaptation disagree, or - the confidence is low (A2), goes to review as "Read as 632.59, not sure — right?". That is the - same queue and the same answers, used only for that sensor. -- **No self-training** on the sensor's own unchecked readings (principle 3). - -## 9. Order, size and dependencies - -| Phase | Builds on | Size | Gives | Status | +## 7. Tried and dropped + +- **Tuning the region by the model's own likelihood** (the region's edges moved until the last K + frames decode most likely). On the maintainer's meter it raised the too-high readings from 9 to + 30. A model is often most sure on a crop it reads wrong, so its likelihood cannot steer the + geometry. The context model makes the geometry matter less instead. +- **Averaging shifted crops** at read time: no gain on most sets, 3–8× the work. +- **A wider model, and drawn off-centre cells without context**: no gain. +- **Weakest-wheel confidence for wheels-v1 on its own:** it works (§1), but it comes with + wheels-v2 instead of as a separate setting for v1 (the maintainer's call, 2026-10-23). + +## 8. Later, when a case asks for it + +- **The last value as a soft expectation** (decoder prior): the window [last value, last + rate × + time] is scored directly; a reading far outside it that the image explains clearly better is + not published but sent to Review. It would have stopped 734.156 (+36 steps in four minutes). + Two risks: a wrong last value that locks the prior in, and a meter that was replaced. Both need + tests that replay sequences. +- **One more decimal from the last wheel's fraction**: a smoother rate, earlier leak detection; + off by default. +- **Camera drift:** a picture that moved at the same size (phase correlation against a reference + frame, numpy FFT). A picture of another size is already caught (readings held, "camera + changed", `SCOPE.md` §5). +- **Four-point perspective correction** for strongly angled cameras: waits for a real angled meter. +- **Learning per sensor** from checked readings (nearest neighbours on an embedding output): the + data so far shows a meter's own frames from another period help little. It waits until shared + data says otherwise. +- **Refuse a frame that is too dark or flat** before reading it, instead of stretching noise into + digits. + +## 9. Order and status + +| Step | Builds on | Size | Gives | Status | |---|---|---|---|---| -| 0 Benchmark | — | small | a fair measure for everything after | planned | -| A1 Expectation | 0 | medium | far fewer too-high and unclear readings | planned | -| A2 Calibration | 0 | small | a threshold that means what it says | planned | -| A3 Extra decimal | — | small | a smoother rate, earlier leak detection | planned | -| B1 Export v2 | — | small–medium | the data we need, labelled properly | **done, 0.8.0** | -| B2 Collecting | B1 | ongoing | more meter types | next (issue forms point to Discussions) | -| C2 Calibration per sensor | 0 | medium | a stable last digit, forgiving regions | planned | -| C3 Camera drift | C2 | small–medium | sensors survive small camera moves | when a real case asks for it | -| C4 4-point perspective | real angle tests | medium | angled cameras | waits for a test with a real angled meter | -| D wheels-v2 | 0, B (data), C1 | large | a better base model | planned | -| E Learning per sensor | D (embedding), A2 | medium | the last step for an unusual meter | planned | - -Update the status column in the same change that builds a phase. - -Suggested order: **0 → A1 + A2 (B2 alongside) → A3 → C2 → D → E**, -with C3 and C4 when real cases ask for them. A and C2 need no new training and help every existing -counter sensor right away. +| Export: where the region is in the image (`"box"`) | — | small | correct cells from every export | **done** | +| Benchmark, labels, replay, geometry cases (§3) | — | small | a fair measure | **done** locally; into `tools/wheelreader` with v2 | +| wheels-v2 data and training (§4) | benchmark | medium | robust to regions that do not fit | three training runs done, gate open | +| wheels-v2 in the app (§4) | training | medium | the model and the weakest-wheel confidence in use | planned | +| Ask for shared meters (§6) | export | ongoing | the data everything else is limited by | next | +| The rest of §8 | real cases | — | — | when a case asks for it | + +Update the status column in the same change that builds a step. ## 10. Risks -- **Prior lock-in (A1):** mitigated by publishing only debounced values, releasing the prior after - N agreeing disagreements, and showing it. It must be tested on sequences with a deliberately wrong start. -- **Calibration on little data (A2):** T fitted on a few hundred readings may not carry over to - other meters. Refit for each new benchmark and report the ECE per set. -- **Overfitting to the few meters we have (D):** split by meter, keep unseen meters in the gate, - and prefer drawn variety over more images of the same meter. -- **Self-calibration fooled by a frame without digits (C2/C3)** (night, a hand in front): use only - frames with a confident reading, and require an improvement over many frames. -- **Per-sensor adaptation that learns a mistake (E):** human labels only, the leave-one-out gate - and an off switch. +- **Overfitting to the few meters we have:** split by meter, keep unseen meters in the gate, + prefer drawn variety over more images of the same meter, and three training runs per decision. +- **Labels from sequences:** an accepted misread on the chain widens intervals around it. Wrong + labels are found by the label check (§5). +- **A slower model on small hardware:** v2 does about twice v1's work; measure on a real Pi 4 before + release. +- **Context from outside the region** can bring in something that moves (a hand, a shadow): drawn + rows and real exports cover it; watch for it in the label check. ## 11. References - Guo, Pleiss, Sun, Weinberger: *On Calibration of Modern Neural Networks*, ICML 2017 — [arXiv:1706.04599](https://arxiv.org/abs/1706.04599). Temperature scaling and ECE. -- Minderer et al.: *Revisiting the Calibration of Modern Neural Networks*, 2021 — - [arXiv:2106.07998](https://arxiv.org/pdf/2106.07998). Calibration under distribution shift. - Geifman, El-Yaniv: *SelectiveNet: A Deep Neural Network with an Integrated Reject Option*, ICML 2019 — [arXiv:1901.09192](https://arxiv.org/abs/1901.09192). Risk–coverage and reject options. - Self-training under distribution shift and confirmation bias: @@ -354,16 +282,13 @@ counter sensor right away. [arXiv:2303.10856](https://arxiv.org/pdf/2303.10856). - Zhao et al.: *A Comprehensive Dataset for Word-Wheel Water Meter Reading Under Challenging Conditions*, Scientific Data 2026 — [PMC13031774](https://pmc.ncbi.nlm.nih.gov/articles/PMC13031774/), - data [doi:10.5061/dryad.7d7wm3860](https://doi.org/10.5061/dryad.7d7wm3860) (CC0). It labels - half-turned wheels but reports no separate results for them (DenseNet 98.15 % overall). + data [doi:10.5061/dryad.7d7wm3860](https://doi.org/10.5061/dryad.7d7wm3860) (CC0). - AI-on-the-edge-device documentation (ideas only): - [parameters](https://jomjol.github.io/AI-on-the-edge-device-docs/Parameters/) (PreValue, - MaxRateValue, ChangeRateThreshold, ExtendedResolution), + [parameters](https://jomjol.github.io/AI-on-the-edge-device-docs/Parameters/), [correction algorithm](https://jomjol.github.io/AI-on-the-edge-device-docs/Correction%20Algorithm/), [alignment with two reference marks](https://jomjol.github.io/AI-on-the-edge-device-docs/Alignment/). - Phase correlation for image shifts: - [scikit-image registration example](https://scikit-image.org/docs/0.25.x/auto_examples/registration/plot_register_translation.html) - (the method; we would write it with numpy FFT). + [scikit-image registration example](https://scikit-image.org/docs/0.25.x/auto_examples/registration/plot_register_translation.html). - Dataset licences: [Dataset Ninja: Water Meters](https://datasetninja.com/water-meters) (CC BY-NC-ND 4.0), [UniDataPro/water-meters](https://huggingface.co/datasets/UniDataPro/water-meters), [Roboflow: water meter digits reading](https://universe.roboflow.com/ai-8rgrs/water-meter-digits-reading) (CC BY 4.0). diff --git a/tools/wheelreader/common.py b/tools/wheelreader/common.py index 670c973..f5eeac9 100644 --- a/tools/wheelreader/common.py +++ b/tools/wheelreader/common.py @@ -28,7 +28,11 @@ def split_cells(region: Image.Image, digits: int) -> list[Image.Image]: MARGIN = 0.15 # export margin around the region (settings.READING["export_margin"]) -def export_region(image: Image.Image) -> Image.Image: +def export_region(image: Image.Image, box: list[int] | None = None) -> Image.Image: + """The region read, out of an exported image: ``box`` from readings.json (format 4), or the + middle without the margin (older exports; wrong where the margin was cut off at the frame's edge).""" + if box: + return image.crop(tuple(box)) w = image.width / (1 + 2 * MARGIN) h = image.height / (1 + 2 * MARGIN) x = (image.width - w) / 2 diff --git a/tools/wheelreader/items.py b/tools/wheelreader/items.py index 3495ecc..39de451 100644 --- a/tools/wheelreader/items.py +++ b/tools/wheelreader/items.py @@ -43,7 +43,7 @@ def pedromfa(folder: str, only: set[int] | None = None): f"{folder}/{i:04d}", 7, float(round(v * 1000)), - export_region(Image.open(LOCAL / folder / r["file"]).convert("RGB")), + export_region(Image.open(LOCAL / folder / r["file"]).convert("RGB"), r.get("box")), ) diff --git a/visionstate/backend/tests/test_reading.py b/visionstate/backend/tests/test_reading.py index 39ad097..e4db644 100644 --- a/visionstate/backend/tests/test_reading.py +++ b/visionstate/backend/tests/test_reading.py @@ -829,6 +829,10 @@ def view(): box = counter_box(7) assert crop.width < frame.width * box["w"] * 1.4 and crop.height < frame.height * box["h"] * 1.4 assert entry["region"] == "as read" + # "box" is the region inside the image: the margin around it on every side. + left, top, right, bottom = entry["box"] + assert 0 < left < right < crop.width and 0 < top < bottom < crop.height + assert abs((right - left) - frame.width * box["w"]) <= 1 and abs((bottom - top) - frame.height * box["h"]) <= 1 # Moving the region later does not change how earlier readings are cut: they keep theirs. assert item["probs"]["roi"] == view()["roi"] @@ -851,6 +855,7 @@ def export_entry(): row.probs = {k: v for k, v in row.probs.items() if k != "roi"} old, size = export_entry() assert old["region"] == "current" and size[0] < crop.size[0] + assert old["box"][0] == 0 # the region starts at the frame's edge: no margin on the left assert client.patch(f"/api/v1/sensors/{sid}", json={"roi": box}).status_code == 200 preview = {"source_type": "http", "source": "http://fake", "roi": counter_box(7)} @@ -1026,7 +1031,7 @@ def export(meter: str = "") -> dict: return manifest manifest = export() - assert manifest["format"] == 3 and manifest["meter"] is None and manifest["left_out"] == 0 + assert manifest["format"] == 4 and manifest["meter"] is None and manifest["left_out"] == 0 assert manifest["reader"] == readers.DEFAULT_WHEEL_READER assert manifest["settings"]["counter_reader"] == "wheels" # The checked reading first, then the accepted one nobody checked, marked as such. diff --git a/visionstate/backend/visionstate/bundle.py b/visionstate/backend/visionstate/bundle.py index 0949342..82deec5 100644 --- a/visionstate/backend/visionstate/bundle.py +++ b/visionstate/backend/visionstate/bundle.py @@ -107,8 +107,8 @@ def export_sensor(db: Database, storage: Storage, sensor_id: int, target: Path, return f"visionstate-{sensor.slug}.zip" -READINGS_FORMAT = 3 # readings.json; 2 added per-reading reader details, "meter", unchecked readings; -# 3: unchecked rejected readings too, the camera picture's size +READINGS_FORMAT = 4 # readings.json; 2 added per-reading reader details, "meter", unchecked readings; +# 3: unchecked rejected readings too, the camera picture's size; 4: where the region is in the image READINGS_README = """Readings from a VisionState reading sensor ========================================== @@ -144,7 +144,9 @@ def export_sensor(db: Database, storage: Storage, sensor_id: int, target: Path, readings.json ("meter"). Look through the images first; share only what you are happy to make public. By sharing it you release the images and data as public domain (see LICENSE.txt). -Each image is cut with the region the reading was made in ("region": "as read"). Readings +Each image is cut with the region the reading was made in ("region": "as read"), with a margin +that is cut off where the region comes near the edge of the camera picture: "box" is the region +inside the image, in pixels (left, top, right, bottom). Readings from before VisionState 0.8.0 (beta 0.7.1b2) are cut with the region the sensor has now ("region": "current"): if the region was moved since, those images may not show the display at all. """ @@ -201,8 +203,13 @@ def export_readings(db: Database, storage: Storage, sensor_id: int, target: Path frame = imaging.decode(path.read_bytes()) # Cut with the region it was read in; rows from before that was kept use today's region. region_then = "roi" in details - box = imaging.region_box(details["roi"] if region_then else current, READING["export_margin"]) + roi = details["roi"] if region_then else current + box = imaging.region_box(roi, READING["export_margin"]) image = imaging.encode_jpeg(imaging.crop_box(frame, box)) + # Where the region read sits in the image: the margin is cut off at the frame's edges. + width, height = frame.size + read, scale = imaging.region_box(roi), (width, height, width, height) + inside = [int(read[i] * scale[i]) - int(box[i % 2] * scale[i]) for i in range(4)] # Room for readings.json still to come: it compresses to well under a fifth. if items and file.tell() + len(image) + listed // 4 + 100_000 > budget: left_out += 1 @@ -221,6 +228,7 @@ def export_readings(db: Database, storage: Storage, sensor_id: int, target: Path "date": row.created_at.date().isoformat(), # the day only "seconds": round((row.created_at - oldest).total_seconds()) if oldest else 0, "region": "as read" if region_then else "current", + "box": inside, "reader": details.get("reader"), "app": details.get("app"), "greyscale": imaging.is_night(frame), From cfdc2aa40f76d1900ac9aa6f18a2b3ed9e4bbb3f Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:40:43 +0200 Subject: [PATCH 03/18] Counters name their wheel reader; ready for more than one Each counter stores the wheel reader it reads with (new counters get the registry's default; database version 16 keeps wheels-v1 for the counters there are). The runtime loads a wheel reader per id on first use and lets go of one no counter reads with any more. WheelReader can feed a model with a second input (each cell with its neighbours, cut from the frame) and take the weakest wheel as its confidence; a registry mistake fails at start-up. A sensor exported by a newer version with a wheel reader this one lacks is imported with the default. With one wheel reader the UI is as before; only wheels-v1 is in the registry. Co-Authored-By: Claude Opus 5.5 --- visionstate/DOCS.md | 5 +- .../backups/backup-format1-schema16.zip | Bin 0 -> 20953 bytes visionstate/backend/tests/test_reading.py | 165 +++++++++++++++++- visionstate/backend/visionstate/api/common.py | 12 +- .../backend/visionstate/api/imports.py | 8 +- visionstate/backend/visionstate/api/system.py | 12 +- visionstate/backend/visionstate/bundle.py | 4 +- visionstate/backend/visionstate/db.py | 23 ++- .../backend/visionstate/engine/base.py | 3 +- .../backend/visionstate/engine/models.py | 42 +++-- .../backend/visionstate/engine/runtime.py | 7 +- visionstate/backend/visionstate/imaging.py | 11 +- visionstate/backend/visionstate/readers.json | 2 +- visionstate/backend/visionstate/readers.py | 135 ++++++++++++-- visionstate/backend/visionstate/settings.py | 7 + .../src/lib/components/ModelCompare.svelte | 17 +- .../src/lib/components/ReadingEditor.svelte | 41 ++++- visionstate/frontend/src/lib/types.ts | 12 +- .../frontend/src/pages/Settings.svelte | 4 +- .../src/pages/sensor/ReadingLiveTab.svelte | 14 +- 20 files changed, 444 insertions(+), 80 deletions(-) create mode 100644 visionstate/backend/tests/assets/backups/backup-format1-schema16.zip diff --git a/visionstate/DOCS.md b/visionstate/DOCS.md index db3353e..f5140b4 100644 --- a/visionstate/DOCS.md +++ b/visionstate/DOCS.md @@ -290,7 +290,8 @@ First choose **what it looks like**: - *Wheel reader* (default) — a small model made for VisionState that reads how far each wheel has turned, so a wheel half way between two digits is read as such. It reads the wheels together, the way they turn: a wheel only moves on while the wheel to its right goes from 9 - to 0, so a reading where they do not fit together is never chosen. + to 0, so a reading where they do not fit together is never chosen. A counter keeps the wheel + reader it was set up with; should a better one come, its Live tab suggests it. - *Text reader* — reads the wheels as text with the number reader (Settings → AI model). Only the middle of each field is read, so the dividers between the wheels are never mistaken for digits, and a reading with another number of digits than the counter has is rejected. @@ -322,7 +323,7 @@ First choose **what it looks like**: you can also leave it out of the region and count one digit and one decimal less. The wheel reader reads mechanical counters better than the text reader on every meter it was tried on. New counters use it; counters set up before it came keep the text reader until you - choose *Wheel reader* on their Settings tab (their Live tab suggests it). + choose a wheel reader on their Settings tab (their Live tab suggests it). - **Safety net**: a reading is rejected — and the last value kept — when the reader is less sure than the minimum (default 70 %), finds no number, a counter reads lower than before, a mechanical counter is read with the wrong number of digits, a counter goes up faster than its diff --git a/visionstate/backend/tests/assets/backups/backup-format1-schema16.zip b/visionstate/backend/tests/assets/backups/backup-format1-schema16.zip new file mode 100644 index 0000000000000000000000000000000000000000..8d084e1b8e9098d6e5917fe692dc4865c9ba0113 GIT binary patch literal 20953 zcmeI42UHW=yXXfI8(li0Q4vv4DWOWqu>hh1Dgp{h3{^lt=^`*kkSd6PigX180Wmgu ziApb01f*AK(mRACleh64EaM)71{7!#+n(13@&aAc!8kYk)Jfu)n;|+}7I4K1fZyK$OLEUbd)IZY+rX71G8S+VcgE=XTIlRmywIpEOS@yi#&PgTDyn!2w6 z;c8EmQ~sp6wyN*k{cfn|LF!Sp59z4KeLg|KT(8aEpSv*TvE|d|&>0N(*la-d{Wt`J z+z495?Kh+1_s8Q`qo!}K8f#QBn>jB^d$>G^kLjGVd5+z;UNbTuI`Q3{X=|$p<5|IE zy{3Vfz>|ZSt(moztsTzJc%PBMXq4V1zQ(I1rVgRk9v;5M7uK=MKc%}o`EXdQiOIMP zqp?7Mk&*D~fE(6N@@OiI{oNwS; zs}hDLulBPPyr|4@ipJfSDG=>6T2sjFrOM!W)(giZCdX#V36pzQyWS`HdokO_s0Go+ z@Lf_vc2M-EqI^w*)yW^5EGjj^q`}|RL!-H&EBD0qdbHUP8pRc2A8)RnEAUIPO4mDV zmc(neyXW$tZ;owU(nuu(so-ODa@vve4HRmdr&d<_&S{IZ>0{y=h6^DfJat|8J@~TF zJ}tMa#sOG;tgddCAlkAgGyRdjs=aStOVtUXfF}`q4U!^mP^~Oy-_oZwgq(UVfpUK5 zQ0ZkhTQ_}5rrD08;*x{{LM|^4KBvbdusBPZOS^md$rwumzgMg?xu_*P*P%~I_6`!& zAT07h8$YenH%wNoyPtR^R9m}GI5gC(t;`KQK-3}z=L(5nZ{sHqJfW^4yr{(93GI~( zBb>-_c<69)%ez>SnoS#x^1qE zJxL$NH2-+q5T0y(F1WTp-5!qz3(XFRaES#-+Ta%wU@@aMUi0nK5d8%&t|ut?S*SE?+4LCsC`_-%E}v@<_y?5=V-3b`9&NCHKNfC_IW^49*t7ej zjal$L*5#1`R#*St@L)uvFSQe@C;dN)~rXO*I z4c>0_QD+;s-DVkU#SlN*9NogZI*u;u^`@Z@aX}`;*XrtSXD<24@a=aRzgXaEM;esPhXVzCR2Pk_S-q?<3=wZd)pzTz;8{4!g zuf)SahN)ITAiHiYzrE97o8|qM5VQ8tc+F;hl_peKzNcMYI{fa&9)6$Jms;{OnVnaB zdzwt*3mqfb+9Vk9JZBSkPGbG+iVDSs`?=}HO}f*S+mDD=^EO<%z*oo}&^V&!@lYcp z>Zv_ODzEeG@kcM3_gov#`eXc+=3=PbnPCa@>Z8?`w^Q4ithR`+J(+w%(>}*o&ZC#r z(5&_8r)x=%4Nx~<=lY&@aJbj5$W?mNDcVdj8&&qK(*1}~S`;0N9VPn1p|m+{WK1L` z>bQx;OT)zZ1FC*ad8(G!TckYVEuMvIupjR^HXNoiD)*Mw$e|=F=IPPdK4Fp63J)(# z#1WI~%{+}0WB%tD6a~lbo*^__pBoeMHW$wmIM*uVPH&knNieXSx#MP~md~?PKCNILOoBVsMhEbWNdJ zY!hP-1GBju%g9<@U8>rnU6y9*4qBTXZf zI|~w;7y`-?(8os;hFDdacWk+$pOBFYx1Qf9t$qv9yr9VxtT43Ij|G_G9 zIJ597QlF#=H))@rPZ;Hvm{1u&;>Or=qT&)=vR=BVXUN%cZOwQ+z))wIQJ+dO*px8P zUo~aSyEQv2DgB}-HG9XAM|H86b<3v@5}%w_h`{gTObNcjm7!5+az8O9I!d1A6oMsTSnI^^~x)Is8|1dm+ef~y@d8E z`;4tNrdO*!-EJ(YKfQQy4Lvriae$;rG$9S7tfohtC6>k%n&-t*tFXZ%x`ZKo5%Hsi-tdght z6L{?r>_+;&U>K>ki4kv0WY%5%WmmUO)gn`IJIAduRH|eF#hGC7<+DE(rX@DTyvvDJ zgUM8fr)KY4_ZFSV7?fr|jb*N)!X=jpt=E>@yTj^%`}yJeqRAaaDL29 zXUV4cYs;AqIOA8wCo3LNtCxPvpB|aLuOBRQs&BVck&%gsNu5K)8DUF<%^^pcOOM%6 zFk*S7m*ID*>aY}LZl;H8*YZTX4&}3UH*A^>JH9_ky^#sEFTvoEdD2#H^B`{J=O^w7 z=?3|D=IV1y>88cr9hwmts3jADu0N%Itiu>)5_+OzkY>dQuI1UB@D%&#&7CfFO=7mt*$eQ^sdq|eWAeeGZOJ&@ zfTL;Q;Qm(n)MnlBDA$o>pQgK;HR-`X(F^Spst<+wg!k9_AAPv}17lfd(+PuDj|;rl zDisxTHnb0LvtPJ)@cD_#V~2$r>FM4)W=|8}%}ys7BVB*6L8MEg`OxtMDKvW5g-eK% z^D_-9v*dyIGtQomwUQ2QI9#D~%=qCS{?2*&cO?!@WvIH3Q%M{J6m!Dtn&q?pbL{|+ z)Vr$ZBQFn;Q0_8eSy9uM^vuEoOvd(zI1s4g{9dGsgyApu%%3WBp%(_k4{ciYhDh~h znD6En24?Et8>#%Elb>vOS$rJ@>$L6zm2e>wJL1^B{4q4~N9D_q%H!F^;tU~kK63qf zbSYZT!@-WG++&p<9Kh!NYjg2!);AFhW5`hq$uDRyFjm{UE;wAI|JWnWd2z zSM44y6sA34x@7K;O3N)v&F9%Y$_=F z6*E4u4)U$c!cViCtymvDGLL`cQ}9r5tV%8M1sXlV9C$7H!*+YEd8#d!c3(?)CBC6b zq|D`f15f{C=z-rh-T&66YsXwqZab}z$Ft$MXFcoDmh=45_1c2*w;ph@-t@~75z=Z@ z&9SJW97#2l@JT#e_DEG%*~s+jlT|tU!^*c{sycGEY`j05)G+)=bJ}Qle&GL}WbT;4fd(%scGVvN_c_Tg|{< zv0<7$;lQ4#!%fbu&(mHT-teX~wJZE|YlpLD8t;x5jxxPos#d4EPl;e8co{dU+7ZNk zR9sk@T)4FS46J9iFt>Ee9G<^3sE+cl?|x{Tfiu@gu+IolHxA^(!8=Ht?`K;ONwydM zTdk_KCu=^sl?eD5Mg(o_yrO=&P9oRDC@Q=V-u<3~t>j6BA$x9wXDhEg{~LLW(=V@I zVKJ1@e`(F=rMPJL*s!f4Zd2{4q)Sp29oP2N$c^4X1U7%zAJFphz3nC=Vd?Y;@5%#r zgt||+TQyg0y;^vtCEa3CUnse$WV@*K_PbG0#u?RgQ`@6Wbpys!#P_`18hq5!lxFpL zLgMU+R0`LC;DCl;NOQ|b(~#+*6rR;yoaTdC|JQ{ch7c{StW9l`4&cJoWa$Ej7P6FpOUs#x>vJ#K2FEM zZk^7#X_5!iw5W;l?D5n+=^=ORDL1%Xia+Vl*5qAJn&}kx-sm0ni4fKi?%e$vbG#}6 zjzDtiW8$fMu5Oo7MfJ2$32p}3W8Q^;>o_&2c-s25I3w?( zi6dY33zw6KA(F2$=+EDDY11XCwEXCDMd=bNQ|BxuPE&r{vq?l`+-KNH* zc8^wXGg`}Stjx&4*sbj`uT}JRuYzVdm6#*Ofm@8n)rbbef-~094C{q=B$DpEV@~h> z)YN~Av=MvcK4t%&iM&-@3szP0_`}Z3-a6aQ>+$mIPg;~caj}bZ^4`%(i29^e3pO6p@bZhU3Si$Y z1M*LN@gnR_`mzL9*wx5Pumz!0^& z#6U$*Jngv22wTww3RS9rA~zq{)=FwOS(e$AQh_S87-4FK|46p<T8H~9i$6AprzRktzxC2Wu>7O zLf{snp`-b{fsudSXjXx9T(frF`VI78hcp&w6%8%zs@1ehXASmt1)oE!S?M-z6Fabm z?IdpP_ABh-*MlCe<3ITP4ace4i5+MI8~61aIJvlacm;L}?%FLRAt@y-BfJ05VI^gi zBdSL=PitxGoH?s&Xk=_+deO|>*3RC+(dnwQ#|=*}Zy#U3;E+3a?}gqE3y+P9f0U5; zI4SwX%U5aX8JStxZwm^Gic3n%%HP-3H+*PpYHsQ7`PkdnKQQ=dXmV$qAV23i_$@n~5g4Ak{F)^Aqh z*@j)QfBtft!!slAs8DM*Q$!N{R;kzZMT=aZ@Q7Tx^@*nH`)L2!BBxZ>q%LM63t5^9 zU1+tYu%{96e5TwZG0e&dYZCeIggXn?y%R&1VY=@ThD@nY88gzVLKbhN9H&CWZQ#%r z9T)K>RH*S16++zMsUz^z_N=bXi3;p2nhN!LQz6SZgvSC&fXTP1&}iDY#7QIv9A}1& z3I!jcLP7M=j8uprjS6XEkR$M1%eR8`rHMMQQ8hMl6xwzY+Cws*klhqpu?V?nu2fiJa zr^K$)wc3;Yec$}=idJK0z!Ih(QK4a%nO60)B;m@Y77@vr0APb#@3 zi^lrxg1&9|Gb7yj`*0^g)CrH&a9B(9IB2giYqoK#|mK%;p> zwg;1|lM1d5GX=HTB?PPIvhtnUd_K4|0Z(y(7msf^0ERIX_rpvq4!qeSJ<*rwKC@1+ zaVFE`rbdP;fAE~75sue=-QL>=MzQ=SvAYZTFvom$Yd+GB^XAXLH$22Ys$ULYY4|PN4KMOY1d^FE@B_BKcwOrL?^LWBnR}Mxy77$C|>{zx4=tEjWR{ zzbiJMw6*k8FZWrTnOd2w?0I55;Q+X}*rPN`>?i(BTT5;Jo;maGQ$oUkH%slGn1eQw zk(Dt*8v(#bNf;X#%8Hrb#0*8n&{8-_V*?}E-^}^VoPUJr{~hK`3W4v$YvRPW+UdXg zKVEl1b%G7yjPV+?w;t^rsopu7>Jsyu*!-5?s`@^mvskPp_g;P)B^c8iX^x%a4531Y z3$W{pv!X?Imkap4(YR-RQs;%U!i-tm`-;K0d2UDjhAR81Q1>l($chS;18`WyA>0E< z!Z=6$UhE z$|v4k-%9dmfyNx0tyNMNu9{BlGzF(iAJc5J*BF$arZaE&oJJMe+Ioxz1rcAb{G<3(*eH-F z$Qw%Wj0bQLZS3E|50`pjdjk8f@o13BXH3&;l{m<;u8;qQ0Ho7D>K3&0K&Qu>_Htruhckdun_yJG#Ir6xS>Ksb^D9^57p5kq(kg><-70r>1g;0Iu^B;xcB zdK9aHR3IWXE|{)OBv(znx>v1U^}Rs`IXSlkg$#Vw(D=xd@+%hSUx>}O#gv&1EC{-% zNVB&%>Z|B)Vwq@Ah}BU<06W6EnHPCyG|1e5`b8(H5Iy=WYDRJ}Wz`Vy)tC%h?)&VmR74(y-OwRF{e&Uye+ zg}Ihu(#_%Rza}mY*=E9wxEs%JMVgbyJ|q&tX?g=QXiZvg!U(0>>}Q^^28{)0c5rvHT#q!7{# z?B;P`Q8wPVBw4+1M6qBFkDf;vOiPUhg^XYIkZp~Q8h!{g{ZXLlQ-G%Lq@?FD&jB|h zSmgoG^d-n0Reeuvr7VHRH1k{qn|GYgE1Yb67B2Rb*(HDz(`zsU=m$BJecq_7q!WB_KL)(YF4}6koj#d;qIMh zoK>Z*D;ISAcHxIYihee^ilvkoa1ud-n7K(R^k}2bL!D@(nhH%FnL&}9sL*>~;hL0% z3=DAKiqS|uFvs7A%}5ZuQHOy`=2|v&(1d@e&&M~(F?%*-$5#* zLUZ)l0Ap(Bi7=@T1!!`(qTBQ>@M7T0F*4u?(8}f-L;|wP_-o+Nk5uUB4*Ws9!VhOS zX$7t!zrQJE$)9+H9yEiQ6n%iwoKX8rxvX&?!jwj?25@w9@D%ojnns05E<|w080GAq>Waiz}#j zIINdnNL~5B+R3NG2&c#mfE&|V!US%~L&4IY#0GuL-T(IEbFC|<3RU+9-$^x2L|f0w z#ofGP5N8>f4R;$XXdqs$n*fejKRZrE6EPHqMQMa#`x1@1&4q6w+ird4%J%%MeI@Of ziRUh0h7TzQP$7?X!0W+$e+aFKG~hRhy`e%i$Mbtp=Rys~ViwNi1KvH1QKmv~ZeYn; zIH&oNe4gA1X`j?lvQm7te70q|3cF)1T8{fZ1YHW&Fkh(hYw`BVJj#L7GT1@Xy|(XG zk5iVU1+ZDL&m(^T`Y$+3KR8J0dRq=>0X8kQf8vv&k*I;Jk%5d9&csAQOiI*18ZBxh z13V?PsDy#2f$?uX{pQnum`}tZphNR8(m&IoKYLF>t}F44#=<#FD`rv_DMhNL;J(J{ zY58h2C9PPV;4;vXhe5a#kwg*lBcLURFa|NiNWy3Z6|!ioOC#46^Wmb6&-U}L^(eHd z3)~yJXET0XM#{}=8TAx;q4H-=k$gf8p1tD@^A>T61^{up0pdXC-SCn#V-T|#Ntg=# z?98+Xe}sFsUJ;?t!M%4tD3A67QUG=w==>-${onLlGKjAi5@t96W;6jLXi3Z>R44}a zCsYz|es}WSe(;bDaNDrJZNn#Cj6=4s-3Szs4x$Q_@NMQ1c>>~!h*Kf-v|-VHK{CI- zV8cwH+$g4bK1BCws}wO>?{2$)q$y9anu(6aih)=Xw z=qzL=?!JPZ8$cqiFBSvGaQh;Rq3m9Y{X|C(F2~h~%`4C06S5mFQ=4?8c!^+UY#YU} zoo5|McbO+trc`=l0zYarAUF_OdvM_goXL^csZeDfm=`QbKR*h8pYZ*Kw(eO=zLSY3 z0l!l1HN}u0aMl$QgWP!wZntU;yXE1LM;5?sA|PGVImev~fLeX7$c_rJ6M7RCa*zV9 zm_^y)`}pxEIh1nm1%4v`B%jJ;;t&-A5l7(~(Ute7qAw*#UNd&V0@+wS>Z)IGbQ&q^ zb4~BtHZ3+_YSn}G_y1V0Q;VH=mCFNw;{aSD4;gqNnLH9p*i|jB_;w)7^Z46GSCy(- zcBICih7Wyuc6Idl=5;mxY{1aEnY9QoMStu~%#8F!%mUgQIAw7NXQ&5e(5eQWFlAoi zW7HiTPvG^$(E+nblN8j7ywnCamc_W~U<`S$$Qrn&9oT1GRj8%#psqdxU|7xpz_3UR zS;Z}mlwmY}hZS7?} zH_EOvn(2IuS(#mvAara@TT#(wAR=N2TxY94mOb!)@l3PQT@$ufwlD=KTxvhZp}2^Y ztQ1I3f!Lstk)(vQxQv0Mv6uwfP*xOeAS+`aW%3(`zj62<#^JOOaM%8vJ-9Nk^bfBY znVCLeT9K`F64lbBB+Gj34y*Q7>CE#@rB&!iBb`htQ~;DlecfLkmcGKoGy-%p_}(|E zH&P+xRM#SGz+@wf$yuqLp7MefF?wLC+ZA zdue?3y5K&tCo0f(9jo==V!1p=R7B83XQAoN&J66eOHK}aq&LJCfAt^zo?J1{%LJc^@2 z;u(}Yc$Am8NOCTMm^rpIK&TY#6`cYCGo}}wsHWT)g=fqg+qB5DJ&vUQO=OX2xo4y$ zI8E9F7mvB$;>yGqJ`cU@t!^V*d7*Y znKh(n6$DAfYB;(j)I9Eb5;V_6-h?oHJmEnTa8phwf^=8@=GedmFB#0@`p`CyVdm$*B;{Mj z?u={T6-q*`(j#zfCx9<#V(#D^cy}%aJrcTpUnTKO)%z=vsM1DnkSL4_F}_{)OkzT! zxA&o|{$e`#PDI-bID$FLi(R$iM*5Z4%q!t_>)|mOJ6Z@@Y7!tB|5sgx3C{5E@(O7K zoP?-24lRk6l9DjONgIfoNEykXC5)vF0n2bw|G$9i59p8I#TB26M*hDqu6U%xuB|e& zls*4hqp?-4Bc#pV+=N1xlTJ^*>9=gpu%(H-5^0jBgYb+aKobxFs9J^JEt$srUciXO zPQZgj_~ewSr(#vp49R)i7lv$-c6J20y3{G!N)&x4P#3zBYTxXZSn_JA2Edj3Z_zZ* zyQ+zN5+aMhx(0~b>#BtnT0kQzjPwYh3+T_ASS_9Ij;&NE?*uTX$<;vs8otbjKDO+N zC0tR=T-KUR6Q<-_yj&C*1b*T;^ONy+T~?mi#5fDw@fh|*OiKsgLhpwwzs1i@<+LvV z^K+?vH9;?$+1gp#T>ZOD3MX!0Xd+{blNFaS1}ST_tdz80gQAWl9{ZeW9_3i*Bp+rH0Qo~aIT+ZV zTU|}(wt#?$A{F9+!=|~&Vqf;I|L!L8s+!}K_3 zT^WfhBxBcTj-nxK%U$9;*+OEQ0|rsBaRS_AAVOhNaB;-eo?`_CRQ}O2Pwa^OeFE16 zVp|UflbPMCMC+erCaU|rcrm`XWInn$eGdG88~?>7Y7uXO2VmlLl3#-~v<9WvwUeF- zRg9AyfxQc+pAa3Gy=<__3;rP7z^$os>iDZ_5`z+m(VOCL4KH>UfW*Q_MApr;k9-Ir zQ&^VL=wZ{#wkyr@3H-km9t-~@RFco-C!_Aoehp=*RnR=GTP8PcGY?W#qsGmVQ)6yN z65m;p@yM^mJFs9Dzr;#LsL+!J!axhD`Iq7>;GS9TdK>=VcAPaG)%%EHUbxxa5oH-a zoNbkOIsA!@gp^}YW%pt9s>0oZJX=lMbjs~cVjo`yu|Mw#VcMfsW>RBh3Ug$Mp}?)w z(fB7zhI_pHMC^)Mxc}xg?MiAnGv3%71c41=8$K~5n@-@A;zq~wnYW96(yBii|*Yk0v^C&?i2jkD08@ZBx(PZBy~p0;(QUQa)XSM#W*>bye6H`P>_Z_BLk z@fY#EowM^GV5UR2pMR3A=%46oEj%wBt7gRRCPylv-hr)T%6!3gIsafuC zN&Lu8D%3(~+2TmfxvGc_eMw2LawOT|(^LA#TQ?!bD3S~KoMV>EU)21;Eb7EtgW+k- zP|3zUoHZf3j}x=ctMfUmWmaL{{Z`7n9cvFtxj?jj=IRt~86eBg=LHKe#9auW{Ij$~ zfHkP~v zX#KuC%q|ARz$rt3+>7l=Vc#l*ucCFY2@_ z1&nJVXz@#ETIXG7x-_{QGe-o`9pO*B#GEDUhN&Q(m^n{ZDntX9sd$2O1NF6SfoL%6 zb}<#xD(~t+f*y<7uso*aXAZ)|T>mB!l#_W+&uRCL&FU?x0(aeGcimMr0P&S7wF@WD z=)|YIkJ>p}8BS{O^UpGR)bv9=lnQxI^*XqwF&6GB1_AzjnpgtYHSqa&@lS zN0tU37bBbetDOAQL;Q4>>>W+{3;zw)zAFa@`~HAAC%FfHJ^*n`?dO<7i{NBsq>RNS z4AEe{iUiJBLRK6tV+aa~Wh7+8OpL^TW9~QR{?nNIACz+~c?SDY-Uvt>-xx0Xz>8`@ zKG$<0*szRCm#B~9pKNUuc#Kh*V{2Qe<^X=d2)`Ikn2G|SoQvPrhvj?55S)5ZV!=umVI%=hP6X>%7CHSejZe1$zb}Ryj3WEE>V{fZ5{JLp zS zl>-XH0eB-yGN$*zr=_YY5Q0{Ir!S3te@@|M>aFr;3z3Jj5_h%d2sjQt3EckVw8?ug zpZPvNL1PInoX?_Vi5l!a+)q_2NvCy;+L)j$h)P7BH6i`lJtp;G{Ie1aZd@>s;=*AFP zyWxVxmv8W7RXRX9-5*lq&RoEy0Ud1*01gFG1cUQS@c^KJxwyzFH~ zZI;VkC6^NNAwCpkb;X!+=TGamP(P81>EZ7BtJInK>9Oa(BnP#evRKCTXz13> zcJei~nPtqi+@Mb=4%qvoLda{{SVlmV=MCRAzbPeoXke>udzU#z-P%E|>vU!`R8ET8E2QhsH5(n={StIoelk=y!n zDZf&KzEaA{+QF| 0.8 assert used.width > used.height # the cells side by side +class ContextSession: + """A model with the second input of a wheel reader with context, made from one without it: it + reads the cells alone and keeps the shapes it was fed (no such model is in the registry).""" + + def __init__(self, inner): + self.inner, self.fed = inner, [] + + def get_inputs(self): + return [*self.inner.get_inputs(), SimpleNamespace(name="context")] + + def get_outputs(self): + return self.inner.get_outputs() + + def run(self, names, feed): + self.fed.append({k: v.shape for k, v in feed.items()}) + cells = self.inner.get_inputs()[0].name + return self.inner.run(names, {cells: feed[cells]}) + + +@requires_reader +def test_a_wheel_reader_with_context_is_given_the_wheels_around_each_cell(monkeypatch): + v1 = readers.WHEEL_READERS["wheels-v1"] + spec = dataclasses.replace(v1, context=0.5, context_width=64) + with pytest.raises(ValueError, match="takes 1 inputs"): # a model without the second input + readers.WheelReader(spec, MODEL_DIR / v1.filename) + real = readers.cpu_session + monkeypatch.setattr(readers, "cpu_session", lambda path: ContextSession(real(path))) + reader = readers.WheelReader(spec, MODEL_DIR / v1.filename) + frame = render_counter(89932.0) + box = imaging.pixel_box(frame, imaging.region_box(counter_box(7))) + result, used = reader.read_counter(frame, 7, box) + assert result.text == "0089932" and result.score > 0.8 + assert sorted(reader.session.fed[-1].values()) == [(7, 1, 48, 32), (7, 1, 48, 64)] + # What the reader saw: the cells with their neighbours side by side, at twice the size. + assert used.size == ((7 * (64 + 2) - 2) * 2, 48 * 2) + # The cells are the region's, cut out or not; their neighbours come from the frame around it. + region = frame.crop(box) + (cut, cut_context), (cells, context) = reader.inputs(region, 7), reader.inputs(frame, 7, box) + assert np.array_equal(cut, cells) + assert not np.array_equal(cut_context[-1], context[-1]) and np.array_equal(cut_context[3], context[3]) + # A region at the picture's edge: beyond it, the edge is repeated. + for edge in ((0, box[1], box[2], box[3]), (box[0], box[1], frame.width, box[3])): + assert reader.inputs(frame, 7, edge)[1].shape == (7, 48, 64) + + +def test_a_wheel_reader_registry_mistake_fails_at_once(): + v1 = readers.WHEEL_READERS["wheels-v1"] + with pytest.raises(ValueError, match="confidence"): + dataclasses.replace(v1, confidence="best") + with pytest.raises(ValueError, match="context"): + dataclasses.replace(v1, context=0.5) + + +def test_the_weakest_wheel_is_the_confidence_of_a_wheel_reader_that_says_so(): + """One unclear wheel among clear ones: the mean hides it, the weakest wheel does not.""" + spec = readers.WHEEL_READERS["wheels-v1"] + weakest = dataclasses.replace(spec, confidence="weakest") + probs = np.full((4, 100), 1e-6) + for k, b in enumerate([10, 20, 30]): + probs[k, b] = 1.0 + probs[3, 40], probs[3, 75] = 0.5, 0.5 # the last wheel: 4.0 or 7.5 + probs /= probs.sum(axis=1, keepdims=True) + at, shift = [10, 20, 30, 40], 0 + reader = object.__new__(readers.WheelReader) + reader.spec = weakest + assert reader.confidence(probs, 0.0, at, shift) == pytest.approx(0.5, abs=0.01) + reader.spec = spec + assert reader.confidence(probs, 4 * math.log(0.84), at, shift) == pytest.approx(0.84) + + @requires_reader @pytest.mark.parametrize( ("asset", "expected"), @@ -340,9 +415,10 @@ def test_wheel_reader_reads_drawn_counters(value, expected, taller): ("counter_turning.jpg", "0632590"), ], ) -def test_wheel_reader_reads_a_real_meter(asset, expected): +@pytest.mark.parametrize("reader_id", list(readers.WHEEL_READERS)) +def test_wheel_reader_reads_a_real_meter(asset, expected, reader_id): """Issue #40: an ESP32 camera with its flash on a water meter with red decimal wheels.""" - spec = readers.WHEEL_READERS[readers.DEFAULT_WHEEL_READER] + spec = readers.WHEEL_READERS[reader_id] reader = readers.WheelReader(spec, MODEL_DIR / spec.filename) result, _ = reader.read_counter(Image.open(ASSETS / asset).convert("RGB"), 7) assert result.text == expected and result.score > 0.8 @@ -383,6 +459,32 @@ def test_counters_made_before_the_wheel_reader_keep_the_text_reader(tmp_path): assert keep_text_reader(None) is None +def test_counters_made_before_they_named_a_wheel_reader_keep_wheels_v1(tmp_path): + path = tmp_path / "old.db" + Database(path).init() + con = sqlite3.connect(path) + row = ( + "INSERT INTO sensor (slug, name, kind, source_type, source, interval_s, threshold, debounce, enabled, " + "entity_prefix, publish, created_at, reading) VALUES (?, ?, 'reading', 'http', 'x', 30, 0.7, 2, 1, 0, 1, " + "'2026-10-01', ?)" + ) + con.execute(row, ("water", "Water", json.dumps({"display": "counter", "digits": 8, "counter_reader": "wheels"}))) + con.execute(row, ("gas", "Gas", json.dumps({"display": "counter", "digits": 7, "counter_reader": "ocr"}))) + con.execute(row, ("lcd", "Display", json.dumps({"mode": "counter", "display": "auto"}))) + con.execute("PRAGMA user_version = 15") # as a database of 0.10.0 + con.commit() + con.close() + db = Database(path) + db.init() + with db.session() as s: + chosen = {r.slug: readers.wheel_reader_of(merge_reading(r.reading)) for r in s.query(Sensor)} + # A counter read as text keeps wheels-v1 too, should it be switched to the wheel reader; the + # display keeps no stored choice: should it become a counter, it gets the default. + assert chosen == {"water": "wheels-v1", "gas": "wheels-v1", "lcd": readers.DEFAULT_WHEEL_READER} + assert keep_wheel_reader({"display": "counter", "wheel_reader": "x"}) == {"display": "counter", "wheel_reader": "x"} + assert keep_wheel_reader(None) is None + + def test_counters_keep_their_change_limit_as_a_rate(tmp_path): path = tmp_path / "old.db" Database(path).init() @@ -986,11 +1088,14 @@ def view(): # Two wheels almost turned over: the wheels together read 89.950. assert wait_for(lambda: view()["reading"]["value"] == "89.950", timeout=60) assert view()["reading"]["counter_reader"] == "wheels" + # A new counter stores the default, so a later default never changes it. + assert view()["reading"]["wheel_reader"] == readers.DEFAULT_WHEEL_READER status = client.get("/api/v1/status").json() - assert status["wheel_reader"] == readers.DEFAULT_WHEEL_READER and status["wheel_reader_error"] == "" + assert status["wheel_readers"] == [readers.DEFAULT_WHEEL_READER] and status["wheel_reader_error"] == "" assert status["reader"] is None # only a counter read with the wheel reader: no text reader loaded models = client.get("/api/v1/settings").json() assert models["wheel_reader"] == readers.DEFAULT_WHEEL_READER + assert models["wheel_readers_in_use"] == [readers.DEFAULT_WHEEL_READER] assert [m["id"] for m in models["wheel_readers"]] == list(readers.WHEEL_READERS) assert models["wheel_readers"][0]["installed"] and models["wheel_readers"][0]["license"] == "Apache-2.0" image = client.get(f"/api/v1/sensors/{sid}/reading/image") @@ -1099,10 +1204,54 @@ def broken(*_args): client.app.state.runtime.grabber = CounterCamera(1234) sid = client.post("/api/v1/sensors", json=body).json()["id"] assert wait_for(lambda: client.get("/api/v1/status").json()["wheel_reader_error"] == "model file damaged") - assert client.get("/api/v1/status").json()["wheel_reader"] is None + assert client.get("/api/v1/status").json()["wheel_readers"] == [] assert "model file damaged" in client.get(f"/api/v1/sensors/{sid}").json()["live"]["error"] +@requires_reader +def test_each_counter_reads_with_the_wheel_reader_it_names(settings): + """Two wheel readers: each counter loads its own, one no counter reads with is let go, and a + sensor exported with a wheel reader this version does not have is imported with the default.""" + other = dataclasses.replace(readers.WHEEL_READERS["wheels-v1"], id="wheels-test", name="Test wheel reader") + settings.models_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(MODEL_DIR / "wheels-v1.onnx", settings.models_dir / other.filename) + reading = {"mode": "counter", "display": "counter", "digits": 7, "change_limit": "off"} + body = {"kind": "reading", "source_type": "http", "source": "http://fake", "roi": counter_box(7), "interval_s": 1} + + def loaded(): + return client.get("/api/v1/status").json()["wheel_readers"] + + with TestClient(create_app(settings)) as client: + client.app.state.runtime.grabber = CounterCamera(1234) + with patch.dict(readers.WHEEL_READERS, {other.id: other}): + assert [w["id"] for w in client.get("/api/v1/config").json()["wheel_readers"]] == ["wheels-v1", other.id] + unknown = {**body, "name": "Bad", "reading": {**reading, "wheel_reader": "wheels-nope"}} + assert client.post("/api/v1/sensors", json=unknown).status_code == 422 + first = client.post("/api/v1/sensors", json={**body, "name": "Water", "reading": reading}).json()["id"] + second = client.post( + "/api/v1/sensors", json={**body, "name": "Gas", "reading": {**reading, "wheel_reader": other.id}} + ).json()["id"] + assert wait_for(lambda: set(loaded()) == {"wheels-v1", other.id}, timeout=60) + assert set(client.get("/api/v1/settings").json()["wheel_readers_in_use"]) == {"wheels-v1", other.id} + for sid in (first, second): + assert wait_for( + lambda sid=sid: client.get(f"/api/v1/sensors/{sid}").json()["reading"]["value"] == "1234" + ) + exported = client.get(f"/api/v1/sensors/{second}/export").content + # Switched to the other one: the one no counter reads with any more is let go. + switched = {**reading, "wheel_reader": "wheels-v1"} + assert client.patch(f"/api/v1/sensors/{second}", json={"reading": switched}).status_code == 200 + assert wait_for(lambda: loaded() == ["wheels-v1"]) + # This version has no wheels-test: the import reads with the default. + imported = client.post("/api/v1/import", files={"file": ("b.zip", exported, "application/zip")}) + assert imported.status_code == 201, imported.text + copy = client.get(f"/api/v1/sensors/{imported.json()['id']}").json() + assert copy["reading"]["wheel_reader"] == readers.DEFAULT_WHEEL_READER + for sid in (first, second, imported.json()["id"]): + assert client.delete(f"/api/v1/sensors/{sid}").status_code in (200, 204) + assert wait_for(lambda: loaded() == []) + + @requires_reader def test_spot_checks_of_accepted_readings(settings): camera = DisplayCamera("00500") diff --git a/visionstate/backend/visionstate/api/common.py b/visionstate/backend/visionstate/api/common.py index 01ded36..85077ad 100644 --- a/visionstate/backend/visionstate/api/common.py +++ b/visionstate/backend/visionstate/api/common.py @@ -11,7 +11,7 @@ from sqlalchemy import func, select from sqlalchemy.orm import Session -from .. import change_limit, detectors, imaging, teach +from .. import change_limit, detectors, imaging, readers, teach from ..db import Face, ModelInfo, Sample, SampleLabel, Sensor from ..engine import LiveState, ObjectTrack, Runtime, SensorConfig, with_people from ..mqtt import main_entities @@ -298,6 +298,7 @@ class ReadingIn(BaseModel): display: str = READING_DEFAULTS["display"] digits: int = Field(default=READING_DEFAULTS["digits"], ge=_dlo["digits"], le=_dhi["digits"]) counter_reader: str = READING_DEFAULTS["counter_reader"] + wheel_reader: str = Field(default=READING_DEFAULTS["wheel_reader"], validate_default=True) max_step: float = Field(default=READING_DEFAULTS["max_step"], ge=_dlo["max_step"], le=_dhi["max_step"]) change_limit: str = READING_DEFAULTS["change_limit"] max_rate: float = Field(default=READING_DEFAULTS["max_rate"], ge=_dlo["max_rate"], le=_dhi["max_rate"]) @@ -334,6 +335,15 @@ def _counter_reader(cls, value: str) -> str: raise ValueError(f"Unknown counter reader {value!r}") return value + @field_validator("wheel_reader") + @classmethod + def _wheel_reader(cls, value: str) -> str: + if not value: + return readers.DEFAULT_WHEEL_READER # stored, so a later default never changes it + if value not in readers.WHEEL_READERS: + raise ValueError(f"Unknown wheel reader {value!r}") + return value + @field_validator("device_class") @classmethod def _device_class(cls, value: str) -> str: diff --git a/visionstate/backend/visionstate/api/imports.py b/visionstate/backend/visionstate/api/imports.py index 9a74a16..b6eca1d 100644 --- a/visionstate/backend/visionstate/api/imports.py +++ b/visionstate/backend/visionstate/api/imports.py @@ -12,8 +12,8 @@ from fastapi import APIRouter, File, HTTPException, Request, UploadFile from sqlalchemy import select -from .. import bundle, detectors, faces, imaging -from ..db import Person, keep_change_step, keep_text_reader +from .. import bundle, detectors, faces, imaging, readers +from ..db import Person, keep_change_step, keep_text_reader, keep_wheel_reader from ..engine import Runtime from ..redact import redact from ..settings import ( @@ -72,6 +72,10 @@ def _import_sync(rt: Runtime, path: Path) -> tuple[int, int, bool]: data["reading"] = keep_text_reader(data.get("reading")) # exported before the wheel reader # exported before a counter had a change limit of its own data["reading"] = keep_change_step(data["reading"]) + data["reading"] = keep_wheel_reader(data["reading"]) # exported before a counter named its wheel reader + if isinstance(data["reading"], dict) and data["reading"].get("wheel_reader") not in (None, *readers.WHEEL_READERS): + # exported by a newer version, with a wheel reader this one does not have: the default here + data["reading"] = {**data["reading"], "wheel_reader": ""} try: spec = validated_sensor(data) except ValueError as err: diff --git a/visionstate/backend/visionstate/api/system.py b/visionstate/backend/visionstate/api/system.py index 12c2cc0..4f96d17 100644 --- a/visionstate/backend/visionstate/api/system.py +++ b/visionstate/backend/visionstate/api/system.py @@ -132,12 +132,15 @@ def ui_config() -> dict: "face_unknown_key": FACE_UNKNOWN_KEY, "faces": {k: FACES[k] for k in ("max_people", "max_faces_per_person", "suggest_faces", "stats_window")}, "reading_sensor_defaults": READING_SENSOR_DEFAULTS, - "reading_defaults": READING_DEFAULTS, + "reading_defaults": {**READING_DEFAULTS, "wheel_reader": readers.DEFAULT_WHEEL_READER}, "reading_limits": READING_LIMITS, "counter_standard_steps_per_min": COUNTER_LIMIT["standard_steps_per_min"], "reading_modes": READING_MODES, "reading_displays": READING_DISPLAYS, "counter_readers": COUNTER_READERS, + "wheel_readers": [ + {"id": s.id, "name": s.name, "description": s.description} for s in readers.WHEEL_READERS.values() + ], "reading_device_classes": READING_DEVICE_CLASSES, "reading_counter_cell_share": READING["counter_cell_share"], "reading_export_meter_max_chars": READING["export_meter_max_chars"], @@ -184,8 +187,8 @@ def counts(): "reader": rt.reader.spec.id if rt.reader else None, "reader_name": rt.reader.spec.name if rt.reader else None, "reader_error": rt.reader_error, - "wheel_reader": rt.wheel_reader.spec.id if rt.wheel_reader else None, - "wheel_reader_name": rt.wheel_reader.spec.name if rt.wheel_reader else None, + "wheel_readers": list(rt.wheel_readers), # loaded + "wheel_reader_name": ", ".join(w.spec.name for w in rt.wheel_readers.values()) or None, "wheel_reader_error": rt.wheel_reader_error, # The face model, the people and how many sensors recognise them (the People page). "faces": {**rt.face_status(), "people": people, "sensors": len(face_sensors)}, @@ -262,8 +265,9 @@ def model(spec, kind: str = "") -> dict: "detectors": [model(spec, "detectors") for spec in detectors.DETECTORS.values()], "reader": rt.db.get_text("reader", readers.DEFAULT_READER), "readers": [model(spec, "readers") for spec in readers.READERS.values()], - # Mechanical counters set to the wheel reader; one model for now, so nothing to choose. + # Mechanical counters: each names its wheel reader (new ones get the default). "wheel_reader": readers.DEFAULT_WHEEL_READER, + "wheel_readers_in_use": sorted(rt.wheel_readers_in_use()), "wheel_readers": [model(spec) for spec in readers.WHEEL_READERS.values()], # Faces: the recogniser to choose (one face detector goes with every one); downloaded when # a sensor turns face recognition on. "size" and "installed" count the detector too. diff --git a/visionstate/backend/visionstate/bundle.py b/visionstate/backend/visionstate/bundle.py index 82deec5..f75164d 100644 --- a/visionstate/backend/visionstate/bundle.py +++ b/visionstate/backend/visionstate/bundle.py @@ -178,7 +178,7 @@ def export_readings(db: Database, storage: Storage, sensor_id: int, target: Path if sensor is None: raise KeyError(sensor_id) settings = merge_reading(sensor.reading) - wheels = settings["display"] == "counter" and settings["counter_reader"] == "wheels" + wheels = readers.uses_wheels(settings) mine = (Prediction.sensor_id == sensor_id, Prediction.frame.is_not(None)) newest = Prediction.created_at.desc() rows = [ @@ -243,7 +243,7 @@ def export_readings(db: Database, storage: Storage, sensor_id: int, target: Path manifest = { "format": READINGS_FORMAT, "app_version": VERSION, - "reader": readers.DEFAULT_WHEEL_READER if wheels else reader, + "reader": readers.wheel_reader_of(settings) if wheels else reader, "meter": meter.strip()[: READING["export_meter_max_chars"]] or None, # How the sensor reads; nothing about where it is (no camera, name or region position). "settings": { diff --git a/visionstate/backend/visionstate/db.py b/visionstate/backend/visionstate/db.py index a52f432..1c00a55 100644 --- a/visionstate/backend/visionstate/db.py +++ b/visionstate/backend/visionstate/db.py @@ -31,7 +31,7 @@ log = logging.getLogger(__name__) -SCHEMA_VERSION = 15 +SCHEMA_VERSION = 16 # Schema upgrades for existing databases, keyed on the version they upgrade to. # Each step is a list of (table, column, SQL type) columns to add. @@ -81,6 +81,25 @@ def _keep_text_readers(conn: Connection) -> None: ) +def keep_wheel_reader(reading: dict | None) -> dict | None: + """Reading settings of a counter made before a counter named its wheel reader, with the one it + read with kept (wheels-v1; switching is a choice on the Settings tab). Anything else is + returned as it is.""" + if not isinstance(reading, dict) or reading.get("display") != "counter" or "wheel_reader" in reading: + return reading + return {**reading, "wheel_reader": "wheels-v1"} + + +def _keep_wheel_readers(conn: Connection) -> None: + for sensor_id, raw in conn.execute(text("SELECT id, reading FROM sensor WHERE reading IS NOT NULL")).all(): + reading = json.loads(raw) if isinstance(raw, str) else raw + kept = keep_wheel_reader(reading) + if kept is not reading: + conn.execute( + text("UPDATE sensor SET reading = :r WHERE id = :id"), {"r": json.dumps(kept), "id": sensor_id} + ) + + def keep_change_step(reading: dict | None) -> dict | None: """Reading settings of a counter made before change_limit, with the limit it had kept. @@ -111,6 +130,8 @@ def _keep_change_steps(conn: Connection) -> None: 12: _keep_text_readers, # Counters keep the change limit they had (see keep_change_step). 14: _keep_change_steps, + # Counters made before a counter named its wheel reader keep wheels-v1 (see keep_wheel_reader). + 16: _keep_wheel_readers, } diff --git a/visionstate/backend/visionstate/engine/base.py b/visionstate/backend/visionstate/engine/base.py index ed553ee..7dbe829 100644 --- a/visionstate/backend/visionstate/engine/base.py +++ b/visionstate/backend/visionstate/engine/base.py @@ -60,7 +60,8 @@ def __init__(self, settings: Settings, db: Database): self.reader: readers.Reader | None = None # loaded on first use by a reading sensor self.reader_error = "" self._reader_lock = asyncio.Lock() - self.wheel_reader: readers.WheelReader | None = None # loaded on first use by a counter + # Wheel readers by id, each loaded on first use by a counter that reads with it. + self.wheel_readers: dict[str, readers.WheelReader] = {} self.wheel_reader_error = "" self._wheel_reader_lock = asyncio.Lock() # Faces (object sensors): the face model, loaded in the background when a sensor turns face diff --git a/visionstate/backend/visionstate/engine/models.py b/visionstate/backend/visionstate/engine/models.py index 31e27b6..832c8bd 100644 --- a/visionstate/backend/visionstate/engine/models.py +++ b/visionstate/backend/visionstate/engine/models.py @@ -8,10 +8,13 @@ from pathlib import Path from PIL import Image +from sqlalchemy import select from .. import backbones, detectors, imaging, readers +from ..db import Sensor +from ..readers import uses_wheels, wheel_reader_of from ..redact import redact -from ..settings import DETECTION, KIND_OBJECTS, KIND_READING, KIND_STATES +from ..settings import DETECTION, KIND_OBJECTS, KIND_READING, KIND_STATES, merge_reading from .base import RuntimeBase from .logic import filter_detections from .training import TrainingMixin @@ -113,24 +116,38 @@ async def set_reader(self, reader_id: str) -> None: for sensor_id in reading_sensors: self.wake(sensor_id, force=True) - async def ensure_wheel_reader(self) -> readers.WheelReader: - """The wheel reader for mechanical counters, loaded on first use (it is bundled).""" + async def ensure_wheel_reader(self, reader_id: str) -> readers.WheelReader: + """A wheel reader for mechanical counters (each counter names its own), loaded on first use.""" + spec = readers.WHEEL_READERS.get(reader_id) or readers.WHEEL_READERS[readers.DEFAULT_WHEEL_READER] async with self._wheel_reader_lock: - if self.wheel_reader is not None: - return self.wheel_reader - spec = readers.WHEEL_READERS[readers.DEFAULT_WHEEL_READER] + if (loaded := self.wheel_readers.get(spec.id)) is not None: + return loaded try: path = self.model_path(spec) if path is None: path = await asyncio.to_thread(backbones.download, spec, self.settings.models_dir) - self.wheel_reader = await asyncio.to_thread(readers.WheelReader, spec, path) + self.wheel_readers[spec.id] = loaded = await asyncio.to_thread(readers.WheelReader, spec, path) except Exception as err: self.wheel_reader_error = redact(str(err)) log.exception("Could not load wheel reader %s", spec.id) raise self.wheel_reader_error = "" log.info("Wheel reader %s loaded", spec.id) - return self.wheel_reader + return loaded + + def wheel_readers_in_use(self) -> set[str]: + """The wheel readers the counters read with.""" + with self.db.session() as s: + stored = s.scalars(select(Sensor.reading).where(Sensor.kind == KIND_READING)) + return {wheel_reader_of(merge_reading(r)) for r in stored if uses_wheels(merge_reading(r))} + + async def release_wheel_readers(self) -> None: + """Let go of the wheel readers no counter reads with any more (deleted, or switched).""" + in_use = await asyncio.to_thread(self.wheel_readers_in_use) + async with self._wheel_reader_lock: + for reader_id in [k for k in self.wheel_readers if k not in in_use]: + del self.wheel_readers[reader_id] + log.info("Wheel reader %s released", reader_id) async def read_number( self, image: Image.Image, roi: dict | None, reading: dict @@ -138,12 +155,13 @@ async def read_number( """Read the region of ``image``; returns the text and the image the reader used. The text's ``details`` name the reader that read it (``reader``).""" - region = imaging.crop_box(image, imaging.region_box(roi)) - if reading["display"] == "counter" and reading["counter_reader"] == "wheels": - wheels = await self.ensure_wheel_reader() + if uses_wheels(reading): + wheels = await self.ensure_wheel_reader(wheel_reader_of(reading)) + box = imaging.pixel_box(image, imaging.region_box(roi)) async with self._sem: - text, used = await asyncio.to_thread(wheels.read_counter, region, int(reading["digits"])) + text, used = await asyncio.to_thread(wheels.read_counter, image, int(reading["digits"]), box) return replace(text, details={"reader": wheels.spec.id, **(text.details or {})}), used + region = imaging.crop_box(image, imaging.region_box(roi)) reader = await self.ensure_reader() async with self._sem: text, used = await asyncio.to_thread( diff --git a/visionstate/backend/visionstate/engine/runtime.py b/visionstate/backend/visionstate/engine/runtime.py index d8bf7a3..5d628d6 100644 --- a/visionstate/backend/visionstate/engine/runtime.py +++ b/visionstate/backend/visionstate/engine/runtime.py @@ -99,7 +99,8 @@ def release(self) -> None: """After ``stop``, when another runtime takes over in this process (a restored backup): let go of the models, the sensors' heads and the frames kept in memory, and give the memory back, so two sets of models are never loaded at once.""" - self.embedder = self.detector = self.reader = self.wheel_reader = None + self.embedder = self.detector = self.reader = None + self.wheel_readers.clear() self.face_model = self.face_index = None self.heads.clear() self.taught.clear() @@ -153,6 +154,8 @@ async def sensor_updated(self, sensor_id: int, retrain: bool) -> None: await self.refresh_trigger_entities() if cfg.is_objects: await self.faces_settled() + if cfg.kind == KIND_READING: + await self.release_wheel_readers() # a counter may read with another one now self.wake(sensor_id) async def sensor_deleted(self, sensor_id: int, descriptor: SensorDescriptor) -> None: @@ -178,6 +181,8 @@ async def sensor_deleted(self, sensor_id: int, descriptor: SensorDescriptor) -> await self.faces_settled() if descriptor.kind == KIND_READING and not await asyncio.to_thread(self._sensor_ids, KIND_READING): self.reader = None + if descriptor.kind == KIND_READING: + await self.release_wheel_readers() await asyncio.to_thread(backbones.release_memory) def set_global_review(self, rules: dict) -> None: diff --git a/visionstate/backend/visionstate/imaging.py b/visionstate/backend/visionstate/imaging.py index ce23d13..fce8e50 100644 --- a/visionstate/backend/visionstate/imaging.py +++ b/visionstate/backend/visionstate/imaging.py @@ -189,11 +189,16 @@ def region_box(roi: dict | None, margin: float = 0.0) -> Box: ) -def crop_box(image: Image.Image, box: Box) -> Image.Image: - """Crop a normalised box without masking (a detector needs to see whole objects).""" +def pixel_box(image: Image.Image, box: Box) -> tuple[int, int, int, int]: + """A normalised box in the pixels of ``image``.""" x1, y1, x2, y2 = box width, height = image.size - return image.crop((int(x1 * width), int(y1 * height), int(x2 * width), int(y2 * height))) + return int(x1 * width), int(y1 * height), int(x2 * width), int(y2 * height) + + +def crop_box(image: Image.Image, box: Box) -> Image.Image: + """Crop a normalised box without masking (a detector needs to see whole objects).""" + return image.crop(pixel_box(image, box)) def in_region(x: float, y: float, roi: dict | None) -> bool: diff --git a/visionstate/backend/visionstate/readers.json b/visionstate/backend/visionstate/readers.json index ee3363d..25ae601 100644 --- a/visionstate/backend/visionstate/readers.json +++ b/visionstate/backend/visionstate/readers.json @@ -39,7 +39,7 @@ "readers": { "wheels-v1": { "name": "VisionState wheel reader v1", - "description": "Reads how far each wheel of a mechanical counter has turned; trained for VisionState (tools/wheelreader).", + "description": "Reads how far each wheel has turned, so a wheel between two digits is read right.", "url": "https://huggingface.co/oleost/visionstate-wheel-reader/resolve/42f32402973b3bd3de0258766404232b11e8216f/wheels-v1.onnx", "sha256": "37912f885b061a7b12298872fbdfa1b73ca0664363b293a55617b1d3680a98ac", "size": 2264691, diff --git a/visionstate/backend/visionstate/readers.py b/visionstate/backend/visionstate/readers.py index a07f2e1..1f1b8fe 100644 --- a/visionstate/backend/visionstate/readers.py +++ b/visionstate/backend/visionstate/readers.py @@ -258,6 +258,9 @@ def decode(probs: np.ndarray, chars: list[str]) -> Text: # --- wheel reader (mechanical counters) ------------------------------------------------------------- +WHEEL_CONFIDENCES = ("mean", "weakest") # see WheelReader.confidence + + @dataclass(frozen=True) class WheelReaderSpec: id: str @@ -273,11 +276,26 @@ class WheelReaderSpec: source: str bundled: bool footprint: Footprint = field(default_factory=lambda: Footprint()) + # A model with a second input: every cell again with this share of a cell on each side (the + # neighbouring wheels, cut from the whole frame: ``wheel_context_cells``), scaled to + # context_width x cell_height. 0 = the cells alone, cut from the region. + context: float = 0.0 + context_width: int = 0 + # The confidence of a read: "mean" (geometric mean of the wheels' decoded probabilities) or + # "weakest" (the weakest wheel's probability near where the value puts it). + confidence: str = "mean" @property def filename(self) -> str: return f"{self.id}.onnx" + def __post_init__(self) -> None: + # A registry mistake fails at start-up (and in the tests), not at a counter's first read. + if self.confidence not in WHEEL_CONFIDENCES: + raise ValueError(f"Wheel reader {self.id}: unknown confidence {self.confidence!r}") + if self.context < 0 or (self.context > 0) != (self.context_width > 0): + raise ValueError(f"Wheel reader {self.id}: context and context_width go together") + def load_wheel_registry() -> tuple[str, dict[str, WheelReaderSpec]]: """The default wheel reader's id and every wheel reader in ``readers.json``.""" @@ -297,15 +315,32 @@ def load_wheel_registry() -> tuple[str, dict[str, WheelReaderSpec]]: source=v.get("source", ""), bundled=bool(v.get("bundled", False)), footprint=Footprint.of(v), + context=float(v.get("context", 0.0)), + context_width=int(v.get("context_width", 0)), + confidence=v.get("confidence", "mean"), ) for key, v in raw["readers"].items() } return raw["default"], specs +# A region in a frame in pixels: left, top, right, bottom. +Pixels = tuple[int, int, int, int] + DEFAULT_WHEEL_READER, WHEEL_READERS = load_wheel_registry() +def uses_wheels(reading: dict) -> bool: + """Whether a reading sensor's settings read a mechanical counter with a wheel reader.""" + return reading.get("display") == "counter" and reading.get("counter_reader") == "wheels" + + +def wheel_reader_of(reading: dict) -> str: + """The wheel reader a counter reads with: its own (``wheel_reader``), else the default.""" + chosen = reading.get("wheel_reader") + return chosen if chosen in WHEEL_READERS else DEFAULT_WHEEL_READER + + def wheel_cells(image: Image.Image, digits: int, width: int, height: int) -> np.ndarray: """The region split into ``digits`` equal cells, each as the wheel model sees it. @@ -322,6 +357,32 @@ def wheel_cells(image: Image.Image, digits: int, width: int, height: int) -> np. return np.stack(out) +def wheel_context_cells( + image: Image.Image, box: Pixels, digits: int, width: int, height: int, context: float +) -> np.ndarray: + """The region ``box`` (pixels in ``image``) split into ``digits`` equal cells, each with + ``context`` of a cell on both sides, as the wheel model sees them. + + The context comes from the picture around the region (beyond the picture's edge, the edge + is repeated), so a wheel the region cuts is still seen whole. Grey, scale and contrast as + ``wheel_cells``. + """ + x0, y0, x1, y1 = box + count = max(1, digits) + cell = (x1 - x0) / count + pad = math.ceil(cell * context) + 2 + grey = np.pad(np.asarray(image.convert("RGB")).min(axis=2), ((0, 0), (pad, pad)), mode="edge") + wide = Image.fromarray(grey) + out = [] + for i in range(count): + a = round(x0 + (i - context) * cell + pad) + b = max(a + 1, round(x0 + (i + 1 + context) * cell + pad)) + part = wide.crop((a, y0, b, max(y0 + 1, y1))) + part = ImageOps.autocontrast(part.resize((width, height), Image.Resampling.BILINEAR), cutoff=1) + out.append(np.asarray(part, dtype=np.float32) / 255.0) + return np.stack(out) + + def wheel_value(logp: np.ndarray) -> tuple[float, float]: """The most likely counter value for per-wheel position log probabilities (wheels x 100 bins). @@ -383,19 +444,39 @@ def wheel_details(probs: np.ndarray, logp: np.ndarray, value: float, score: floa class WheelReader: - """Reads a mechanical counter wheel by wheel: one small ONNX model gives each wheel's position.""" + """Reads a mechanical counter wheel by wheel: one small ONNX model gives each wheel's position. + + ``image`` and ``box`` below: the frame and the region in it in pixels (no ``box``: the image + is the region). Only a model with context (``WheelReaderSpec.context``) looks beyond the region. + """ def __init__(self, spec: WheelReaderSpec, model_path: Path): self.spec = spec self.session = cpu_session(model_path) - self.input_name = self.session.get_inputs()[0].name + # The cells, then (with context) the cells with their neighbours. + self.input_names = [i.name for i in self.session.get_inputs()] + # The first output is the positions; a later one (such as each cell's embedding) is not used yet. + self.output_name = self.session.get_outputs()[0].name + if len(self.input_names) != (2 if spec.context else 1): + raise ValueError(f"Wheel reader {spec.id}: the model takes {len(self.input_names)} inputs") self._lock = threading.Lock() - def probabilities(self, image: Image.Image, digits: int) -> np.ndarray: + def inputs(self, image: Image.Image, digits: int, box: Pixels | None = None) -> list[np.ndarray]: + """What the model is given: the cells, and with context the cells with their neighbours + (each cells x height x width, 0..1).""" + h = self.spec.cell_height + out = [wheel_cells(image.crop(box) if box else image, digits, self.spec.cell_width, h)] + if self.spec.context: + whole = box or (0, 0, *image.size) + out.append(wheel_context_cells(image, whole, digits, self.spec.context_width, h, self.spec.context)) + return out + + def probabilities(self, image: Image.Image, digits: int, box: Pixels | None = None) -> np.ndarray: """The model's probability of every wheel's position (wheels x bins).""" - cells = wheel_cells(image, digits, self.spec.cell_width, self.spec.cell_height) + feed = {name: x[:, None] for name, x in zip(self.input_names, self.inputs(image, digits, box), strict=True)} with self._lock: - logits = np.asarray(self.session.run(None, {self.input_name: cells[:, None]})[0], dtype=np.float64) + out = self.session.run([self.output_name], feed) + logits = np.asarray(out[0], dtype=np.float64) logits -= logits.max(axis=1, keepdims=True) probs = np.exp(logits) return probs / probs.sum(axis=1, keepdims=True) @@ -407,19 +488,19 @@ def with_slack(probs: np.ndarray) -> np.ndarray: near = sum(np.roll(probs, s, axis=1) for s in range(-slack, slack + 1)) return np.log(np.maximum(near, 1e-12)) - def positions(self, image: Image.Image, digits: int) -> np.ndarray: + def positions(self, image: Image.Image, digits: int, box: Pixels | None = None) -> np.ndarray: """Log probabilities of every wheel's position (wheels x bins), as decoded.""" - return self.with_slack(self.probabilities(image, digits)) + return self.with_slack(self.probabilities(image, digits, box)) - def read_counter(self, image: Image.Image, digits: int) -> tuple[Text, Image.Image]: + def read_counter(self, image: Image.Image, digits: int, box: Pixels | None = None) -> tuple[Text, Image.Image]: """Read ``digits`` wheels; returns the read (all digits, last wheel rounded) and the cells used. A region drawn a little above or below the digits shifts every wheel alike, so a few common shifts are tried (each costs a little) and the most likely reading wins. The - confidence is the mean probability per wheel of the positions read. ``details`` keeps - how the value was found (``wheel_details``), so it can be checked without the model. + confidence is the spec's (``WheelReaderSpec.confidence``). ``details`` keeps how the + value was found (``wheel_details``), so it can be checked without the model. """ - probs = self.probabilities(image, digits) + probs = self.probabilities(image, digits, box) logp = self.with_slack(probs) best: tuple[float, float, list[int], int] | None = None for shift in range(-READING["wheel_max_shift_bins"], READING["wheel_max_shift_bins"] + 1): @@ -432,19 +513,35 @@ def read_counter(self, image: Image.Image, digits: int) -> tuple[Text, Image.Ima number = math.floor(value + 0.5) % 10**digits text = Text( f"{number:0{digits}d}", - math.exp(score / max(1, digits)), + self.confidence(probs, score, at, shift), {"wheels": wheel_details(probs, logp, value, score, at, shift)}, ) - return text, self.cells_image(image, digits) - - def cells_image(self, image: Image.Image, digits: int) -> Image.Image: - """The cells side by side as the model sees them (for the "what the reader saw" image).""" - cells = wheel_cells(image, digits, self.spec.cell_width, self.spec.cell_height) + return text, self.cells_image(image, digits, box) + + def confidence(self, probs: np.ndarray, score: float, at: list[int], shift: int) -> float: + """How sure the read is: "mean" is the geometric mean of the wheels' decoded probabilities; + "weakest" the smallest probability of a wheel within ``wheel_weakest_bins`` of where the + value puts it (one unclear wheel is not hidden by the clear ones).""" + if self.spec.confidence != "weakest": + return math.exp(score / max(1, len(at))) + bins = probs.shape[1] + near = np.arange(-READING["wheel_weakest_bins"], READING["wheel_weakest_bins"] + 1) + return float(min(probs[k, (b + shift + near) % bins].sum() for k, b in enumerate(at))) + + def cells_image(self, image: Image.Image, digits: int, box: Pixels | None = None) -> Image.Image: + """The cells side by side as the model sees them (for the "what the reader saw" image); + with context, the cells with their neighbours faded.""" + cells = (self.inputs(image, digits, box)[-1] * 255).astype(np.uint8) + if self.spec.context: + side = round(cells.shape[2] * self.spec.context / (1 + 2 * self.spec.context)) + faded = (cells * 0.35 + 255 * 0.65).astype(np.uint8) + cells[:, :, :side] = faded[:, :, :side] + cells[:, :, -side:] = faded[:, :, -side:] gap = 2 - w, h = self.spec.cell_width, self.spec.cell_height + h, w = cells.shape[1:] line = Image.new("L", (len(cells) * (w + gap) - gap, h), 255) for i, cell in enumerate(cells): - line.paste(Image.fromarray((cell * 255).astype(np.uint8)), (i * (w + gap), 0)) + line.paste(Image.fromarray(cell), (i * (w + gap), 0)) return line.resize((line.width * 2, h * 2), Image.Resampling.NEAREST).convert("RGB") diff --git a/visionstate/backend/visionstate/settings.py b/visionstate/backend/visionstate/settings.py index 08984db..b1e377e 100644 --- a/visionstate/backend/visionstate/settings.py +++ b/visionstate/backend/visionstate/settings.py @@ -94,6 +94,9 @@ "display": "auto", "digits": 6, # "counter" display only: wheels inside the region; other digit counts are rejected "counter_reader": "wheels", # "counter" display only: see COUNTER_READERS + # The wheel reader (an id in readers.json) a counter read by "wheels" uses. A new sensor + # stores the registry's default, so a later default never changes it; "" = that default. + "wheel_reader": "", # Values and times left: the largest plausible change between two readings (0 = no limit). # Counters use change_limit instead (see COUNTER_LIMIT). "max_step": 0.0, @@ -191,6 +194,10 @@ "wheel_slack_bins": 1, "wheel_max_shift_bins": 2, "wheel_shift_penalty": 0.3, + # Wheel readers whose confidence is the weakest wheel ("confidence": "weakest" in the + # registry): a wheel's share is the model's probability within this many bins of where the + # value puts it. + "wheel_weakest_bins": 4, "accepted_window_s": 86_400, # the "accepted" entity: share of the readings in the last 24 h "rate_min_span_s": 30.0, # no rate until two accepted readings are at least this far apart # "Export readings" (to share, e.g. on GitHub): the checked readings, then the accepted ones diff --git a/visionstate/frontend/src/lib/components/ModelCompare.svelte b/visionstate/frontend/src/lib/components/ModelCompare.svelte index fa73f09..7ad250e 100644 --- a/visionstate/frontend/src/lib/components/ModelCompare.svelte +++ b/visionstate/frontend/src/lib/components/ModelCompare.svelte @@ -15,14 +15,11 @@ let open = $state(false); type Kind = (typeof MODEL_KINDS)[number]['key']; - const chosen = (kind: Kind) => - ({ - backbones: info.backbone, - detectors: info.detector, - face_models: info.face_model, - readers: info.reader, - wheel_readers: info.wheel_reader, - })[kind]; + // Counters each name their wheel reader, so several can be in use. + const chosen = (kind: Kind, id: string) => + kind === 'wheel_readers' + ? info.wheel_readers_in_use.includes(id) + : id === { backbones: info.backbone, detectors: info.detector, face_models: info.face_model, readers: info.reader }[kind]; const deletable = (kind: Kind): kind is ModelKind => kind !== 'wheel_readers'; async function remove(kind: ModelKind, m: BackboneInfo) { @@ -49,9 +46,9 @@ {kind.title} {kind.text} {#each bySize(info[kind.key] as BackboneInfo[]) as m (m.id)} -
+
- {m.name}{#if m.id === chosen(kind.key)} · in use{/if} + {m.name}{#if chosen(kind.key, m.id)} · in use{/if} ~{ram(m)} RAM diff --git a/visionstate/frontend/src/lib/components/ReadingEditor.svelte b/visionstate/frontend/src/lib/components/ReadingEditor.svelte index b234277..28a806e 100644 --- a/visionstate/frontend/src/lib/components/ReadingEditor.svelte +++ b/visionstate/frontend/src/lib/components/ReadingEditor.svelte @@ -17,6 +17,22 @@ const displays = $derived((app.config?.reading_displays ?? []).filter((d) => d !== 'counter')); const digitLimits = $derived(app.config?.reading_limits.digits ?? [1, 12]); const counterReaders = $derived(app.config?.counter_readers ?? []); + // "Read with": every wheel reader (the default first) and the text reader in one list; with one + // wheel reader, that one is simply "wheels". + const defaultWheels = $derived(app.config?.reading_defaults.wheel_reader ?? ''); + const wheelReaders = $derived( + [...(app.config?.wheel_readers ?? [])].sort( + (a, b) => Number(b.id === defaultWheels) - Number(a.id === defaultWheels), + ), + ); + const severalWheels = $derived(wheelReaders.length > 1); + const wheelsOption = (id: string) => (severalWheels ? `wheels:${id}` : 'wheels'); + const wheelsChosen = $derived( + value.counter_reader === 'wheels' + ? (wheelReaders.find((w) => w.id === (value.wheel_reader || defaultWheels)) ?? wheelReaders[0]) + : undefined, + ); + const readWith = $derived(wheelsChosen ? wheelsOption(wheelsChosen.id) : value.counter_reader); const deviceClasses = $derived(app.config?.reading_device_classes ?? []); const decimals = $derived(app.config?.reading_limits.decimals ?? [0, 4]); const suggested = $derived.by(() => { @@ -37,6 +53,13 @@ } } + function setReadWith(choice: string) { + const [reader, wheels] = choice.split(':'); + value.counter_reader = reader as ReadingSettings['counter_reader']; + if (wheels) value.wheel_reader = wheels; + else if (reader === 'wheels' && wheelReaders[0]) value.wheel_reader = wheelReaders[0].id; + } + function setMode(mode: ReadingMode) { value.mode = mode; if (mode === 'time_left') value.device_class = 'duration'; @@ -137,14 +160,18 @@ {:else}

A counter that reads lower than before is rejected; under Sensor output you can also limit how much it diff --git a/visionstate/frontend/src/lib/types.ts b/visionstate/frontend/src/lib/types.ts index 421ca97..c3c0a87 100644 --- a/visionstate/frontend/src/lib/types.ts +++ b/visionstate/frontend/src/lib/types.ts @@ -115,6 +115,8 @@ export interface ReadingSettings { digits: number; /** Mechanical counters only: what reads the wheels. */ counter_reader: CounterReader; + /** counter_reader "wheels": the wheel reader's id ("" = the default). */ + wheel_reader: string; /** Values and times left, and counters with change_limit "step": the largest change between two * readings (0 = no limit). */ max_step: number; @@ -560,6 +562,8 @@ export interface AppConfig { reading_modes: ReadingMode[]; reading_displays: ReadingDisplay[]; counter_readers: CounterReader[]; + /** The wheel readers a counter can read with (the default is in reading_defaults). */ + wheel_readers: { id: string; name: string; description: string }[]; reading_device_classes: string[]; /** Mechanical counters: the share of each digit field's width that is read. */ reading_counter_cell_share: number; @@ -586,8 +590,8 @@ export interface Status { reader: string | null; reader_name: string | null; reader_error: string; - /** Mechanical counters read wheel by wheel; loaded on first use. */ - wheel_reader: string | null; + /** Mechanical counters read wheel by wheel: the wheel readers loaded (each on first use). */ + wheel_readers: string[]; wheel_reader_name: string | null; wheel_reader_error: string; /** The face model, and how many people and sensors that recognise faces there are. */ @@ -769,9 +773,11 @@ export interface SettingsInfo { detectors: DetectorInfo[]; reader: string; readers: DetectorInfo[]; - /** The wheel reader for mechanical counters (one model for now, nothing to choose). */ + /** The wheel reader new counters get; each counter names its own. */ wheel_reader: string; wheel_readers: DetectorInfo[]; + /** The wheel readers counters read with. */ + wheel_readers_in_use: string[]; /** Recognising faces on object sensors: the recogniser chosen; downloaded when a sensor turns faces on. */ face_model: string; face_models: FaceModelInfo[]; diff --git a/visionstate/frontend/src/pages/Settings.svelte b/visionstate/frontend/src/pages/Settings.svelte index a238470..9698691 100644 --- a/visionstate/frontend/src/pages/Settings.svelte +++ b/visionstate/frontend/src/pages/Settings.svelte @@ -294,8 +294,8 @@ {wheelReader.name} · included {wheelReader.description} {wheelReader.license}, - source. Used by counters set to - Read with → Wheel reader (the default).source. New counters read with it + (Read with on a counter's Settings tab).

{/if} diff --git a/visionstate/frontend/src/pages/sensor/ReadingLiveTab.svelte b/visionstate/frontend/src/pages/sensor/ReadingLiveTab.svelte index 443717c..e782875 100644 --- a/visionstate/frontend/src/pages/sensor/ReadingLiveTab.svelte +++ b/visionstate/frontend/src/pages/sensor/ReadingLiveTab.svelte @@ -1,7 +1,7 @@