From c36c346ed2c725f2546e67cb534a101d89d72136 Mon Sep 17 00:00:00 2001 From: Emre Date: Wed, 2 Sep 2026 21:28:59 +0300 Subject: [PATCH 1/2] fix Co-authored-by: Cursor --- CHANGELOG.md | 6 +++ CONTRIBUTING.md | 6 ++- ROADMAP.md | 16 +++---- SPEC.md | 5 +- src/iqforge/cli.py | 25 +++++++++- src/iqforge/measurement.py | 47 ++++++++++++++---- src/iqforge/training.py | 5 +- tests/test_measurement.py | 36 ++++++++++++++ tests/test_models.py | 97 ++++++++++++++++++++++++++++++++++++-- tests/test_preflight.py | 16 +++++++ 10 files changed, 231 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02ea7f0..c970b8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ can tell whether the format it is looking at is one it understands. ### Added +- **`--device cpu|cuda` on `iqforge measure-leakage`**, defaulting to `cpu`. + CUDA is opt-in for new measurements and is never selected just because a GPU + is present; `cuda` errors when torch reports none. Published tables, the + parity gate, and `artifacts/*.json` stay on CPU. `TrainingResult.environment` + already recorded the device; the measurement path now stamps the device that + was actually requested. - `iqforge measure-leakage` now accepts `--balance-by`, so the command path can run the same nuisance-balancing setup that the published synthetic measurement tables used. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e73b12..6e19d1a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,6 +71,7 @@ Then: ```bash uv run iqforge train dataset/ --device cuda +uv run iqforge measure-leakage records/ --device cuda ``` `--device auto` picks CUDA when it is available and CPU otherwise; `--device @@ -80,7 +81,10 @@ prints a warning that its numbers are not bit-comparable with CPU runs, and `TrainingResult.environment` records the device, torch version, CUDA version, and the numpy / scipy / sigmf versions windowing, normalisation and reading depend on, so a results file can be checked later. The sweep scripts in `scripts/` -refuse to extend a checkpoint that was measured on a different environment. +and `scripts/parity_gate.py` do not pass `--device`, so published tables stay +on CPU. A CUDA leakage run is a new labeled measurement, not a replacement of +`artifacts/leakage_*.json`. The sweep scripts refuse to extend a checkpoint +that was measured on a different environment. ## Engineering conventions diff --git a/ROADMAP.md b/ROADMAP.md index 7ef3e4b..133db0d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -50,15 +50,15 @@ After Now is done — still reliability-first: - [ ] Hardware capture on **one** device (RTL-SDR / HackRF / Pluto) — parallel to public-file work, not a blocker for it. One device proves one path, not all integer/I/Q conventions. -- [ ] Optional CUDA for **new** measurements only. Published tables and the +- [x] Optional CUDA for **new** measurements only. Published tables and the parity gate stay CPU; never re-measure those grids on GPU and call them - the same result. `iqforge train` stays CPU by default — CUDA is opt-in, - never a silent fallback because a GPU is present. Record `cpu` / `cuda` - in `TrainingResult.environment` (or equivalent) so two tables can be - told apart. A CUDA run is a new labeled measurement (the real-recording - leakage curve, when that happens), not a replacement of - `artifacts/leakage_*.json`. Do not claim GPU bit-equality. Do not - enlarge *n* after seeing results. + the same result. `iqforge train` and `iqforge measure-leakage` stay CPU + by default — CUDA is `--device cuda`, never a silent fallback because a + GPU is present. Record `cpu` / `cuda` in `TrainingResult.environment` + (or equivalent) so two tables can be told apart. A CUDA run is a new + labeled measurement (the real-recording leakage curve, when that + happens), not a replacement of `artifacts/leakage_*.json`. Do not claim + GPU bit-equality. Do not enlarge *n* after seeing results. - [ ] Frequency-aware labeling (SPEC §5.3 deferred item) - [x] **`--group-by`: keep related recordings together.** Shipped with two schemes, `path:` and `csv:`. Recordings sharing a key become diff --git a/SPEC.md b/SPEC.md index 7133b0b..544997f 100644 --- a/SPEC.md +++ b/SPEC.md @@ -142,7 +142,7 @@ iqforge measure-leakage [--window 1024] [--stride 512] [--dirname-level 1] [--group-by {path:,csv:,collection}] [--balance-by ] [--split 0.6,0.2,0.2] [--force] [--format {text,json}] - [--sweep stride] + [--sweep stride] [--device {auto,cpu,cuda}] Runs `audit`, classifies into the six categories in methodology §6, then: REFUSED / INCONCLUSIVE => exit 1; WOULD MEASURE => run the paired measurement cell (recording-level and window-level) and report inflation. @@ -150,6 +150,9 @@ iqforge measure-leakage [--window 1024] [--stride 512] --force overrides a refusal and keeps the overridden category in the header so a pasted block cannot be mistaken for a clean run. --sweep supports only `stride`; no SNR-injection flag (see 5.10). + --device: default `cpu`, same promise as `train`. CUDA is opt-in and + errors rather than falling back when no CUDA device is present. + Published tables and the parity gate stay on CPU. ``` --- diff --git a/src/iqforge/cli.py b/src/iqforge/cli.py index e87b43c..b6ca0fe 100644 --- a/src/iqforge/cli.py +++ b/src/iqforge/cli.py @@ -1159,6 +1159,9 @@ def measure_leakage( # noqa: PLR0913 — flags match `audit` plus --force / --g train_seeds: Annotated[ str, typer.Option("--train-seeds", help="Comma-separated training seeds") ] = DEFAULT_TRAIN_SEEDS, + device: Annotated[ + str, typer.Option("--device", help="auto, cpu or cuda. Default cpu, deliberately") + ] = "cpu", ) -> None: """Measure leakage inflation after running the refuse-path gate. @@ -1175,6 +1178,9 @@ def measure_leakage( # noqa: PLR0913 — flags match `audit` plus --force / --g run can be made cheaper deliberately and visibly, and the count is printed with the result: a measurement whose sample size is not on the page cannot be read. + + `--device` defaults to CPU, the same promise as `train`. CUDA is opt-in and + never selected just because a GPU is present. Published tables stay CPU. """ if output_format not in ("text", "json"): raise _fail(IQForgeError(f"--format must be text or json, got '{output_format}'.")) @@ -1233,6 +1239,13 @@ def measure_leakage( # noqa: PLR0913 — flags match `audit` plus --force / --g raise typer.Exit(code=1) if not _torch_available(): + if device != "cpu": + raise _fail( + IQForgeError( + f"--device {device} was requested but torch is not installed. " + + TORCH_REQUIRED.format(what="`iqforge measure-leakage`") + ) + ) if sweep is not None: raise _fail(IQForgeError(TORCH_REQUIRED.format(what="iqforge measure-leakage --sweep"))) return @@ -1256,6 +1269,14 @@ def measure_leakage( # noqa: PLR0913 — flags match `audit` plus --force / --g balance_by=balance_by, ) + def _measured(cells: list[GridCell]) -> list[Any]: + try: + return run_grid( + cells, split_seed_list, train_seed_list, verbose=False, device_choice=device + ) + except IQForgeError as exc: + raise _fail(exc) from exc + if sweep == "stride": strides = (1024, 768, 512, 256, 128) cells = [ @@ -1276,7 +1297,7 @@ def measure_leakage( # noqa: PLR0913 — flags match `audit` plus --force / --g ) for s in strides ] - runs = run_grid(cells, split_seed_list, train_seed_list, verbose=False) + runs = _measured(cells) pairs = len(split_seed_list) * len(train_seed_list) caption = ( f"Stride sweep over {pairs} seed pair(s) " @@ -1319,7 +1340,7 @@ def measure_leakage( # noqa: PLR0913 — flags match `audit` plus --force / --g return cell = GridCell(records=path, spec=spec, stride=stride, label=f"stride={stride}") - runs = run_grid([cell], split_seed_list, train_seed_list, verbose=False) + runs = _measured([cell]) pairs = len(split_seed_list) * len(train_seed_list) stats = cell_stats(runs) if stats is None: diff --git a/src/iqforge/measurement.py b/src/iqforge/measurement.py index 43c90aa..78c5d1d 100644 --- a/src/iqforge/measurement.py +++ b/src/iqforge/measurement.py @@ -272,13 +272,26 @@ def split_counts(dataset: Path) -> tuple[int, int]: def train_once( - dataset: Path, train_seed: int, epochs: int = EPOCHS + dataset: Path, + train_seed: int, + epochs: int = EPOCHS, + *, + device_choice: str | None = None, ) -> tuple[float, float, int, int]: - """Train the baseline and return accuracies and split sizes.""" - from iqforge.training import train_baseline + """Train the baseline and return accuracies and split sizes. + + `device_choice` is the same `cpu` / `cuda` / `auto` string `iqforge train` + takes. Left unset, training stays on CPU even when a GPU is present. + """ + from iqforge.training import DEFAULT_DEVICE, train_baseline manifest = json.loads((dataset / "manifest.json").read_text(encoding="utf-8")) - result = train_baseline(dataset, epochs=epochs, seed=train_seed) + result = train_baseline( + dataset, + epochs=epochs, + seed=train_seed, + device_choice=DEFAULT_DEVICE if device_choice is None else device_choice, + ) return ( result.test_accuracy or 0.0, result.final_train_accuracy, @@ -287,7 +300,7 @@ def train_once( ) -def current_environment() -> dict[str, str]: +def current_environment(device_choice: str | None = None) -> dict[str, str]: """Device, torch, CUDA, and numeric-stack versions of this process. Degrades rather than raising when torch is absent. The guards that use this @@ -295,6 +308,10 @@ def current_environment() -> dict[str, str]: still worth answering in a torch-free install -- `audit` and the refuse path both run there. Windowing and normalisation are numpy and scipy work regardless, so those versions are reported either way. + + `device_choice` is the training device that will produce the numbers, not + "whatever GPU happens to be plugged in". Unset means CPU, matching + `iqforge train` and `iqforge measure-leakage`. """ import numpy import scipy @@ -309,7 +326,8 @@ def current_environment() -> dict[str, str]: from iqforge.training import DEFAULT_DEVICE, describe_environment, resolve_device except ModuleNotFoundError: return {"device": "none (torch not installed)", **base} - return describe_environment(resolve_device(DEFAULT_DEVICE)) + choice = DEFAULT_DEVICE if device_choice is None else device_choice + return describe_environment(resolve_device(choice)) def check_environment(runs_path: Path) -> None: @@ -481,6 +499,7 @@ def run_grid( checkpoint: Callable[[list[Run]], None] | None = None, epochs: int = EPOCHS, verbose: bool = True, + device_choice: str | None = None, ) -> list[Run]: """Build both arms and train them, for every cell and seed pair. @@ -492,9 +511,11 @@ def run_grid( train_seeds: Seeds for weight init and batch order. checkpoint: Called with the runs so far after every run. epochs: Training length. Default is the published-table value. + device_choice: Training device (`cpu`, `cuda`, `auto`). Defaults to + CPU; a GPU that happens to be present is not used unless asked. """ runs: list[Run] = [] - environment = current_environment() + environment = current_environment(device_choice) work = Path(tempfile.mkdtemp(prefix="iqforge-measure-")) total = len(cells) * len(split_seeds) * len(train_seeds) * 2 done = 0 @@ -513,7 +534,7 @@ def run_grid( for train_seed in train_seeds: t0 = time.time() acc, train_acc, n_train, n_test = train_once( - dataset, train_seed, epochs=epochs + dataset, train_seed, epochs=epochs, device_choice=device_choice ) done += 1 run = Run( @@ -562,6 +583,7 @@ def measure_pair( train_seed: int, epochs: int = EPOCHS, verbose: bool = True, + device_choice: str | None = None, ) -> tuple[Run, Run]: """One seed pair, both arms. The unit the LoRaIQ acceptance cell is.""" cell = GridCell( @@ -570,7 +592,14 @@ def measure_pair( stride=spec.stride, label=f"stride={spec.stride}" if spec.stride is not None else "", ) - runs = run_grid([cell], [split_seed], [train_seed], epochs=epochs, verbose=verbose) + runs = run_grid( + [cell], + [split_seed], + [train_seed], + epochs=epochs, + verbose=verbose, + device_choice=device_choice, + ) rec = next(r for r in runs if r.strategy == "recording-level") win = next(r for r in runs if r.strategy == "window-level") return rec, win diff --git a/src/iqforge/training.py b/src/iqforge/training.py index 54db2ea..5bd47e1 100644 --- a/src/iqforge/training.py +++ b/src/iqforge/training.py @@ -199,8 +199,9 @@ def train_baseline( seed. learning_rate: Adam learning rate. on_epoch: Callback invoked at the end of every epoch. - device_choice: `auto`, `cpu` or `cuda`. Defaults to CPU; see - `DEFAULT_DEVICE` for why that is deliberate. + device_choice: `auto`, `cpu` or `cuda`. Defaults to CPU; see + `DEFAULT_DEVICE` for why that is deliberate. CUDA is never selected + just because a GPU is present. Returns: The results of the run. diff --git a/tests/test_measurement.py b/tests/test_measurement.py index d480207..737d738 100644 --- a/tests/test_measurement.py +++ b/tests/test_measurement.py @@ -362,3 +362,39 @@ def blocked(name, *args, **kwargs): assert "numpy" in env assert "scipy" in env assert "torch" not in env + + +def test_train_once_forwards_the_requested_device(tmp_path, monkeypatch): + """measure-leakage's training path must honour --device, not ignore it.""" + from iqforge.measurement import train_once + + (tmp_path / "manifest.json").write_text( + json.dumps({"splits": {"train": {"count": 1}, "test": {"count": 1}}}), + encoding="utf-8", + ) + seen: dict[str, str] = {} + + class _Result: + test_accuracy = 0.5 + final_train_accuracy = 0.75 + environment = {"device": "cuda"} + + def fake_train_baseline(dataset, *, epochs, seed, device_choice="cpu"): + seen["device_choice"] = device_choice + return _Result() + + monkeypatch.setattr("iqforge.training.train_baseline", fake_train_baseline) + train_once(tmp_path, train_seed=0, epochs=1, device_choice="cuda") + assert seen["device_choice"] == "cuda" + + train_once(tmp_path, train_seed=0, epochs=1) + assert seen["device_choice"] == "cpu" + + +def test_current_environment_stamps_an_opt_in_cuda_device(monkeypatch): + torch = pytest.importorskip("torch") + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_name", lambda *args, **kwargs: "mocked-gpu") + assert current_environment("cpu")["device"] == "cpu" + assert current_environment("cuda")["device"] == "cuda" + assert current_environment()["device"] == "cpu" diff --git a/tests/test_models.py b/tests/test_models.py index 46ecd8f..f258574 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,6 +2,7 @@ from __future__ import annotations +import numpy as np import pytest torch = pytest.importorskip("torch", reason="the baseline model needs torch") @@ -85,15 +86,101 @@ def test_device_defaults_to_cpu_and_is_recorded(tmp_path): assert env["sigmf"] != "absent" -def test_an_unavailable_device_errors_rather_than_falling_back(): - """A run that silently used another device is worse than one that stops.""" - import torch +def _pretend_cuda_is_available(monkeypatch: pytest.MonkeyPatch) -> None: + """A GPU is present, without requiring one on this machine.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_name", lambda *args, **kwargs: "mocked-gpu") + + +def test_default_stays_cpu_even_when_cuda_is_available(monkeypatch: pytest.MonkeyPatch) -> None: + """A GPU being present is not an opt-in. The default must stay CPU.""" + from iqforge.training import DEFAULT_DEVICE, resolve_device + + _pretend_cuda_is_available(monkeypatch) + assert DEFAULT_DEVICE == "cpu" + assert resolve_device().type == "cpu" + assert resolve_device(DEFAULT_DEVICE).type == "cpu" + + +def test_opt_in_cuda_is_requested_when_available(monkeypatch: pytest.MonkeyPatch) -> None: + """`--device cuda` is the opt-in; it must actually select CUDA.""" + from iqforge.training import resolve_device + + _pretend_cuda_is_available(monkeypatch) + assert resolve_device("cuda").type == "cuda" + + +def test_environment_stamps_the_resolved_device(monkeypatch: pytest.MonkeyPatch) -> None: + """cpu vs cuda must be distinguishable later from the results file alone.""" + from iqforge.training import describe_environment, resolve_device + + _pretend_cuda_is_available(monkeypatch) + cpu = describe_environment(resolve_device("cpu")) + cuda = describe_environment(resolve_device("cuda")) + assert cpu["device"] == "cpu" + assert cuda["device"] == "cuda" + assert cuda["device_name"] == "mocked-gpu" + +def test_cpu_flag_must_not_silently_use_cuda(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: + """If training started using CUDA whenever a GPU exists, this must go red. + + The failure mode is not an explicit `--device cuda`. It is the default, or + an explicit `--device cpu`, landing on a GPU because one was present. + """ + from iqforge.storage import ShardWriter, write_manifest + from iqforge.training import train_baseline + + _pretend_cuda_is_available(monkeypatch) + + window = 32 + splits: dict[str, dict] = {} + for split, rows in (("train", 8), ("val", 0), ("test", 4)): + writer = ShardWriter(tmp_path, split) + for i in range(rows): + writer.add(np.full((1, 2, window), float(i), dtype=np.float32), [i % 2]) + writer.flush() + splits[split] = { + "shards": writer.shards, + "labels": writer.labels, + "count": writer.count, + "records": [], + } + write_manifest( + tmp_path, + version="0.1.0", + config={"window": window, "stride": 8, "repr": "iq2ch", "normalize": True, "seed": 42}, + label_map={"a": 0, "b": 1}, + source_files=[], + splits=splits, + ) + + destinations: list[str] = [] + original_to = torch.nn.Module.to + + def tracking_to(self, *args, **kwargs): + target = args[0] if args else kwargs.get("device") + if isinstance(target, (str, torch.device)): + destinations.append(torch.device(target).type) + return original_to(self, *args, **kwargs) + + monkeypatch.setattr(torch.nn.Module, "to", tracking_to) + + result = train_baseline(tmp_path, epochs=1, batch_size=4, device_choice="cpu") + assert result.environment["device"] == "cpu" + assert destinations + assert all(kind == "cpu" for kind in destinations) + assert "cuda" not in destinations + + +def test_an_unavailable_device_errors_rather_than_falling_back( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A run that silently used another device is worse than one that stops.""" from iqforge.io import IQForgeError from iqforge.training import resolve_device - if torch.cuda.is_available(): - pytest.skip("CUDA is available here, so the refusal cannot be exercised") + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) with pytest.raises(IQForgeError, match="no CUDA device"): resolve_device("cuda") diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 0619667..d1aaa09 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -332,6 +332,22 @@ def _measure_leakage_option_flags() -> set[str]: return flags +def test_measure_leakage_device_defaults_to_cpu() -> None: + """CUDA is opt-in. A GPU being present must not change the default.""" + from typer.main import get_group + + flags = _measure_leakage_option_flags() + assert "--device" in flags + group = get_group(app) + param = next( + p for p in group.commands["measure-leakage"].params if "--device" in (p.opts or ()) + ) + assert param.default == "cpu" + + train = next(p for p in group.commands["train"].params if "--device" in (p.opts or ())) + assert train.default == "cpu" + + def test_help_does_not_offer_sweep_snr() -> None: result = runner.invoke( app, From 66cccb3a599af39231f43f14a4a44ce3df386b4a Mon Sep 17 00:00:00 2001 From: Emre Date: Thu, 3 Sep 2026 11:15:46 +0300 Subject: [PATCH 2/2] fix Co-authored-by: Cursor --- tests/test_measurement.py | 1 + tests/test_models.py | 41 +++++++++++++-------------------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/tests/test_measurement.py b/tests/test_measurement.py index 737d738..391619a 100644 --- a/tests/test_measurement.py +++ b/tests/test_measurement.py @@ -366,6 +366,7 @@ def blocked(name, *args, **kwargs): def test_train_once_forwards_the_requested_device(tmp_path, monkeypatch): """measure-leakage's training path must honour --device, not ignore it.""" + pytest.importorskip("torch") from iqforge.measurement import train_once (tmp_path / "manifest.json").write_text( diff --git a/tests/test_models.py b/tests/test_models.py index f258574..9c28caa 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,7 +2,6 @@ from __future__ import annotations -import numpy as np import pytest torch = pytest.importorskip("torch", reason="the baseline model needs torch") @@ -122,39 +121,21 @@ def test_environment_stamps_the_resolved_device(monkeypatch: pytest.MonkeyPatch) assert cuda["device_name"] == "mocked-gpu" -def test_cpu_flag_must_not_silently_use_cuda(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_cpu_flag_must_not_silently_use_cuda(monkeypatch: pytest.MonkeyPatch) -> None: """If training started using CUDA whenever a GPU exists, this must go red. The failure mode is not an explicit `--device cuda`. It is the default, or an explicit `--device cpu`, landing on a GPU because one was present. + + A full `train_baseline` step is not run: mocking `is_available` True still + makes `cuda.manual_seed_all` and Adam touch a real CUDA context, which GitHub + Actions does not have. Placement is the contract -- `resolve_device("cpu")` + and `Module.to` destinations -- the same path `train_baseline` uses. """ - from iqforge.storage import ShardWriter, write_manifest - from iqforge.training import train_baseline + from iqforge.training import describe_environment, resolve_device _pretend_cuda_is_available(monkeypatch) - window = 32 - splits: dict[str, dict] = {} - for split, rows in (("train", 8), ("val", 0), ("test", 4)): - writer = ShardWriter(tmp_path, split) - for i in range(rows): - writer.add(np.full((1, 2, window), float(i), dtype=np.float32), [i % 2]) - writer.flush() - splits[split] = { - "shards": writer.shards, - "labels": writer.labels, - "count": writer.count, - "records": [], - } - write_manifest( - tmp_path, - version="0.1.0", - config={"window": window, "stride": 8, "repr": "iq2ch", "normalize": True, "seed": 42}, - label_map={"a": 0, "b": 1}, - source_files=[], - splits=splits, - ) - destinations: list[str] = [] original_to = torch.nn.Module.to @@ -166,8 +147,12 @@ def tracking_to(self, *args, **kwargs): monkeypatch.setattr(torch.nn.Module, "to", tracking_to) - result = train_baseline(tmp_path, epochs=1, batch_size=4, device_choice="cpu") - assert result.environment["device"] == "cpu" + device = resolve_device("cpu") + assert device.type == "cpu" + + BaselineCNN(num_classes=2).to(device) + + assert describe_environment(device)["device"] == "cpu" assert destinations assert all(kind == "cpu" for kind in destinations) assert "cuda" not in destinations