diff --git a/.github/workflows/ovs-heritage-p0.yml b/.github/workflows/ovs-heritage-p0.yml index 164c3d5..0b022e1 100644 --- a/.github/workflows/ovs-heritage-p0.yml +++ b/.github/workflows/ovs-heritage-p0.yml @@ -4,6 +4,9 @@ on: pull_request: paths: - "ovs_heritage/**" + - "segmentation/evaluation/lposs_eval.py" + - "tools/check_stock_lposs_gpu.py" + - "tools/run_official_lposs_parity.py" - ".github/workflows/ovs-heritage-p0.yml" - "environment.yml" push: @@ -19,6 +22,6 @@ jobs: python-version: "3.9" - run: python -m pip install --upgrade pip - run: python -m pip install --extra-index-url https://download.pytorch.org/whl/cpu torch==1.12.1 numpy==1.26.4 pillow==10.4.0 pyyaml==6.0.2 pycocotools==2.0.8 pytest==8.3.3 ruff==0.6.9 - - run: python -m compileall -q ovs_heritage + - run: python -m compileall -q ovs_heritage segmentation/evaluation/lposs_eval.py tools/check_stock_lposs_gpu.py tools/run_official_lposs_parity.py - run: pytest -q ovs_heritage/tests - - run: ruff check ovs_heritage + - run: ruff check ovs_heritage segmentation/evaluation/lposs_eval.py tools/check_stock_lposs_gpu.py tools/run_official_lposs_parity.py diff --git a/configs/stock_lposs.yaml b/configs/stock_lposs.yaml index cff9438..c17fa3c 100644 --- a/configs/stock_lposs.yaml +++ b/configs/stock_lposs.yaml @@ -9,7 +9,7 @@ clip: image_size: 224 dino: - repository: facebookresearch/dino:main + repository: facebookresearch/dino:7c446df5b9f45747937fb0d72314eb9f7b66930a model: dino_vitb16 source: github weights: null @@ -27,7 +27,10 @@ graph: r: 13 max_graph_nodes: 250000 max_dense_graph_bytes: 8589934592 - max_pixel_nodes: 20000000 + max_pixel_nodes: 2000000 + max_pixel_edges: 250000000 + max_pixel_graph_bytes: 8589934592 + gpu_memory_reserve_bytes: 1073741824 pixel_refine: false diff --git a/configs/stock_lposs_plus.yaml b/configs/stock_lposs_plus.yaml index 13a4572..f1ebe26 100644 --- a/configs/stock_lposs_plus.yaml +++ b/configs/stock_lposs_plus.yaml @@ -9,7 +9,7 @@ clip: image_size: 224 dino: - repository: facebookresearch/dino:main + repository: facebookresearch/dino:7c446df5b9f45747937fb0d72314eb9f7b66930a model: dino_vitb16 source: github weights: null @@ -27,7 +27,10 @@ graph: r: 13 max_graph_nodes: 250000 max_dense_graph_bytes: 8589934592 - max_pixel_nodes: 20000000 + max_pixel_nodes: 2000000 + max_pixel_edges: 250000000 + max_pixel_graph_bytes: 8589934592 + gpu_memory_reserve_bytes: 1073741824 pixel_refine: true diff --git a/docs/stock_ovs_lposs_inference.md b/docs/stock_ovs_lposs_inference.md index 46b2afb..1c36d01 100644 --- a/docs/stock_ovs_lposs_inference.md +++ b/docs/stock_ovs_lposs_inference.md @@ -18,7 +18,7 @@ Only these dedicated configurations are stock references: Both pin implementation ID `stock-maskclip-lposs-p1a-v1`, upstream repository/commit, OpenCLIP `ViT-B-16` with `laion2b_s34b_b88k`, and the original -`facebookresearch/dino:main` `dino_vitb16` Torch Hub entrypoint. Every manifest records these +`facebookresearch/dino:7c446df5b9f45747937fb0d72314eb9f7b66930a` `dino_vitb16` Torch Hub entrypoint. Every manifest records these identifiers, resolved graph values, available local weight hashes, device/dependency facts, ontology/prototype metadata, and artifact hashes. @@ -99,20 +99,108 @@ read back, validated, and hashed. Existing scientific runs are never overwritten The ledger begins before model loading, records snapshots and stage transitions, registers verified artifacts with `ArtifactDescriptor`, and ends in `run.completed` or sanitized `run.failed`. -## Verification status and opt-in parity +## Verification ladder and exact commands -CPU tests use deterministic fake features/graphs to verify contracts and are **not numerical -upstream parity evidence**. A real parity run was not executed in the PR environment. Generate -reference tensors from the pinned official checkout using the same image, OpenCLIP/DINO caches, -vocabulary, and graph settings, then run: +The following outcomes are deliberately distinct. First create a deterministic 320×320 RGB input; +this yields exactly 400 ViT-B/16 patch nodes, so the upstream stock `k=400` is not reduced: ```bash -python tools/check_stock_lposs_gpu.py --image /data/fixed-small.png \ - --upstream-root /src/LPOSS-e489a7445528922ddfe4e39631ef2fe34827c873 \ - --upstream-scores /data/pinned-upstream-scores.pt --device cuda:0 \ - --work-dir outputs/parity-e489a744 +mkdir -p runs/lposs-fixture +python - <<'PYFIXTURE' +from pathlib import Path +import numpy as np +from PIL import Image +y, x = np.indices((320, 320)) +rgb = np.stack(((x * 13 + y * 3) % 256, (x * 5 + y * 11) % 256, + (x * 7 + y * 17) % 256), axis=-1).astype(np.uint8) +Image.fromarray(rgb, "RGB").save(Path("runs/lposs-fixture/deterministic-320.png")) +PYFIXTURE ``` -The opt-in check verifies the checkout commit, runs all three modes, verifies stages, finiteness, -device/grid/output IDs, and compares raw seeds and propagated scores with explicit tolerances. -Successful execution is implementation evidence only, not a scientific-quality claim. +1. **CPU unit and contract tests** (no model download and not numerical parity): + + ```bash + pytest -q ovs_heritage/tests + ``` + +2. **Local CPU configuration/preflight smoke** (no model execution): + + ```bash + python - <<'PYCONTRACT' +from pathlib import Path +from ovs_heritage.infer_ovs import load_config +from ovs_heritage.stock_lposs import DeviceInfo, patch_graph_preflight, pixel_graph_preflight +cfg = load_config(Path("configs/stock_lposs_plus.yaml")) +p = {**cfg["graph"], "available_gpu_bytes": 16 * 1024**3, "gpu_memory_reserve_bytes": 1024**3} +d = DeviceInfo("cuda:0", "cpu", 0, "contract-only") +print(patch_graph_preflight(400, 14, 4, d, p)) +print(pixel_graph_preflight(320, 320, 14, 4, d, p)) +PYCONTRACT + ``` + +3. **Real CUDA model execution** (successful local execution, not upstream parity): + + ```bash + python -m ovs_heritage.infer_ovs --image runs/lposs-fixture/deterministic-320.png \ + --model-config configs/stock_lposs.yaml --mode lposs --device cuda:0 \ + --vocabulary ovs_heritage/configs/heritage_vocab.yaml --ornament-threshold 0.5 \ + --output-dir runs/lposs-smoke --save-scores + ``` + +4. **Pinned official-reference generation and all-mode numerical comparison**: + + ```bash + git clone https://github.com/vladan-stojnic/LPOSS third_party/LPOSS + git -C third_party/LPOSS checkout e489a7445528922ddfe4e39631ef2fe34827c873 + test "$(git -C third_party/LPOSS status --porcelain)" = "" + git init --bare runs/dino-revision-check.git + git -C runs/dino-revision-check.git fetch --depth=1 \ + https://github.com/facebookresearch/dino.git 7c446df5b9f45747937fb0d72314eb9f7b66930a + python tools/check_stock_lposs_gpu.py --upstream-root third_party/LPOSS \ + --image runs/lposs-fixture/deterministic-320.png --device cuda:0 \ + --work-dir runs/lposs-parity --atol 1e-5 --rtol 1e-4 + ``` + + The checker launches the unmodified official checkout in an isolated subprocess, injects the + exact same versioned prototype tensor into both implementations, validates checkout/tree/model/ + input/configuration/prototype provenance, compares normalized dense CLIP and DINO features, and + compares raw, LPOSS, and LPOSS+ score stages. It never + accepts an externally supplied official score file. `parity_manifest.json` is created with + `real_gpu_parity=true` only after every required real-CUDA comparison succeeds; any missing or + mismatched stage exits non-zero. + + The harness being implemented and CPU-contract-tested is not evidence that parity passed. Until + this command completes on a compatible CUDA/CuPy/FAISS-GPU environment and writes the validated + manifest, real upstream/local numerical parity remains pending. + +5. **Stock dataset-v2 `lposs` inference**: + + ```bash + python -m ovs_heritage.infer_ovs --manifest dataset-v2/test.jsonl \ + --model-config configs/stock_lposs.yaml --mode lposs --device cuda:0 \ + --vocabulary ovs_heritage/configs/heritage_vocab.yaml --ornament-threshold 0.5 \ + --output-dir runs/dataset-v2-lposs --save-scores + ``` + +6. **Stock dataset-v2 `lposs_plus` inference**: + + ```bash + python -m ovs_heritage.infer_ovs --manifest dataset-v2/test.jsonl \ + --model-config configs/stock_lposs_plus.yaml --mode lposs_plus --device cuda:0 \ + --vocabulary ovs_heritage/configs/heritage_vocab.yaml --ornament-threshold 0.5 \ + --output-dir runs/dataset-v2-lposs-plus --save-scores + ``` + +7. **Later scientific dataset-v2 benchmark**: only after successful model execution and real + numerical parity, use the project evaluation tooling against the sealed facade-disjoint split. + This PR does not run or claim that benchmark. + +Direct `--image`/`--image-dir` records explicitly report unavailable dataset/split metadata. Generic +JSONL is supported but marked non-canonical; canonical dataset-v2 status requires a validated schema, +ontology, facade, split, image path, and the two canonical mask paths emitted by the converter. +Declared dataset-v2 JSONL is checked once, as a complete manifest, by the converter's authoritative +`validate_manifest` implementation; missing artifacts, invalid mask domains, invalid splits, and +image/mask grid mismatches fail closed rather than producing canonical provenance. +`facade_disjoint_split_verified` remains false for a single inference manifest because cross-split +validation evidence is unavailable. The original JSON mapping and its source manifest path, line, and +single computed manifest hash are retained in the existing ledger. diff --git a/ovs_heritage/infer_ovs.py b/ovs_heritage/infer_ovs.py index 84a3c9a..af90838 100644 --- a/ovs_heritage/infer_ovs.py +++ b/ovs_heritage/infer_ovs.py @@ -4,6 +4,7 @@ import argparse from dataclasses import asdict, dataclass from hashlib import sha256 +from importlib import metadata as package_metadata import json import math import os @@ -22,7 +23,8 @@ from .metadata import make_metadata from .ontology import load_ontology from .projection import MAIN_SEMANTIC_IDS, OntologyProjection -from .stock_features import StockFeatureModel, optional_weight_hash +from .stock_features import (StockFeatureModel, compatible_torch_load, + DINO_REPOSITORY, model_state_sha256, optional_weight_hash) from .stock_lposs import ( IMPLEMENTATION_ID, MODES, @@ -33,7 +35,7 @@ metadata_dict, resolve_device, ) -from .vocabulary import RuntimeClass, build_prototypes, heritage_runtime_vocabulary +from .vocabulary import PrototypeSet, RuntimeClass, build_prototypes, heritage_runtime_vocabulary UPSTREAM_REPOSITORY = "https://github.com/vladan-stojnic/LPOSS" SAFE_SAMPLE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") @@ -44,6 +46,7 @@ class InputSample: sample_id: str image_path: Path + provenance: dict[str, Any] | None = None def parse_args(argv=None): @@ -67,6 +70,8 @@ def parse_args(argv=None): parser.add_argument("--ornament-threshold", type=float) parser.add_argument("--ledger-dir") parser.add_argument("--run-id") + parser.add_argument("--prototype-artifact", help=argparse.SUPPRESS) + parser.add_argument("--parity-feature-artifact", action="store_true", help=argparse.SUPPRESS) return parser.parse_args(argv) @@ -83,6 +88,16 @@ def canonical_hash(value: Any) -> str: allow_nan=False).encode()).hexdigest() +def installed_versions() -> dict[str, str | None]: + result = {} + for distribution in ("open-clip-torch", "torch", "faiss-gpu", "faiss-cpu", "cupy-cuda11x", "cupy-cuda12x"): + try: + result[distribution] = package_metadata.version(distribution) + except package_metadata.PackageNotFoundError: + result[distribution] = None + return result + + def load_config(path: Path) -> dict[str, Any]: try: config = yaml.safe_load(path.read_text(encoding="utf-8")) @@ -96,9 +111,20 @@ def load_config(path: Path) -> dict[str, Any]: raise ValueError("incompatible stock implementation identifier") if config["upstream_repository"] != UPSTREAM_REPOSITORY or config["upstream_commit"] != UPSTREAM_COMMIT: raise ValueError("stock config has an incompatible upstream LPOSS reference") - expected_dino = {"model": "dino_vitb16", "patch_size": 16, "feature_type": "v"} - if any(config["dino"].get(key) != value for key, value in expected_dino.items()): - raise ValueError("stock config must use upstream DINO ViT-B/16 value features") + expected = { + "clip": {"model": "ViT-B-16", "pretrained": "laion2b_s34b_b88k", "patch_size": 16, "image_size": 224}, + "dino": {"repository": DINO_REPOSITORY, + "model": "dino_vitb16", "source": "github", "weights": None, + "architecture": "vit_base", "patch_size": 16, "feature_type": "v"}, + "graph": {"alpha": .95, "gamma": 3.0, "k": 400, "sigma": .01, + "pix_dist_pow": 1.0, "tau": .01, "r": 13}, + "slide": {"crop_size": [512, 512], "stride": [341, 341]}, + } + for section, values in expected.items(): + if not isinstance(config.get(section), dict) or any(config[section].get(k) != v for k, v in values.items()): + raise ValueError(f"stock config changes scientifically meaningful {section} parameters") + if not isinstance(config["pixel_refine"], bool): + raise ValueError("pixel_refine must be boolean") return config @@ -110,24 +136,30 @@ def _sample_id(value: Any, path: Path, line: int | None = None) -> str: return sample -def discover_inputs(args) -> list[InputSample]: +def discover_inputs(args, ontology=None) -> list[InputSample]: samples = [] if args.image: path = Path(args.image).expanduser().resolve() - samples.append(InputSample(_sample_id(None, path), path)) + samples.append(InputSample(_sample_id(None, path), path, + {"input_kind": "image", "dataset_metadata_available": False})) elif args.image_dir: directory = Path(args.image_dir).expanduser().resolve() if not directory.is_dir(): raise ValueError(f"image directory does not exist: {directory}") for path in sorted(directory.iterdir()): if path.is_file() and path.suffix.lower() in IMAGE_SUFFIXES: - samples.append(InputSample(_sample_id(None, path), path.resolve())) + samples.append(InputSample(_sample_id(None, path), path.resolve(), + {"input_kind": "image_dir", "dataset_metadata_available": False})) else: manifest = Path(args.manifest).expanduser().resolve() try: lines = manifest.read_text(encoding="utf-8").splitlines() except OSError as exc: raise ValueError(f"cannot read manifest {manifest}: {exc}") from exc + manifest_sha256 = file_hash(manifest) + parsed = [] + from .ontology import V2_VERSION + from .validate_dataset import V2_DATASET_SCHEMA for line_number, line in enumerate(lines, 1): if not line.strip(): continue @@ -135,12 +167,46 @@ def discover_inputs(args) -> list[InputSample]: item = json.loads(line) except json.JSONDecodeError as exc: raise ValueError(f"{manifest}: malformed JSON at line {line_number}: {exc.msg}") from exc + parsed.append((line_number, item)) + declares_v2 = any(isinstance(item, dict) and ( + item.get("schema_version") == V2_DATASET_SCHEMA + or item.get("ontology_version") == V2_VERSION) for _, item in parsed) + canonical = False + if declares_v2: + from .coco_converter import validate_manifest + validation = validate_manifest(manifest) + if not validation["valid"]: + failures = "; ".join(validation["errors"][:8]) + if len(validation["errors"]) > 8: + failures += f"; and {len(validation['errors']) - 8} more" + raise ValueError(f"canonical dataset-v2 manifest validation failed: {failures}") + canonical = True + for line_number, item in parsed: if not isinstance(item, dict) or not isinstance(item.get("image_path"), str) or not item["image_path"].strip(): raise ValueError(f"{manifest}: line {line_number} requires canonical image_path") path = Path(item["image_path"]).expanduser() path = path if path.is_absolute() else manifest.parent / path sample = item.get("sample_id", item.get("source_id", item.get("image_id"))) - samples.append(InputSample(_sample_id(sample, path, line_number), path.resolve())) + canonical_fields = { + "schema_version": item.get("schema_version"), + "ontology_version": item.get("ontology_version"), + "facade_id": item.get("facade_id"), "split": item.get("split"), + "image_path": item.get("image_path"), + "main_mask_path": item.get("main_mask_path"), + "ornament_mask_path": item.get("ornament_mask_path"), + } + metadata_available = canonical + if not declares_v2 and canonical_fields["schema_version"] is not None: + raise ValueError(f"{manifest}: line {line_number} has incompatible dataset schema") + if not declares_v2 and canonical_fields["ontology_version"] is not None: + raise ValueError(f"{manifest}: line {line_number} has incompatible ontology revision") + samples.append(InputSample(_sample_id(sample, path, line_number), path.resolve(), { + "input_kind": "manifest", "dataset_metadata_available": metadata_available, + "canonical_dataset_v2": canonical, "canonical_fields": canonical_fields, + "facade_disjoint_split_verified": False, + "facade_disjoint_split_evidence": "unavailable from a single inference manifest", + "source_manifest_path": str(manifest), "source_manifest_sha256": manifest_sha256, + "source_manifest_line_number": line_number, "source_record": item})) if not samples: raise ValueError("input contains no supported images") ids = [item.sample_id for item in samples] @@ -169,6 +235,13 @@ def prepare_output(path: Path, samples: list[InputSample]) -> Path: return output +def dataset_snapshot(samples: list[InputSample]) -> dict[str, Any]: + """Payload for the existing research ledger; no parallel provenance store.""" + return {"inputs": [{"sample_id": item.sample_id, "path": str(item.image_path), + "sha256": file_hash(item.image_path), "provenance": item.provenance} + for item in samples]} + + def load_runtime_classes(ontology, extra_path: str | None, contrast_path: Path): classes = list(heritage_runtime_vocabulary(ontology)) if extra_path: @@ -223,7 +296,7 @@ def _atomic_torch(path: Path, payload): temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") torch.save(payload, temporary) os.replace(temporary, path) - torch.load(path, map_location="cpu", weights_only=True) + compatible_torch_load(path, map_location="cpu") def export_sample(output: Path, sample: InputSample, result, prototypes, @@ -259,6 +332,7 @@ def export_sample(output: Path, sample: InputSample, result, prototypes, "semantic_ids": prototypes.semantic_ids, "vocabulary_hash": prototypes.vocabulary_hash, "prompt_settings": dict(prototypes.prompt_settings)} + payload["prototypes"] = prototypes.prototypes.cpu() _atomic_torch(score_path, payload) artifacts.append((score_path, "raw_score_tensors", "application/x-pytorch")) return artifacts @@ -274,6 +348,7 @@ def _ledger_stage(ledger, name, function): def main(argv=None): args = parse_args(argv) + torch.manual_seed(0) validate_threshold(args.ornament_threshold) if args.ornament_threshold is None: raise ValueError("--ornament-threshold is required for the canonical binary output") @@ -283,9 +358,12 @@ def main(argv=None): raise ValueError("mode/config mismatch: use stock_lposs_plus.yaml only with lposs_plus") if args.mode == "lposs" and config["pixel_refine"]: raise ValueError("lposs requires stock_lposs.yaml with pixel_refine:false") - samples = discover_inputs(args) - output = prepare_output(Path(args.output_dir), samples) ontology = load_ontology(args.vocabulary) + samples = discover_inputs(args, ontology) + if args.parity_feature_artifact and (len(samples) != 1 or args.inference != "whole" + or args.mode == "maskclip_raw"): + raise ValueError("parity feature capture requires one whole-inference graph mode sample") + output = prepare_output(Path(args.output_dir), samples) projection = OntologyProjection.from_ontology(ontology) classes, contrast = load_runtime_classes( ontology, args.extra_concepts, Path(args.ornament_contrast).resolve()) @@ -311,8 +389,7 @@ def main(argv=None): "upstream_commit": UPSTREAM_COMMIT}), ("config.snapshot", {"config": config, "canonical_hash": config_hash, "path": str(config_path), "sha256": file_hash(config_path)}), - ("dataset.snapshot", {"inputs": [{"sample_id": item.sample_id, - "path": str(item.image_path), "sha256": file_hash(item.image_path)} for item in samples]}), + ("dataset.snapshot", dataset_snapshot(samples)), ("environment.snapshot", {"device": asdict(device), "torch": torch.__version__, "cuda": torch.version.cuda, "graph_dependencies": "pending mode-specific preflight"}), @@ -334,6 +411,7 @@ def build_feature_model(): clip = config["clip"] model = StockFeatureModel(clip_model=clip["model"], clip_pretrained=clip["pretrained"], patch_size=clip["patch_size"], image_size=clip["image_size"]) + model.parity_capture_enabled = args.parity_feature_artifact model.to(device.resolved_device).eval() if args.mode != "maskclip_raw": dino = config["dino"] @@ -343,8 +421,28 @@ def build_feature_model(): model.to(device.resolved_device).eval() return model model, _ = _ledger_stage(ledger, "model_loading", build_feature_model) - prototypes, _ = _ledger_stage(ledger, "prototype_construction", lambda: build_prototypes( - classes, model.encode_text, device=device.resolved_device, ontology_hash=ontology.hash)) + model_fingerprints = {"clip_state_sha256": model_state_sha256(model.clip), + "dino_state_sha256": (model_state_sha256(model.dino_encoder) + if model.dino_encoder is not None else None)} + def construct_prototypes(): + if not args.prototype_artifact: + return build_prototypes(classes, model.encode_text, + device=device.resolved_device, ontology_hash=ontology.hash) + artifact = np.load(args.prototype_artifact, allow_pickle=False) + metadata = json.loads(str(artifact["metadata_json"].item())) + expected_names = [item.name for item in classes] + expected_ids = [item.semantic_id for item in classes] + if metadata.get("schema_version") != "lposs-prototypes-v1" or metadata.get("channel_names") != expected_names: + raise ValueError("prototype artifact channel ordering is incompatible") + if metadata.get("semantic_ids") != expected_ids or metadata.get("ontology_hash") != ontology.hash: + raise ValueError("prototype artifact semantic IDs or ontology hash are incompatible") + tensor = torch.from_numpy(artifact["prototypes"]).to(device.resolved_device) + digest = sha256(artifact["prototypes"].tobytes(order="C")).hexdigest() + if digest != metadata.get("prototypes_sha256"): + raise ValueError("prototype artifact fingerprint mismatch") + return PrototypeSet(tensor, tuple(expected_names), tuple(expected_ids), + metadata["vocabulary_specification_hash"], ontology.hash, metadata["prompt_settings"]) + prototypes, _ = _ledger_stage(ledger, "prototype_construction", construct_prototypes) graph = dict(config["graph"]) graph["vit_patch_size"] = config["dino"]["patch_size"] engine = StockOVSEngine(model, UpstreamGraphPropagator() if args.mode != "maskclip_raw" else None, @@ -359,19 +457,28 @@ def run_inference(): image = torch.from_numpy(rgb).permute(2, 0, 1).unsqueeze(0) if device.logical_index is not None: torch.cuda.reset_peak_memory_stats(device.logical_index) + torch.cuda.synchronize(device.logical_index) + free_before, total_memory = torch.cuda.mem_get_info(device.logical_index) + else: + free_before = total_memory = None started = time.perf_counter() result = engine.run(image, prototypes, mode=args.mode, inference=args.inference, crop_size=crop, stride=stride, ornament_negative_index=len(classes) - 1, ornament_threshold=args.ornament_threshold) + if device.logical_index is not None: + torch.cuda.synchronize(device.logical_index) + free_after, _ = torch.cuda.mem_get_info(device.logical_index) + else: + free_after = None peak = (torch.cuda.max_memory_allocated(device.logical_index) if device.logical_index is not None else None) completed.append((sample, result, rgb.shape[:2], - time.perf_counter() - started, peak)) + time.perf_counter() - started, peak, free_before, free_after, total_memory)) _, inference_event = _ledger_stage(ledger, "inference_and_propagation", run_inference) def run_export(): - for sample, result, original_size, elapsed, peak in completed: + for sample, result, original_size, elapsed, peak, free_before, free_after, total in completed: artifacts = export_sample( output, sample, result, prototypes, args.save_scores, args.visualize) artifact_rows.extend(artifacts) @@ -379,9 +486,24 @@ def run_export(): "image_sha256": file_hash(sample.image_path), "original_size": list(original_size), "output_grid": list(result.propagated_scores.shape[-2:]), "execution": metadata_dict(result), "elapsed_seconds": elapsed, - "peak_gpu_bytes": peak, + "provenance": sample.provenance, + "peak_pytorch_allocated_bytes": peak, + "device_memory": {"free_before_bytes": free_before, "free_after_bytes": free_after, + "total_bytes": total, "limitation": "snapshots are not total peak GPU memory; PyTorch peak excludes CuPy and FAISS"}, "artifacts": [{"path": str(path), "byte_size": path.stat().st_size, "sha256": file_hash(path)} for path, _, _ in artifacts]}) + if args.parity_feature_artifact: + clip_nodes, dino_nodes, patch_grid = model.parity_features() + feature_path = output / "parity_features.npz" + temporary = output / f".parity-features.{uuid4().hex}.tmp" + with temporary.open("wb") as stream: + np.savez(stream, clip_features=clip_nodes.cpu().numpy(), + dino_features=dino_nodes.cpu().numpy(), + patch_grid=np.asarray(patch_grid, dtype=np.int64), + image_grid=np.asarray(completed[0][2], dtype=np.int64)) + os.replace(temporary, feature_path) + artifact_rows.append((feature_path, "parity_normalized_dense_features", + "application/x-npz")) _, export_event = _ledger_stage(ledger, "export", run_export) manifest = {"schema_version": "stock-ovs-lposs-run-v1", "run_id": run_id, "stock_implementation": IMPLEMENTATION_ID, "upstream_repository": UPSTREAM_REPOSITORY, @@ -390,15 +512,20 @@ def run_export(): "clip_weight_sha256": optional_weight_hash(config["clip"]["pretrained"]), "dino_repository": config["dino"]["repository"], "dino_model": config["dino"]["model"], "dino_weights": config["dino"]["weights"], - "dino_weight_sha256": optional_weight_hash(config["dino"]["weights"])}, - "device": asdict(device), "dependencies": dependencies, "graph_parameters": graph, + "dino_weight_sha256": optional_weight_hash(config["dino"]["weights"]), + **model_fingerprints}, + "device": asdict(device), "dependencies": dependencies, + "installed_versions": installed_versions(), "graph_parameters": graph, "ontology_hash": ontology.hash, "projection": projection.as_dict(), + "projection_hash": canonical_hash(projection.as_dict()), "vocabulary_hash": prototypes.vocabulary_hash, "channel_names": list(prototypes.channel_names), "semantic_ids": list(prototypes.semantic_ids), "prompt_settings": dict(prototypes.prompt_settings), "runtime_vocabulary": [{"name": item.name, "semantic_id": item.semantic_id, "prompts": list(item.prompts), "aliases": list(item.aliases)} for item in classes], "ornament_contrast": contrast, "ornament_threshold": args.ornament_threshold, + "determinism": {"seed": 0, "policy": "inference-only; fixed seed; no stochastic augmentation", + "torch_deterministic_algorithms": False}, "records": records} manifest_path = output / "run_manifest.json" temporary = output / f".manifest.{uuid4().hex}.tmp" diff --git a/ovs_heritage/stock_features.py b/ovs_heritage/stock_features.py index a01af48..b731912 100644 --- a/ovs_heritage/stock_features.py +++ b/ovs_heritage/stock_features.py @@ -8,6 +8,25 @@ import torch.nn.functional as F from torch import nn +DINO_REPOSITORY_COMMIT = "7c446df5b9f45747937fb0d72314eb9f7b66930a" +DINO_REPOSITORY = f"facebookresearch/dino:{DINO_REPOSITORY_COMMIT}" + + +def validate_pinned_hub_repository(repository: str) -> None: + """Reject moving or malformed Torch Hub references before any download.""" + if repository != DINO_REPOSITORY: + raise ValueError(f"DINO repository must be immutable {DINO_REPOSITORY}, got {repository!r}") + + +def compatible_torch_load(path, *, map_location="cpu"): + """Load tensor-only artifacts on both torch 1.12 and newer releases.""" + try: + return torch.load(path, map_location=map_location, weights_only=True) + except TypeError as exc: + if "weights_only" not in str(exc): + raise + return torch.load(path, map_location=map_location) + class StockFeatureModel(nn.Module): """Load image/text CLIP once; runtime classes never enter module state.""" @@ -34,6 +53,9 @@ def __init__(self, *, clip_model: str, clip_pretrained: str, self.dino_patch_size = None self.dino_feature_type = None self._dino_hook = {} + self.parity_capture_enabled = False + self._parity_clip_dense = None + self._parity_dino_dense = None @torch.no_grad() def encode_text(self, prompts: list[str]) -> torch.Tensor: @@ -61,12 +83,16 @@ def clip_dense(self, image: torch.Tensor) -> torch.Tensor: value = visual.ln_post(value.permute(1, 0, 2)).permute(1, 0, 2)[:, 1:] value = value.reshape(b, *grid, -1).permute(0, 3, 1, 2).contiguous() projection = visual.proj - return F.conv2d(value, projection.t()[:, :, None, None]) + output = F.conv2d(value, projection.t()[:, :, None, None]) + if self.parity_capture_enabled: + self._parity_clip_dense = output.detach() + return output finally: visual.positional_embedding.data = saved def configure_dino(self, *, repository: str, model: str, patch_size: int, feature_type: str, source: str = "github", weights: str | None = None): + validate_pinned_hub_repository(repository) kwargs = {"source": source} if weights is not None: kwargs["weights"] = weights @@ -96,7 +122,22 @@ def dino_dense(self, image: torch.Tensor) -> torch.Tensor: qkv = qkv.reshape(qkv.shape[0], qkv.shape[1], 3, heads, -1).permute(2, 0, 3, 1, 4) selected = {"q": 0, "k": 1, "v": 2}[self.dino_feature_type] features = qkv[selected][:, :, 1:, :].permute(0, 2, 1, 3).flatten(-2) - return F.normalize(features, dim=-1).reshape(batch.shape[0], h, w, -1).permute(0, 3, 1, 2) + output = F.normalize(features, dim=-1).reshape(batch.shape[0], h, w, -1).permute(0, 3, 1, 2) + if self.parity_capture_enabled: + self._parity_dino_dense = output.detach() + return output + + def parity_features(self) -> tuple[torch.Tensor, torch.Tensor, tuple[int, int]]: + """Return normalized aligned patch nodes only when explicitly enabled.""" + if not self.parity_capture_enabled or self._parity_clip_dense is None or self._parity_dino_dense is None: + raise RuntimeError("parity feature capture did not execute both CLIP and DINO") + dino = self._parity_dino_dense + clip = self._parity_clip_dense + if clip.shape[-2:] != dino.shape[-2:]: + clip = F.interpolate(clip, dino.shape[-2:], mode="bilinear", align_corners=False) + clip_nodes = F.normalize(clip.permute(0, 2, 3, 1).reshape(-1, clip.shape[1]), dim=-1) + dino_nodes = F.normalize(dino.permute(0, 2, 3, 1).reshape(-1, dino.shape[1]), dim=-1) + return clip_nodes, dino_nodes, tuple(dino.shape[-2:]) def _resize_position(self, grid): position = self._original_position.to(next(self.clip.parameters()).device) @@ -126,6 +167,18 @@ def module_state_fingerprint(module: nn.Module) -> tuple[tuple[str, tuple[int, . return tuple(result) +def model_state_sha256(module: nn.Module) -> str: + """Hash state deterministically, moving only one tensor at a time to the CPU.""" + digest = sha256() + for key, value in sorted(module.state_dict().items()): + cpu = value.detach().cpu().contiguous() + digest.update(key.encode()) + digest.update(str(tuple(cpu.shape)).encode()) + digest.update(str(cpu.dtype).encode()) + digest.update(cpu.view(torch.uint8).numpy().tobytes()) + return digest.hexdigest() + + def optional_weight_hash(identifier: str | None) -> str | None: if not identifier: return None diff --git a/ovs_heritage/stock_lposs.py b/ovs_heritage/stock_lposs.py index ef45677..647a4f7 100644 --- a/ovs_heritage/stock_lposs.py +++ b/ovs_heritage/stock_lposs.py @@ -58,6 +58,8 @@ class ExecutionMetadata: window_count: int graph_nodes: int | None estimated_dense_graph_bytes: int | None + estimated_pixel_graph_edges: int | None + estimated_pixel_graph_bytes: int | None crop_size: tuple[int, int] | None stride: tuple[int, int] | None @@ -125,6 +127,52 @@ def graph_preflight(mode: str, device: DeviceInfo) -> dict[str, str | None]: "faiss": getattr(faiss, "__version__", "unknown")} +def _available_graph_bytes(device: DeviceInfo, parameters: dict) -> tuple[int, int]: + reserve = int(parameters.get("gpu_memory_reserve_bytes", 1024**3)) + if "available_gpu_bytes" in parameters: # deterministic contract-test override + free = int(parameters["available_gpu_bytes"]) + else: + free, _ = torch.cuda.mem_get_info(device.logical_index) + return free, max(0, free - reserve) + + +def patch_graph_preflight(nodes: int, channels: int, element_size: int, + device: DeviceInfo, parameters: dict) -> int: + """Conservative peak for affinity, distances, masks, Laplacian and solver work.""" + dense = nodes * nodes + estimated = dense * (element_size * 6 + 2) + nodes * channels * element_size * 4 + free, usable = _available_graph_bytes(device, parameters) + max_nodes = int(parameters.get("max_graph_nodes", 250_000)) + max_bytes = int(parameters.get("max_dense_graph_bytes", 8 * 1024**3)) + if nodes > max_nodes or estimated > max_bytes or estimated > usable: + raise RuntimeError("patch graph preflight rejected: " + f"nodes={nodes}, edges={dense}, estimated_bytes={estimated}, available_bytes={free}, " + f"usable_after_reserve_bytes={usable}; reduce image/window count or select a larger GPU; " + "stock inference will not resize automatically") + return estimated + + +def pixel_graph_preflight(height: int, width: int, channels: int, element_size: int, + device: DeviceInfo, parameters: dict) -> tuple[int, int]: + nodes = height * width + radius = int(parameters["r"]) // 2 + neighbours = (2 * radius + 1) ** 2 - 1 + edges = nodes * neighbours # safe upper bound; boundaries only reduce it + # COO row/column + value, two sparse matrices, CSR work, labels and solver vectors. + estimated = edges * (8 * 2 + element_size) * 3 + (nodes + 1) * 8 * 3 + estimated += nodes * channels * element_size * 5 + free, usable = _available_graph_bytes(device, parameters) + limits = (int(parameters.get("max_pixel_nodes", 2_000_000)), + int(parameters.get("max_pixel_edges", 250_000_000)), + int(parameters.get("max_pixel_graph_bytes", 8 * 1024**3))) + if nodes > limits[0] or edges > limits[1] or estimated > limits[2] or estimated > usable: + raise RuntimeError("pixel graph preflight rejected: " + f"nodes={nodes}, edges={edges}, estimated_bytes={estimated}, available_bytes={free}, " + f"usable_after_reserve_bytes={usable}; reduce input size explicitly or select a larger GPU; " + "stock inference will not resize automatically") + return edges, estimated + + def _windows(h: int, w: int, crop: tuple[int, int], stride: tuple[int, int]): ch, cw = crop sh, sw = stride @@ -174,6 +222,7 @@ def run(self, image: torch.Tensor, prototypes: PrototypeSet, *, mode: str, window_seed_maps = self.scorer(clip, prototypes.prototypes) dino_done = patch_done = pixel_done = False effective_k = graph_nodes = estimated_graph_bytes = None + pixel_edges = estimated_pixel_bytes = None if mode == "maskclip_raw": window_outputs = [window_seed_maps[i:i + 1] for i in range(len(windows))] else: @@ -188,15 +237,8 @@ def run(self, image: torch.Tensor, prototypes: PrototypeSet, *, mode: str, dino_nodes = torch.cat([item.permute(1, 2, 0).reshape(-1, item.shape[0]) for item in dino]) seed_nodes = torch.cat([item.permute(1, 2, 0).reshape(-1, item.shape[0]) for item in seeds]) graph_nodes = dino_nodes.shape[0] - estimated_graph_bytes = graph_nodes * graph_nodes * dino_nodes.element_size() - max_nodes = int(self.parameters.get("max_graph_nodes", 250_000)) - if graph_nodes > max_nodes: - raise RuntimeError(f"resolved graph has {graph_nodes} nodes, exceeding max_graph_nodes={max_nodes}") - max_bytes = int(self.parameters.get("max_dense_graph_bytes", 8 * 1024**3)) - if estimated_graph_bytes > max_bytes: - raise RuntimeError( - f"estimated dense geometry graph requires {estimated_graph_bytes} bytes, " - f"exceeding max_dense_graph_bytes={max_bytes}") + estimated_graph_bytes = patch_graph_preflight( + graph_nodes, seed_nodes.shape[1], dino_nodes.element_size(), self.device, self.parameters) locations = torch.tensor(windows, device=target, dtype=torch.long) context = torch.cuda.device(self.device.logical_index) with context: @@ -213,12 +255,9 @@ def run(self, image: torch.Tensor, prototypes: PrototypeSet, *, mode: str, seeds_full = self._stitch(window_seed_maps, windows, original) scores = self._stitch(window_outputs, windows, original) if mode == "lposs_plus": - pixel_nodes = original[0] * original[1] - max_pixel_nodes = int(self.parameters.get("max_pixel_nodes", 20_000_000)) - if pixel_nodes > max_pixel_nodes: - raise RuntimeError( - f"LPOSS+ pixel graph has {pixel_nodes} nodes, exceeding " - f"max_pixel_nodes={max_pixel_nodes}") + pixel_edges, estimated_pixel_bytes = pixel_graph_preflight( + original[0], original[1], scores.shape[1], scores.element_size(), + self.device, self.parameters) with torch.cuda.device(self.device.logical_index): scores = self.propagator.pixel(image, scores, parameters=self.parameters, device_index=self.device.logical_index) @@ -230,6 +269,7 @@ def run(self, image: torch.Tensor, prototypes: PrototypeSet, *, mode: str, mode, mode, dino_done, patch_done, pixel_done, self.scorer.scale, self.parameters.get("k"), effective_k, inference, len(windows), graph_nodes, estimated_graph_bytes, + pixel_edges, estimated_pixel_bytes, crop_size if inference == "slide" else None, stride if inference == "slide" else None) return result @@ -279,7 +319,7 @@ def _project(scores, seeds, prototypes, negative, threshold): extras = {name: scores[:, i:i + 1] for i, (name, sid) in enumerate(zip(names, ids)) if sid is None and i not in excluded} placeholder = ExecutionMetadata("", "", False, False, False, 1, None, None, - "whole", 1, None, None, None, None) + "whole", 1, None, None, None, None, None, None) return StockOutput(seeds, scores, main_scores, main_mask, contrast, probability, binary, extras, placeholder) diff --git a/ovs_heritage/tests/test_stock_lposs.py b/ovs_heritage/tests/test_stock_lposs.py index 8c1866f..f304314 100644 --- a/ovs_heritage/tests/test_stock_lposs.py +++ b/ovs_heritage/tests/test_stock_lposs.py @@ -14,6 +14,7 @@ from ovs_heritage.infer_ovs import ( InputSample, + dataset_snapshot, discover_inputs, export_sample, load_config, @@ -22,11 +23,17 @@ _ledger_stage, ) from ovs_heritage.projection import MAIN_SEMANTIC_IDS -from ovs_heritage.stock_features import StockFeatureModel, module_state_fingerprint +from ovs_heritage.ontology import load_ontology +from ovs_heritage.stock_features import ( + DINO_REPOSITORY, StockFeatureModel, module_state_fingerprint, + validate_pinned_hub_repository, +) from ovs_heritage.stock_lposs import ( DeviceInfo, StockOVSEngine, graph_preflight, + patch_graph_preflight, + pixel_graph_preflight, resolve_device, ) from ovs_heritage.vocabulary import PrototypeSet @@ -79,7 +86,8 @@ def engine(mode): instance = StockOVSEngine(configured and model or model, configured, device=DeviceInfo("cpu", "cpu", None, None) if mode == "maskclip_raw" else DeviceInfo("cuda:0", "cpu", 0, "fake"), - graph_parameters={"k": 99}) + graph_parameters={"k": 99, "r": 13, "available_gpu_bytes": 10**12, + "gpu_memory_reserve_bytes": 0}) return instance, model, propagation @@ -184,7 +192,90 @@ def test_dataset_v2_jsonl_resolves_relative_images(tmp_path): manifest = tmp_path / "manifest.jsonl" manifest.write_text('\n{"sample_id":"sample-a","image_path":"images/a.png"}\n') result = discover_inputs(_args(manifest=str(manifest))) - assert result == [InputSample("sample-a", (image_dir / "a.png").resolve())] + assert result[0].sample_id == "sample-a" + assert result[0].image_path == (image_dir / "a.png").resolve() + assert result[0].provenance["source_record"]["image_path"] == "images/a.png" + assert result[0].provenance["source_manifest_sha256"] == sha256(manifest.read_bytes()).hexdigest() + assert result[0].provenance["dataset_metadata_available"] is False + + +def test_source_manifest_hash_is_computed_once(tmp_path, monkeypatch): + import ovs_heritage.infer_ovs as inference + for name in ("a.png", "b.png"): + Image.fromarray(np.zeros((2, 2, 3), dtype=np.uint8)).save(tmp_path / name) + manifest = tmp_path / "generic.jsonl" + manifest.write_text("\n".join(json.dumps({"sample_id": name[0], "image_path": name}) + for name in ("a.png", "b.png"))) + calls = [] + original = inference.file_hash + monkeypatch.setattr(inference, "file_hash", lambda path: calls.append(Path(path)) or original(path)) + samples = discover_inputs(_args(manifest=str(manifest))) + assert calls == [manifest.resolve()] + assert {sample.provenance["source_manifest_sha256"] for sample in samples} == { + sha256(manifest.read_bytes()).hexdigest()} + snapshot = dataset_snapshot(samples) + assert snapshot["inputs"][0]["provenance"]["source_record"]["image_path"] == "a.png" + + +def _converter_manifest_fixture(tmp_path): + ontology = load_ontology() + Image.fromarray(np.zeros((2, 2, 3), dtype=np.uint8), "RGB").save(tmp_path / "a.png") + Image.fromarray(np.array([[0, 1], [2, 11]], dtype=np.uint8)).save(tmp_path / "main.png") + Image.fromarray(np.array([[0, 1], [1, 0]], dtype=np.uint8)).save(tmp_path / "ornament.png") + row = {"sample_id": "a", "image_id": 1, "source_coco_file_name": "a.png", + "canonical_file_name": "a.png", "resolved_image_path": str((tmp_path / "a.png").resolve()), + "image_path": "a.png", "main_mask_path": "main.png", "ornament_mask_path": "ornament.png", + "facade_id": "facade-a", "building_id": "building-a", "split": "test", + "schema_version": "heritage_two_map_v2", "ontology_version": ontology.version, + "source_coco_sha256": "a" * 64, "source_annotation_ids": [1], "width": 2, "height": 2} + manifest = tmp_path / "canonical.jsonl" + manifest.write_text(json.dumps(row) + "\n") + return manifest, row + + +def test_authoritatively_valid_converter_manifest_is_canonical(tmp_path, monkeypatch): + import ovs_heritage.coco_converter as converter + manifest, _ = _converter_manifest_fixture(tmp_path) + calls = [] + original = converter.validate_manifest + monkeypatch.setattr(converter, "validate_manifest", + lambda path: calls.append(path) or original(path)) + sample = discover_inputs(_args(manifest=str(manifest)))[0] + assert calls == [manifest.resolve()] + assert sample.provenance["dataset_metadata_available"] is True + assert sample.provenance["canonical_dataset_v2"] is True + assert sample.provenance["facade_disjoint_split_verified"] is False + assert sample.provenance["canonical_fields"]["facade_id"] == "facade-a" + + +@pytest.mark.parametrize("defect,expected", [ + ("missing_mask", "unreadable artifact"), + ("invalid_split", "invalid split"), + ("invalid_domain", "invalid main-mask value domain"), + ("grid_mismatch", "image/mask grid or dtype mismatch"), +]) +def test_declared_canonical_manifest_defects_fail_closed(tmp_path, defect, expected): + manifest, row = _converter_manifest_fixture(tmp_path) + if defect == "missing_mask": + (tmp_path / "main.png").unlink() + elif defect == "invalid_split": + row["split"] = "holdout" + manifest.write_text(json.dumps(row) + "\n") + elif defect == "invalid_domain": + Image.fromarray(np.full((2, 2), 8, dtype=np.uint8)).save(tmp_path / "main.png") + else: + Image.fromarray(np.zeros((1, 2), dtype=np.uint8)).save(tmp_path / "main.png") + with pytest.raises(ValueError, match=f"canonical dataset-v2 manifest validation failed:.*{expected}"): + discover_inputs(_args(manifest=str(manifest))) + + +def test_partial_dataset_identity_is_explicitly_noncanonical(tmp_path): + Image.fromarray(np.zeros((2, 2, 3), dtype=np.uint8)).save(tmp_path / "a.png") + manifest = tmp_path / "partial.jsonl" + manifest.write_text(json.dumps({"image_path": "a.png", "facade_id": "f"})) + sample = discover_inputs(_args(manifest=str(manifest)))[0] + assert sample.provenance["dataset_metadata_available"] is False + assert sample.provenance["canonical_dataset_v2"] is False @pytest.mark.parametrize("name,pixel", [("stock_lposs.yaml", False), @@ -195,6 +286,7 @@ def test_dedicated_stock_configs_pin_upstream_dino(name, pixel): assert config["dino"]["model"] == "dino_vitb16" assert config["dino"]["patch_size"] == 16 assert config["dino"]["feature_type"] == "v" + assert config["dino"]["repository"] == DINO_REPOSITORY assert config["pixel_refine"] is pixel @@ -227,6 +319,179 @@ def test_output_is_immutable_and_ornament_png_is_binary(tmp_path): prepare_output(output, [sample]) +def test_scores_are_saved_and_reloaded_with_112_compatible_loader(tmp_path, monkeypatch): + from ovs_heritage.stock_features import compatible_torch_load + sample = InputSample("sample", tmp_path / "input.png") + output = prepare_output(tmp_path / "out", [sample]) + instance, _, _ = engine("maskclip_raw") + result = instance.run(torch.rand(1, 3, 2, 3), prototype_set(), mode="maskclip_raw", + ornament_negative_index=12, ornament_threshold=0.5) + original = torch.load + def old_load(path, **kwargs): + if "weights_only" in kwargs: + raise TypeError("load() got an unexpected keyword argument 'weights_only'") + return original(path, **kwargs) + monkeypatch.setattr(torch, "load", old_load) + export_sample(output, sample, result, prototype_set(), True, False) + payload = compatible_torch_load(output / "sample" / "scores.pt") + assert torch.equal(payload["seed_scores"], result.seed_scores.cpu()) + + +def test_graph_preflights_safe_and_reject_before_propagation(monkeypatch): + monkeypatch.setattr(torch.cuda, "device", lambda _index: __import__("contextlib").nullcontext()) + device = DeviceInfo("cuda:0", "cpu", 0, "fake") + safe = {"r": 13, "available_gpu_bytes": 10**9, "gpu_memory_reserve_bytes": 0, + "max_graph_nodes": 100, "max_dense_graph_bytes": 10**9, + "max_pixel_nodes": 100, "max_pixel_edges": 10000, "max_pixel_graph_bytes": 10**9} + assert patch_graph_preflight(4, 2, 4, device, safe) > 0 + assert pixel_graph_preflight(2, 2, 2, 4, device, safe)[0] == 4 * 168 + instance, _, propagation = engine("lposs") + instance.parameters["max_graph_nodes"] = 1 + with pytest.raises(RuntimeError, match="patch graph preflight rejected"): + instance.run(torch.rand(1, 3, 5, 7), prototype_set(), mode="lposs") + assert propagation.patch_calls == 0 + instance, _, propagation = engine("lposs_plus") + instance.parameters["max_pixel_edges"] = 1 + with pytest.raises(RuntimeError, match="pixel graph preflight rejected"): + instance.run(torch.rand(1, 3, 5, 7), prototype_set(), mode="lposs_plus") + assert propagation.pixel_calls == 0 + + +def test_official_adapter_command_is_isolated_and_explicit(tmp_path): + from tools.check_stock_lposs_gpu import official_adapter_command + command = official_adapter_command(python="python", adapter=tmp_path / "adapter.py", + request=tmp_path / "request.json", output=tmp_path / "official") + assert command == ["python", str(tmp_path / "adapter.py"), "--request", + str(tmp_path / "request.json"), "--output-dir", str(tmp_path / "official")] + + +def _official_artifact(tmp_path): + from tools.check_stock_lposs_gpu import canonical_hash, file_hash + arrays = {f"{mode}.{stage}": np.ones((1, 2, 3, 4), dtype=np.float32) + for mode in ("maskclip_raw", "lposs", "lposs_plus") + for stage in ("seed_scores", "propagated_scores")} + arrays.update(clip_features=np.ones((4, 2), np.float32), + dino_features=np.ones((4, 2), np.float32)) + np.savez(tmp_path / "stages.npz", **arrays) + configurations = {"maskclip_raw": {"pixel_refine": False}, + "lposs": {"pixel_refine": False}, "lposs_plus": {"pixel_refine": True}} + expected = {"input_sha256": "input", "prototype_artifact_sha256": "file-proto", + "prototypes_sha256": "proto", "configurations": configurations, + "configuration_sha256": canonical_hash(configurations), + "channel_names": ["a", "b"], "semantic_ids": [1, None], "ontology_hash": "ontology", + "model_hashes": {"clip_state_sha256": "clip", "dino_state_sha256": "dino"}, + "device": "cuda:0", "seed": 0, "upstream": { + "repository": "https://github.com/vladan-stojnic/LPOSS", + "commit": "e489a7445528922ddfe4e39631ef2fe34827c873", "tree": "tree"}} + manifest = {**expected, "schema_version": "official-lposs-parity-artifact-v1", + "producer": "official-upstream", "upstream": expected["upstream"], + "dino_hub_loads": [{"requested_repository": "facebookresearch/dino:main", + "resolved_repository": DINO_REPOSITORY, "model": "dino_vitb16", + "args": [], "kwargs": {}}], + "patch_grid": [2, 2], "image_grid": [3, 4], + "stages": {"maskclip_raw": ["seed_scores"], + "lposs": ["seed_scores", "propagated_scores"], + "lposs_plus": ["seed_scores", "propagated_scores", "pixel_refinement"]}, + "stage_artifact": "stages.npz", "stage_artifact_sha256": file_hash(tmp_path / "stages.npz")} + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest)) + return path, expected, manifest + + +def test_official_artifact_validation_and_rejections(tmp_path): + from tools.check_stock_lposs_gpu import validate_official_artifact + path, expected, manifest = _official_artifact(tmp_path) + validated, arrays = validate_official_artifact(path, expected=expected) + assert validated["upstream"]["commit"] == "e489a7445528922ddfe4e39631ef2fe34827c873" + arrays.close() + for key, value in (("input_sha256", "wrong"), ("prototypes_sha256", "wrong")): + changed = {**manifest, key: value} + path.write_text(json.dumps(changed)) + with pytest.raises(ValueError, match="provenance mismatch"): + validate_official_artifact(path, expected=expected) + changed = {**manifest, "stages": {**manifest["stages"], "lposs_plus": ["seed_scores"]}} + path.write_text(json.dumps(changed)) + with pytest.raises(ValueError, match="incomplete"): + validate_official_artifact(path, expected=expected) + changed = {**manifest, "upstream": {**manifest["upstream"], "commit": "0" * 40}} + path.write_text(json.dumps(changed)) + with pytest.raises(ValueError, match="wrong repository"): + validate_official_artifact(path, expected=expected) + changed = {**manifest, "channel_names": ["b", "a"]} + path.write_text(json.dumps(changed)) + with pytest.raises(ValueError, match="channel_names"): + validate_official_artifact(path, expected=expected) + + +def test_correct_dino_commit_is_passed_to_torch_hub(monkeypatch): + validate_pinned_hub_repository(DINO_REPOSITORY) + with pytest.raises(ValueError, match="immutable"): + validate_pinned_hub_repository("facebookresearch/dino:main") + calls = [] + attention = types.SimpleNamespace(qkv=nn.Linear(2, 6), num_heads=1) + encoder = nn.Module() + encoder.blocks = [types.SimpleNamespace(attn=attention)] + monkeypatch.setattr(torch.hub, "load", lambda repository, model, **kwargs: + calls.append((repository, model, kwargs)) or encoder) + model = object.__new__(StockFeatureModel) + nn.Module.__init__(model) + model._dino_hook = {} + StockFeatureModel.configure_dino(model, repository=DINO_REPOSITORY, model="dino_vitb16", + patch_size=16, feature_type="v") + assert calls == [(DINO_REPOSITORY, "dino_vitb16", {"source": "github"})] + + +def test_official_constructor_boundary_and_scoped_dino_redirect(monkeypatch): + from tools.run_official_lposs_parity import construct_official_lposs + constructed = [] + downloads = [] + + class FakeOfficialLPOSS: + def __init__(self, clip_backbone, class_names, vit_arch="vit_base", + vit_patch_size=16, enc_type_feats="k"): + constructed.append((clip_backbone, class_names, vit_arch, + vit_patch_size, enc_type_feats)) + self.encoder = torch.hub.load("facebookresearch/dino:main", "dino_vitb16") + + def original(repository, model, *args, **kwargs): + downloads.append((repository, model, args, kwargs)) + return object() + monkeypatch.setattr(torch.hub, "load", original) + config = {"dino": {"architecture": "vit_base", "patch_size": 16, "feature_type": "v"}} + model, intercepted = construct_official_lposs( + FakeOfficialLPOSS, torch, class_names=["wall", "window"], config=config) + assert isinstance(model, FakeOfficialLPOSS) + assert constructed == [("maskclip", ["wall", "window"], "vit_base", 16, "v")] + assert downloads == [(DINO_REPOSITORY, "dino_vitb16", (), {})] + assert intercepted[0]["requested_repository"] == "facebookresearch/dino:main" + assert intercepted[0]["resolved_repository"] == DINO_REPOSITORY + assert torch.hub.load is original + + +@pytest.mark.parametrize("repository,model", [ + ("facebookresearch/dino:other", "dino_vitb16"), + ("facebookresearch/dino:main", "dino_vits16"), +]) +def test_official_dino_redirect_rejects_unexpected_calls_and_restores(monkeypatch, repository, model): + from tools.run_official_lposs_parity import pinned_dino_hub_load + def original(*_args, **_kwargs): + return object() + monkeypatch.setattr(torch.hub, "load", original) + with pytest.raises(RuntimeError, match="unexpected official"): + with pinned_dino_hub_load(torch, []): + torch.hub.load(repository, model) + assert torch.hub.load is original + + +def test_stock_config_rejects_changed_scientific_parameter(tmp_path): + root = Path(__file__).resolve().parents[2] + config = (root / "configs/stock_lposs.yaml").read_text().replace("alpha: 0.95", "alpha: 0.90") + path = tmp_path / "changed.yaml" + path.write_text(config) + with pytest.raises(ValueError, match="scientifically meaningful graph"): + load_config(path) + + @pytest.mark.parametrize("threshold", [-0.1, 1.1, float("nan"), float("inf")]) def test_threshold_validation(threshold): with pytest.raises(ValueError, match="finite"): diff --git a/segmentation/evaluation/lposs_eval.py b/segmentation/evaluation/lposs_eval.py index 8426e11..b33d057 100644 --- a/segmentation/evaluation/lposs_eval.py +++ b/segmentation/evaluation/lposs_eval.py @@ -1,21 +1,21 @@ +import importlib +import importlib.util from itertools import product import logging -import math -import os -import time from typing import List -import importlib - +import faiss +import faiss.contrib.torch_utils +from kornia.color import rgb_to_lab +from mmcv import ConfigDict +from mmcv.parallel import DataContainer +from mmseg.models import EncoderDecoder +from mmseg.ops import resize import torch from torch import Tensor import torch.nn.functional as F log = logging.getLogger(__name__) -from mmcv.parallel import DataContainer -from mmcv import ConfigDict -from mmseg.ops import resize -from mmseg.models import EncoderDecoder _cupy_spec = importlib.util.find_spec("cupyx") if _cupy_spec is not None: @@ -31,14 +31,6 @@ "post-processing." ) -import faiss -import numpy as np -import faiss.contrib.torch_utils -from PIL import Image -import numpy as np -from kornia.color import rgb_to_lab - - def _require_cupy(): if cp is None: raise ImportError( diff --git a/tools/check_stock_lposs_gpu.py b/tools/check_stock_lposs_gpu.py index 86520d4..13f0abd 100644 --- a/tools/check_stock_lposs_gpu.py +++ b/tools/check_stock_lposs_gpu.py @@ -1,67 +1,239 @@ -"""Opt-in real-GPU contract and pinned-upstream tensor parity check.""" +"""Auditable pinned-upstream versus local LPOSS CUDA parity workflow.""" from __future__ import annotations import argparse +from copy import deepcopy +from hashlib import sha256 import json +import os from pathlib import Path import subprocess import sys +import numpy as np import torch +from ovs_heritage.stock_features import compatible_torch_load + UPSTREAM_COMMIT = "e489a7445528922ddfe4e39631ef2fe34827c873" +UPSTREAM_REPOSITORY = "https://github.com/vladan-stojnic/LPOSS" +MODES = ("maskclip_raw", "lposs", "lposs_plus") +REQUIRED_STAGES = {"maskclip_raw": {"seed_scores"}, + "lposs": {"seed_scores", "propagated_scores"}, + "lposs_plus": {"seed_scores", "propagated_scores", "pixel_refinement"}} + + +def file_hash(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_hash(value) -> str: + return sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def official_adapter_command(*, python: str, adapter: Path, request: Path, + output: Path) -> list[str]: + return [python, str(adapter), "--request", str(request), "--output-dir", str(output)] + +def verified_checkout_identity(root: Path) -> dict: + def git(*args): + return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() + commit = git("rev-parse", "HEAD") + if commit != UPSTREAM_COMMIT or git("status", "--porcelain"): + raise ValueError("upstream checkout must be at the pinned commit and clean") + if "github.com/vladan-stojnic/LPOSS" not in git("remote", "-v"): + raise ValueError("upstream checkout has no authoritative LPOSS remote") + return {"repository": UPSTREAM_REPOSITORY, "commit": commit, + "tree": git("rev-parse", "HEAD^{tree}")} -def main(): - parser = argparse.ArgumentParser() + +def validate_official_artifact(manifest_path: Path, *, expected: dict) -> tuple[dict, np.lib.npyio.NpzFile]: + """Fail closed on fabricated, stale, partial, or provenance-mismatched output.""" + manifest = json.loads(manifest_path.read_text()) + if manifest.get("schema_version") != "official-lposs-parity-artifact-v1" or manifest.get("producer") != "official-upstream": + raise ValueError("not a versioned official LPOSS parity artifact") + upstream = manifest.get("upstream", {}) + if upstream != expected.get("upstream"): + raise ValueError("official artifact has wrong repository, commit, or source-tree identity") + checks = ("input_sha256", "prototype_artifact_sha256", "prototypes_sha256", + "configurations", "configuration_sha256", "channel_names", "semantic_ids", "ontology_hash", + "model_hashes", "device", "seed") + for key in checks: + if manifest.get(key) != expected.get(key): + raise ValueError(f"official artifact provenance mismatch for {key}") + if canonical_hash(manifest["configurations"]) != manifest["configuration_sha256"]: + raise ValueError("official artifact configuration content does not match its fingerprint") + dino_loads = manifest.get("dino_hub_loads", []) + if len(dino_loads) != 1 or dino_loads[0].get("requested_repository") != "facebookresearch/dino:main": + raise ValueError("official artifact lacks the single expected DINO interception") + if (dino_loads[0].get("resolved_repository") != + "facebookresearch/dino:7c446df5b9f45747937fb0d72314eb9f7b66930a" + or dino_loads[0].get("model") != "dino_vitb16"): + raise ValueError("official artifact did not resolve DINO to the pinned model revision") + for mode, required in REQUIRED_STAGES.items(): + if not required.issubset(set(manifest.get("stages", {}).get(mode, []))): + raise ValueError(f"official artifact is incomplete for {mode}") + artifact = manifest_path.parent / manifest.get("stage_artifact", "") + if not artifact.is_file() or file_hash(artifact) != manifest.get("stage_artifact_sha256"): + raise ValueError("official stage artifact is missing or fingerprint-mismatched") + arrays = np.load(artifact, allow_pickle=False) + for mode in MODES: + for stage in ("seed_scores", "propagated_scores"): + key = f"{mode}.{stage}" + if key not in arrays or arrays[key].ndim != 4 or not np.isfinite(arrays[key]).all(): + raise ValueError(f"official artifact lacks valid {key}") + for feature in ("clip_features", "dino_features"): + if feature not in arrays or arrays[feature].ndim != 2 or not np.isfinite(arrays[feature]).all(): + raise ValueError(f"official artifact lacks valid {feature}") + if list(arrays["dino_features"].shape[:1]) != [manifest["patch_grid"][0] * manifest["patch_grid"][1]]: + raise ValueError("official DINO feature shape disagrees with patch grid") + return manifest, arrays + + +def compare_scores(reference: np.ndarray, observed: torch.Tensor, *, atol: float, rtol: float) -> float: + expected = torch.from_numpy(reference) + if expected.shape != observed.shape: + raise AssertionError(f"shape mismatch: local {tuple(observed.shape)}, official {tuple(expected.shape)}") + torch.testing.assert_close(observed.cpu(), expected, atol=atol, rtol=rtol) + return float((observed.cpu() - expected).abs().max()) + + +def main(argv=None): + parser = argparse.ArgumentParser(description="real pinned-upstream LPOSS CUDA parity") parser.add_argument("--image", required=True) parser.add_argument("--upstream-root", required=True) - parser.add_argument("--upstream-scores", required=True, - help="Pinned-upstream seed_scores and propagated_scores tensor mapping") parser.add_argument("--device", required=True) parser.add_argument("--work-dir", required=True) parser.add_argument("--atol", type=float, default=1e-5) parser.add_argument("--rtol", type=float, default=1e-4) - args = parser.parse_args() - commit = subprocess.check_output( - ["git", "-C", args.upstream_root, "rev-parse", "HEAD"], text=True).strip() - if commit != UPSTREAM_COMMIT: - raise SystemExit(f"upstream checkout is {commit}, expected {UPSTREAM_COMMIT}") + args = parser.parse_args(argv) if not torch.cuda.is_available(): - raise SystemExit("SKIP: CUDA unavailable") - root = Path(args.work_dir) - if root.exists(): - raise SystemExit(f"work directory already exists: {root}") - reference = torch.load(args.upstream_scores, map_location="cpu", weights_only=True) - outputs = {} - for mode, config in (("maskclip_raw", "configs/stock_lposs.yaml"), - ("lposs", "configs/stock_lposs.yaml"), - ("lposs_plus", "configs/stock_lposs_plus.yaml")): - target = root / mode - subprocess.run([sys.executable, "-m", "ovs_heritage.infer_ovs", "--image", args.image, - "--model-config", config, "--mode", mode, "--device", args.device, - "--vocabulary", "ovs_heritage/configs/heritage_vocab.yaml", - "--ornament-threshold", "0.5", "--output-dir", str(target), "--save-scores"], check=True) - manifest = json.loads((target / "run_manifest.json").read_text()) + raise SystemExit("CUDA unavailable: CPU contracts are not real numerical parity") + local_root = Path(__file__).resolve().parents[1] + image, upstream, work = Path(args.image).resolve(), Path(args.upstream_root).resolve(), Path(args.work_dir).resolve() + if work.exists(): + raise SystemExit(f"work directory already exists: {work}") + work.mkdir(parents=True) + upstream_identity = verified_checkout_identity(upstream) + + common = ["--image", str(image), "--device", args.device, + "--vocabulary", str(local_root / "ovs_heritage/configs/heritage_vocab.yaml"), + "--ornament-threshold", "0.5", "--save-scores"] + bootstrap = work / "local-maskclip_raw" + subprocess.run([sys.executable, "-m", "ovs_heritage.infer_ovs", *common, + "--model-config", str(local_root / "configs/stock_lposs.yaml"), "--mode", "maskclip_raw", + "--output-dir", str(bootstrap)], cwd=local_root, check=True) + bootstrap_manifest = json.loads((bootstrap / "run_manifest.json").read_text()) + bootstrap_scores = compatible_torch_load(next(bootstrap.glob("*/scores.pt"))) + prototypes_np = bootstrap_scores["prototypes"].numpy() + prototype_meta = {"schema_version": "lposs-prototypes-v1", + "channel_names": list(bootstrap_scores["channel_names"]), + "semantic_ids": list(bootstrap_scores["semantic_ids"]), + "ontology_hash": bootstrap_manifest["ontology_hash"], + "vocabulary_specification_hash": bootstrap_scores["vocabulary_hash"], + "prompt_settings": bootstrap_scores["prompt_settings"], + "prototypes_sha256": sha256(prototypes_np.tobytes(order="C")).hexdigest()} + prototype_artifact = work / "prototypes.npz" + np.savez(prototype_artifact, prototypes=prototypes_np, + metadata_json=np.asarray(json.dumps(prototype_meta, sort_keys=True))) + config = bootstrap_manifest["config"] + plus_config = deepcopy(config) + plus_config["pixel_refine"] = True + configurations = {"maskclip_raw": config, "lposs": config, "lposs_plus": plus_config} + request = {"schema_version": "official-lposs-parity-request-v1", "local_root": str(local_root), + "upstream_root": str(upstream), "image_path": str(image), "input_sha256": file_hash(image), + "prototype_artifact": str(prototype_artifact), "prototype_artifact_sha256": file_hash(prototype_artifact), + "prototypes_sha256": prototype_meta["prototypes_sha256"], "configurations": configurations, + "configuration_sha256": canonical_hash(configurations), "channel_names": prototype_meta["channel_names"], + "semantic_ids": prototype_meta["semantic_ids"], "ontology_hash": prototype_meta["ontology_hash"], + "model_hashes": {k: bootstrap_manifest["weights"][k] + for k in ("clip_state_sha256", "dino_state_sha256")}, + "device": args.device, "seed": 0, "upstream": upstream_identity} + if request["model_hashes"]["dino_state_sha256"] is None: + # Bootstrap raw mode intentionally omits DINO; run lposs once with the exact prototypes. + dino_bootstrap = work / "local-lposs" + subprocess.run([sys.executable, "-m", "ovs_heritage.infer_ovs", *common, + "--prototype-artifact", str(prototype_artifact), "--model-config", str(local_root / "configs/stock_lposs.yaml"), + "--parity-feature-artifact", "--mode", "lposs", "--output-dir", str(dino_bootstrap)], + cwd=local_root, check=True) + dino_manifest = json.loads((dino_bootstrap / "run_manifest.json").read_text()) + request["model_hashes"] = {k: dino_manifest["weights"][k] + for k in ("clip_state_sha256", "dino_state_sha256")} + request_path = work / "request.json" + request_path.write_text(json.dumps(request, indent=2, sort_keys=True) + "\n") + official_dir = work / "official" + env = os.environ.copy() + env["PYTHONPATH"] = str(upstream) + command = official_adapter_command(python=sys.executable, + adapter=local_root / "tools/run_official_lposs_parity.py", request=request_path, output=official_dir) + subprocess.run(command, cwd=upstream, env=env, check=True) + expected = {**request, "prototype_artifact_sha256": file_hash(prototype_artifact)} + official_manifest, official = validate_official_artifact(official_dir / "manifest.json", expected=expected) + local_features = np.load(work / "local-lposs/parity_features.npz", allow_pickle=False) + if local_features["patch_grid"].tolist() != official_manifest["patch_grid"]: + raise ValueError("local and official patch grids differ") + if local_features["image_grid"].tolist() != official_manifest["image_grid"]: + raise ValueError("local and official image grids differ") + differences = {} + shapes = {} + for feature in ("clip_features", "dino_features"): + observed = torch.from_numpy(local_features[feature]) + differences[feature] = compare_scores(official[feature], observed, atol=args.atol, rtol=args.rtol) + shapes[feature] = {"official": list(official[feature].shape), + "local": list(local_features[feature].shape)} + + local_outputs = {"maskclip_raw": (bootstrap_manifest, bootstrap_scores)} + for mode, config_name in (("lposs", "stock_lposs.yaml"), ("lposs_plus", "stock_lposs_plus.yaml")): + target = work / f"local-{mode}" + if target.exists(): # reuse the DINO bootstrap + manifest = json.loads((target / "run_manifest.json").read_text()) + scores = compatible_torch_load(next(target.glob("*/scores.pt"))) + else: + subprocess.run([sys.executable, "-m", "ovs_heritage.infer_ovs", *common, + "--prototype-artifact", str(prototype_artifact), "--model-config", str(local_root / f"configs/{config_name}"), + "--mode", mode, "--output-dir", str(target)], cwd=local_root, check=True) + manifest = json.loads((target / "run_manifest.json").read_text()) + scores = compatible_torch_load(next(target.glob("*/scores.pt"))) + local_outputs[mode] = (manifest, scores) + + for mode, (manifest, scores) in local_outputs.items(): + if manifest["config"] != configurations[mode]: + raise ValueError(f"local {mode} did not execute the requested resolved configuration") + if list(scores["channel_names"]) != request["channel_names"] or list(scores["semantic_ids"]) != request["semantic_ids"]: + raise ValueError(f"local {mode} channel/semantic identity mismatch") execution = manifest["records"][0]["execution"] - expected = (mode != "maskclip_raw", mode != "maskclip_raw", mode == "lposs_plus") - observed = (execution["dino_executed"], execution["patch_propagation_executed"], - execution["pixel_refinement_executed"]) - if observed != expected: - raise AssertionError(f"{mode} stages {observed} != {expected}") - score_path = next(target.glob("*/scores.pt")) - scores = torch.load(score_path, map_location="cpu", weights_only=True) - for key in ("seed_scores", "propagated_scores"): - tensor = scores[key] - if not torch.isfinite(tensor).all(): - raise AssertionError(f"{mode} {key} is non-finite") - outputs[mode] = scores - for mode in ("maskclip_raw", "lposs", "lposs_plus"): - for key in ("seed_scores", "propagated_scores"): - expected = reference[mode][key] - torch.testing.assert_close(outputs[mode][key], expected, - atol=args.atol, rtol=args.rtol) - print("Pinned upstream tensor parity passed.") + expected_path = (mode != "maskclip_raw", mode != "maskclip_raw", mode == "lposs_plus") + actual_path = (execution["dino_executed"], execution["patch_propagation_executed"], execution["pixel_refinement_executed"]) + if actual_path != expected_path: + raise ValueError(f"local {mode} execution path {actual_path} != {expected_path}") + if mode != "maskclip_raw" and (execution["requested_k"] != config["graph"]["k"] + or execution["effective_k"] != config["graph"]["k"]): + raise ValueError(f"local {mode} silently changed stock neighbor count") + for stage in ("seed_scores", "propagated_scores"): + key = f"{mode}.{stage}" + differences[key] = compare_scores( + official[f"{mode}.{stage}"], scores[stage], atol=args.atol, rtol=args.rtol) + shapes[key] = {"official": list(official[key].shape), + "local": list(scores[stage].shape)} + parity = {"schema_version": "lposs-upstream-parity-v2", "real_gpu_parity": True, + "passed": True, "upstream": official_manifest["upstream"], "input_sha256": request["input_sha256"], + "configurations": configurations, "configuration_sha256": request["configuration_sha256"], + "prototype_artifact_sha256": request["prototype_artifact_sha256"], + "prototypes_sha256": request["prototypes_sha256"], "model_hashes": request["model_hashes"], + "channel_names": request["channel_names"], "semantic_ids": request["semantic_ids"], + "device": args.device, "seed": 0, "atol": args.atol, "rtol": args.rtol, + "maximum_absolute_differences": differences, "tensor_shapes": shapes, + "patch_grid": official_manifest["patch_grid"], "image_grid": official_manifest["image_grid"], + "modes": list(MODES), + "artifacts": {"request": str(request_path), "prototypes": str(prototype_artifact), + "official_manifest": str(official_dir / "manifest.json")}} + (work / "parity_manifest.json").write_text(json.dumps(parity, indent=2, sort_keys=True) + "\n") + print(json.dumps(parity, sort_keys=True)) if __name__ == "__main__": diff --git a/tools/run_official_lposs_parity.py b/tools/run_official_lposs_parity.py new file mode 100644 index 0000000..34277c3 --- /dev/null +++ b/tools/run_official_lposs_parity.py @@ -0,0 +1,219 @@ +"""Isolated driver for unmodified LPOSS at the reviewed upstream commit. + +Run this with the official checkout first on ``PYTHONPATH``. Communication with +the parent checker is exclusively through versioned JSON/NPZ files. +""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from hashlib import sha256 +import inspect +import json +import os +from pathlib import Path +import subprocess +import sys + +EXPECTED_COMMIT = "e489a7445528922ddfe4e39631ef2fe34827c873" +EXPECTED_REPOSITORY = "https://github.com/vladan-stojnic/LPOSS" +DINO_REQUESTED_REPOSITORY = "facebookresearch/dino:main" +DINO_RESOLVED_REPOSITORY = "facebookresearch/dino:7c446df5b9f45747937fb0d72314eb9f7b66930a" +DINO_MODEL = "dino_vitb16" +LPOSS_CONSTRUCTOR_PARAMETERS = ( + "clip_backbone", "class_names", "vit_arch", "vit_patch_size", "enc_type_feats") + + +@contextmanager +def pinned_dino_hub_load(torch_module, calls): + """Narrowly redirect the single hard-coded official DINO load, then restore it.""" + original = torch_module.hub.load + + def load(repository, model, *args, **kwargs): + if repository != DINO_REQUESTED_REPOSITORY or model != DINO_MODEL: + raise RuntimeError(f"unexpected official torch.hub.load request: {repository!r}, {model!r}") + call = {"requested_repository": repository, "resolved_repository": DINO_RESOLVED_REPOSITORY, + "model": model, "args": list(args), "kwargs": dict(kwargs)} + calls.append(call) + return original(DINO_RESOLVED_REPOSITORY, model, *args, **kwargs) + + torch_module.hub.load = load + try: + yield + finally: + torch_module.hub.load = original + + +def construct_official_lposs(lposs_class, torch_module, *, class_names, config): + """Validate and call the exact constructor exposed by pinned official LPOSS.""" + parameters = tuple(inspect.signature(lposs_class).parameters) + if parameters != LPOSS_CONSTRUCTOR_PARAMETERS: + raise RuntimeError("pinned official LPOSS constructor drift: " + f"expected {LPOSS_CONSTRUCTOR_PARAMETERS}, observed {parameters}") + calls = [] + with pinned_dino_hub_load(torch_module, calls): + model = lposs_class(clip_backbone="maskclip", class_names=class_names, + vit_arch=config["dino"]["architecture"], vit_patch_size=config["dino"]["patch_size"], + enc_type_feats=config["dino"]["feature_type"]) + if len(calls) != 1: + raise RuntimeError(f"official LPOSS must issue exactly one pinned DINO load; observed {len(calls)}") + return model, calls + + +def file_hash(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_hash(value) -> str: + return sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def checkout_identity(root: Path) -> dict: + def git(*args): + return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() + commit = git("rev-parse", "HEAD") + if commit != EXPECTED_COMMIT: + raise ValueError(f"official LPOSS commit mismatch: {commit}") + if git("status", "--porcelain"): + raise ValueError("official LPOSS checkout is dirty") + remotes = git("remote", "-v") + if "github.com/vladan-stojnic/LPOSS" not in remotes: + raise ValueError(f"official LPOSS repository URL is not present in remotes: {remotes}") + tree = git("rev-parse", "HEAD^{tree}") + return {"repository": EXPECTED_REPOSITORY, "commit": commit, "tree": tree} + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--request", required=True) + parser.add_argument("--output-dir", required=True) + args = parser.parse_args(argv) + request_path = Path(args.request).resolve() + request = json.loads(request_path.read_text()) + upstream = Path(request["upstream_root"]).resolve() + identity = checkout_identity(upstream) + if identity != request.get("upstream"): + raise ValueError("upstream checkout identity changed after request construction") + if file_hash(Path(request["image_path"])) != request["input_sha256"]: + raise ValueError("input fingerprint changed after parity request construction") + if file_hash(Path(request["prototype_artifact"])) != request["prototype_artifact_sha256"]: + raise ValueError("prototype artifact fingerprint changed after request construction") + if canonical_hash(request["configurations"]) != request["configuration_sha256"]: + raise ValueError("resolved configuration fingerprint mismatch") + # Fail before importing a same-named local package. The verified upstream root + # is the first import location and is also the working directory for configs. + local_root = Path(request["local_root"]).resolve() + sys.path[:] = [str(upstream)] + [p for p in sys.path + if Path(p or ".").resolve() not in (local_root, local_root / "tools")] + os.chdir(upstream) + + import numpy as np + from PIL import Image + import torch + import torch.nn.functional as F + from models.lposs.lposs import LPOSS + from segmentation.evaluation.lposs_eval import ( + get_lposs_laplacian, get_lposs_plus_laplacian, perform_lp) + + def state_hash(module): + digest = sha256() + for key, value in sorted(module.state_dict().items()): + cpu = value.detach().cpu().contiguous() + digest.update(key.encode()) + digest.update(str(tuple(cpu.shape)).encode()) + digest.update(str(cpu.dtype).encode()) + digest.update(cpu.view(torch.uint8).numpy().tobytes()) + return digest.hexdigest() + + torch.manual_seed(request["seed"]) + torch.cuda.manual_seed_all(request["seed"]) + torch.cuda.set_device(torch.device(request["device"])) + proto_file = Path(request["prototype_artifact"]) + proto_npz = np.load(proto_file, allow_pickle=False) + proto_meta = json.loads(str(proto_npz["metadata_json"].item())) + prototypes_np = proto_npz["prototypes"] + if sha256(prototypes_np.tobytes(order="C")).hexdigest() != request["prototypes_sha256"]: + raise ValueError("prototype payload does not match request fingerprint") + for key in ("channel_names", "semantic_ids", "ontology_hash"): + if proto_meta.get(key) != request.get(key): + raise ValueError(f"prototype metadata disagrees with request {key}") + prototypes = torch.from_numpy(prototypes_np).to(request["device"]) + configurations = request["configurations"] + config = configurations["lposs"] + if configurations["maskclip_raw"] != config: + raise ValueError("raw and LPOSS configurations unexpectedly differ") + plus_expected = dict(config) + plus_expected["pixel_refine"] = True + if configurations["lposs_plus"] != plus_expected: + raise ValueError("LPOSS+ configuration differs by more than pixel_refine") + graph = config["graph"] + model, dino_loads = construct_official_lposs( + LPOSS, torch, class_names=proto_meta["channel_names"], config=config) + model = model.to(request["device"]).eval() + model.clip_backbone.decode_head.class_embeddings = prototypes + model_hashes = {"clip_state_sha256": state_hash(model.clip_backbone.backbone), + "dino_state_sha256": state_hash(model.dino_encoder)} + if model_hashes != request["model_hashes"]: + raise ValueError(f"official/local loaded model fingerprints differ: {model_hashes} != {request['model_hashes']}") + + rgb = np.asarray(Image.open(request["image_path"]).convert("RGB"), dtype=np.float32) / 255.0 + image = torch.from_numpy(rgb).permute(2, 0, 1).unsqueeze(0).to(request["device"]) + with torch.no_grad(): + dino, clip, classifier = model(image) + if tuple(dino.shape[1:3]) != tuple(clip.shape[1:3]): + clip = F.interpolate(clip.permute(0, 3, 1, 2), dino.shape[1:3], + mode="bilinear", align_corners=False).permute(0, 2, 3, 1) + h, w = dino.shape[1:3] + if h * w < graph["k"]: + raise RuntimeError(f"official stock k={graph['k']} requires at least {graph['k']} patch nodes; got {h*w}") + dino_nodes = F.normalize(dino[0].reshape(h * w, -1), dim=-1) + clip_nodes = F.normalize(clip[0].reshape(h * w, -1), dim=-1) + seed_nodes = clip_nodes @ prototypes.T + locations = torch.tensor([[0, image.shape[-2], 0, image.shape[-1]]], device=image.device) + laplacian = get_lposs_laplacian(dino_nodes, locations, [(h, w)], + sigma=graph["sigma"], pix_dist_pow=graph["pix_dist_pow"], k=graph["k"], + gamma=graph["gamma"], alpha=graph["alpha"], patch_size=config["dino"]["patch_size"]) + propagated_nodes = perform_lp(laplacian, seed_nodes) + seed_full = F.interpolate(seed_nodes.T.reshape(1, -1, h, w), image.shape[-2:], + mode="bilinear", align_corners=False) + propagated_full = F.interpolate(propagated_nodes.T.reshape(1, -1, h, w), image.shape[-2:], + mode="bilinear", align_corners=False) + flat = propagated_full[0].permute(1, 2, 0).reshape(-1, prototypes.shape[0]) + pixel_laplacian = get_lposs_plus_laplacian(image, flat, tau=graph["tau"], + neigh=graph["r"] // 2, alpha=graph["alpha"]) + refined = perform_lp(pixel_laplacian, flat).reshape( + image.shape[-2], image.shape[-1], -1).permute(2, 0, 1).unsqueeze(0) + + output = Path(args.output_dir).resolve() + output.mkdir(parents=True) + arrays = {"clip_features": clip_nodes.detach().cpu().numpy(), + "dino_features": dino_nodes.detach().cpu().numpy(), + "maskclip_raw.seed_scores": seed_full.detach().cpu().numpy(), + "maskclip_raw.propagated_scores": seed_full.detach().cpu().numpy(), + "lposs.seed_scores": seed_full.detach().cpu().numpy(), + "lposs.propagated_scores": propagated_full.detach().cpu().numpy(), + "lposs_plus.seed_scores": seed_full.detach().cpu().numpy(), + "lposs_plus.propagated_scores": refined.detach().cpu().numpy()} + np.savez(output / "stages.npz", **arrays) + manifest = {"schema_version": "official-lposs-parity-artifact-v1", "producer": "official-upstream", + "upstream": identity, "input_sha256": file_hash(Path(request["image_path"])), + "prototype_artifact_sha256": file_hash(proto_file), "prototypes_sha256": request["prototypes_sha256"], + "configurations": configurations, "configuration_sha256": request["configuration_sha256"], + "channel_names": proto_meta["channel_names"], "semantic_ids": proto_meta["semantic_ids"], + "ontology_hash": proto_meta["ontology_hash"], "patch_grid": [h, w], + "image_grid": list(image.shape[-2:]), "device": request["device"], "seed": request["seed"], + "model_hashes": model_hashes, + "dino_hub_loads": dino_loads, + "stages": {"maskclip_raw": ["seed_scores"], + "lposs": ["seed_scores", "propagated_scores"], + "lposs_plus": ["seed_scores", "propagated_scores", "pixel_refinement"]}, + "stage_artifact": "stages.npz", "stage_artifact_sha256": file_hash(output / "stages.npz")} + (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main()