From 6f024dc0feae39b4c06e60d45478172179939b99 Mon Sep 17 00:00:00 2001 From: runck014 Date: Wed, 14 Oct 2026 16:00:00 -0500 Subject: [PATCH 1/7] pin sqlalchemy <2.1: keep postgresql:// on psycopg2 A fresh install resolves SQLAlchemy 2.1, whose default PostgreSQL driver is psycopg (v3), which is not a dependency; every GEMS query then fails with "No module named 'psycopg'" (seen 2026-10-14 in a clean venv of 7098865). uv.lock already holds 2.0.41, but install.sh runs `uv pip install -e .[all]`, which does not read the lock, so a rebuilt MSI venv would break the same way. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cdd28a6..6220e4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ 'geopandas', "earthengine-api>=0.1.375,<0.2", "numpy>=1.21.0", - "sqlalchemy>=1.4.0", + "sqlalchemy>=1.4.0,<2.1", "psycopg2-binary>=2.9.0", "python-dotenv>=0.19.0", "requests>=2.25.0", From 2ac843f31ba83aac5aed76f70c9eeee8a9f56dee Mon Sep 17 00:00:00 2001 From: runck014 Date: Wed, 14 Oct 2026 16:00:00 -0500 Subject: [PATCH 2/7] device_monitoring: publish each run to the ingest service GEMS is reachable only inside UMN, so the cron on MSI stays the puller. With DEVICEMON_INGEST_URL and DEVICEMON_INGEST_TOKEN set, monitor() now saves the run as a devmon-batch/1 JSON file and POSTs it to rtgs- infrastructure's device-monitoring-ingest (retries with backoff on network errors, 429 and 5xx) instead of writing Cloud SQL with a service-account key. Without them the old direct write still runs. produce_db gains monitoring_records / logger_info_records / app_config_defaults so both paths store identical values. New command: rtgs device-monitoring publish FILE re-sends a saved batch (idempotent). Co-Authored-By: Claude Opus 5.5 --- src/rtgs_lab_tools/device_monitoring/cli.py | 9 + .../device_monitoring/config.py | 1 + src/rtgs_lab_tools/device_monitoring/core.py | 44 +++- .../devmon-batch-1.schema.json | 199 ++++++++++++++++++ .../device_monitoring/publisher.py | 149 +++++++++++++ .../device_monitoring/web_app/produce_db.py | 50 +++++ tests/device_monitoring/test_publisher.py | 71 +++++++ 7 files changed, 513 insertions(+), 10 deletions(-) create mode 100644 src/rtgs_lab_tools/device_monitoring/devmon-batch-1.schema.json create mode 100644 src/rtgs_lab_tools/device_monitoring/publisher.py create mode 100644 tests/device_monitoring/test_publisher.py diff --git a/src/rtgs_lab_tools/device_monitoring/cli.py b/src/rtgs_lab_tools/device_monitoring/cli.py index 1714934..72d1237 100644 --- a/src/rtgs_lab_tools/device_monitoring/cli.py +++ b/src/rtgs_lab_tools/device_monitoring/cli.py @@ -96,5 +96,14 @@ def monitor_cmd(ctx, start_date, end_date, node_ids, project, no_email): ) +@device_monitoring_cli.command("publish") +@click.argument("batch_file", type=click.Path(exists=True, dir_okay=False)) +def publish_cmd(batch_file): + """Re-send a saved batch file to the ingest service (a stored batch is a no-op).""" + from .publisher import publish_file + + publish_file(batch_file) + + if __name__ == "__main__": device_monitoring_cli() diff --git a/src/rtgs_lab_tools/device_monitoring/config.py b/src/rtgs_lab_tools/device_monitoring/config.py index 192ab6f..516db8c 100644 --- a/src/rtgs_lab_tools/device_monitoring/config.py +++ b/src/rtgs_lab_tools/device_monitoring/config.py @@ -79,6 +79,7 @@ STEP_TIMEOUT_DATABASE_WRITE = 600 # Cloud SQL connect + commits STEP_TIMEOUT_MESSAGE_BUILD = 600 # includes Particle API lookups STEP_TIMEOUT_NOTIFY = 300 # SMTP handshake + send +STEP_TIMEOUT_PUBLISH = 540 # POST to device-monitoring-ingest, retries included # Socket timeout for the outgoing SMTP connection (seconds). SMTP_TIMEOUT = 60 diff --git a/src/rtgs_lab_tools/device_monitoring/core.py b/src/rtgs_lab_tools/device_monitoring/core.py index 8bed766..061f0b6 100644 --- a/src/rtgs_lab_tools/device_monitoring/core.py +++ b/src/rtgs_lab_tools/device_monitoring/core.py @@ -22,6 +22,7 @@ STEP_TIMEOUT_DATA_RETRIEVAL, STEP_TIMEOUT_MESSAGE_BUILD, STEP_TIMEOUT_NOTIFY, + STEP_TIMEOUT_PUBLISH, ) from .data_analyzer import analyze_data from .data_formatter import format_data_with_parser @@ -32,7 +33,11 @@ from .watchdog import step_timeout from .web_app.models import app -from .web_app.produce_db import init_db, build_db, build_logger_info, build_app_config +from .web_app.produce_db import ( + init_db, build_db, build_logger_info, build_app_config, + monitoring_records, logger_info_records, app_config_defaults, +) +from .publisher import build_batch, publish, save_batch def _run_timestamp(): @@ -83,15 +88,34 @@ def monitor( analysis_dict = analyze_data(formatted_data) print(f"--Data analysis complete--") - # Step 3.5: Add data to SQLite database - print(f"--Beginning adding data to SQLite database--") - with step_timeout(STEP_TIMEOUT_DATABASE_WRITE, "Database write"): - with app.app_context(): - init_db() - build_db(analysis_dict, monitoring_timestamp) - build_logger_info(analysis_dict) - build_app_config() - print(f"--Addition to SQLite database complete--") + # Step 3.5: Store the results. On MSI they are published to device-monitoring-ingest + # (rtgs-infrastructure); the direct database write remains for local use. + if os.getenv("DEVICEMON_INGEST_URL"): + print("--Beginning batch build (Particle lookups)--") + with step_timeout(STEP_TIMEOUT_DATABASE_WRITE, "Batch build"): + batch = build_batch( + monitoring_records(analysis_dict), + logger_info_records(analysis_dict), + app_config_defaults(), + monitoring_timestamp, + start_date, + end_date, + full_fleet=node_ids is None, + ) + batch_path = save_batch(batch) + print(f"--Batch saved: {batch_path}--") + with step_timeout(STEP_TIMEOUT_PUBLISH, "Publish"): + publish(batch) + print("--Publish complete--") + else: + print(f"--Beginning adding data to SQLite database--") + with step_timeout(STEP_TIMEOUT_DATABASE_WRITE, "Database write"): + with app.app_context(): + init_db() + build_db(analysis_dict, monitoring_timestamp) + build_logger_info(analysis_dict) + build_app_config() + print(f"--Addition to SQLite database complete--") # Step 4: Build notification messages print(f"--Beginning message building--") diff --git a/src/rtgs_lab_tools/device_monitoring/devmon-batch-1.schema.json b/src/rtgs_lab_tools/device_monitoring/devmon-batch-1.schema.json new file mode 100644 index 0000000..87fb880 --- /dev/null +++ b/src/rtgs_lab_tools/device_monitoring/devmon-batch-1.schema.json @@ -0,0 +1,199 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "devmon-batch/1", + "title": "One device-monitoring run, published by the MSI cron to device-monitoring-ingest", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "batch_id", + "run_timestamp", + "window", + "full_fleet", + "producer", + "monitoring", + "logger_info", + "app_config" + ], + "properties": { + "schema": { + "const": "devmon-batch/1" + }, + "batch_id": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "description": "sha256 of the canonical JSON of every other field" + }, + "run_timestamp": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$", + "description": "UTC; becomes Monitoring.monitoring_timestamp" + }, + "window": { + "type": "object", + "additionalProperties": false, + "required": [ + "start", + "end" + ], + "properties": { + "start": { + "type": "string", + "format": "date" + }, + "end": { + "type": "string", + "format": "date" + } + } + }, + "full_fleet": { + "type": "boolean", + "description": "true = every node was analysed; loggers absent from the batch are marked inactive" + }, + "producer": { + "type": "object", + "required": [ + "host", + "release" + ], + "properties": { + "host": { + "type": "string", + "maxLength": 255 + }, + "release": { + "type": "string", + "maxLength": 100 + }, + "lab_tools_commit": { + "type": "string", + "maxLength": 64 + } + } + }, + "monitoring": { + "type": "array", + "maxItems": 5000, + "items": { + "$ref": "#/$defs/monitoring" + } + }, + "logger_info": { + "type": "array", + "maxItems": 5000, + "items": { + "$ref": "#/$defs/logger_info" + } + }, + "app_config": { + "type": "object", + "maxProperties": 50 + } + }, + "$defs": { + "stamp": { + "type": [ + "string", + "null" + ], + "maxLength": 30 + }, + "flag": { + "type": [ + "boolean", + "string" + ] + }, + "metric": { + "type": [ + "number", + "null" + ] + }, + "monitoring": { + "type": "object", + "additionalProperties": false, + "required": [ + "node_id", + "device_timestamp", + "flagged", + "battery", + "system", + "humidity", + "errors", + "is_missing", + "last_heard" + ], + "properties": { + "node_id": { + "type": "string", + "minLength": 1, + "maxLength": 50 + }, + "device_timestamp": { + "$ref": "#/$defs/stamp" + }, + "flagged": { + "$ref": "#/$defs/flag" + }, + "battery": { + "$ref": "#/$defs/metric" + }, + "system": { + "$ref": "#/$defs/metric" + }, + "humidity": { + "$ref": "#/$defs/metric" + }, + "errors": { + "type": "array" + }, + "is_missing": { + "$ref": "#/$defs/flag" + }, + "last_heard": { + "$ref": "#/$defs/stamp" + } + } + }, + "logger_info": { + "type": "object", + "additionalProperties": false, + "required": [ + "node_id", + "field_name", + "product_name", + "particle_url" + ], + "properties": { + "node_id": { + "type": "string", + "minLength": 1, + "maxLength": 50 + }, + "field_name": { + "type": [ + "string", + "null" + ], + "maxLength": 50 + }, + "product_name": { + "type": [ + "string", + "null" + ], + "maxLength": 50 + }, + "particle_url": { + "type": [ + "string", + "null" + ], + "maxLength": 100 + } + } + } + } +} diff --git a/src/rtgs_lab_tools/device_monitoring/publisher.py b/src/rtgs_lab_tools/device_monitoring/publisher.py new file mode 100644 index 0000000..9b1e1fc --- /dev/null +++ b/src/rtgs_lab_tools/device_monitoring/publisher.py @@ -0,0 +1,149 @@ +""" +Overview: + - Publishes one monitoring run to device-monitoring-ingest (rtgs-infrastructure, + services/device-monitoring-ingest) instead of writing Cloud SQL directly. + - Why: GEMS is reachable only from UMN/MSI, so the cron on MSI stays the puller and the + cloud side only receives. MSI holds a bearer token for one endpoint instead of a + service-account key and the database password. +Payload: + - devmon-batch/1, defined by devmon-batch-1.schema.json next to this file. It is a copy of + rtgs-infrastructure services/device-monitoring-ingest/schema/devmon-batch-1.schema.json; + change both together. +Environment: + DEVICEMON_INGEST_URL base URL of the ingest service (deploy run summary) + DEVICEMON_INGEST_TOKEN bearer token (Secret Manager devicemon-ingest-token) + DEVICEMON_BATCH_DIR where each batch is saved before sending + (default ~/logs/device-monitoring-batches) + DEVICEMON_RELEASE release tag the cron checked out (recorded in the batch) + DEVICEMON_PUBLISH_ATTEMPTS / DEVICEMON_PUBLISH_BACKOFF retry policy (5 / 15 s, doubling) +""" + +import hashlib +import json +import math +import os +import socket +import subprocess +import time +from pathlib import Path + +import requests + +SCHEMA_ID = "devmon-batch/1" +SCHEMA_PATH = Path(__file__).resolve().parent / "devmon-batch-1.schema.json" +RETRY_STATUS = {408, 425, 429, 500, 502, 503, 504} +REQUEST_TIMEOUT = (10, 60) # (connect, read) seconds + + +class PublishError(RuntimeError): + """The batch was not accepted.""" + + +def canonical(obj): + """Byte form batch_id is computed over; the ingest service recomputes it the same way.""" + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +def batch_id_of(batch): + body = {k: v for k, v in batch.items() if k != "batch_id"} + return hashlib.sha256(canonical(body).encode("utf-8")).hexdigest() + + +def _plain(value): + """numpy/pandas scalars and timestamps -> JSON types; NaN -> None (JSON has no NaN).""" + if hasattr(value, "item") and not isinstance(value, (list, dict, str)): + value = value.item() + if isinstance(value, float) and math.isnan(value): + return None + if hasattr(value, "isoformat"): + return value.isoformat() + return value + + +def _normalize(obj): + if isinstance(obj, dict): + return {str(k): _normalize(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_normalize(v) for v in obj] + return _plain(obj) + + +def _lab_tools_commit(): + try: + return subprocess.run(["git", "rev-parse", "--short=12", "HEAD"], cwd=Path(__file__).parent, + capture_output=True, text=True, timeout=5).stdout.strip() or "unknown" + except Exception: + return "unknown" + + +def build_batch(monitoring, logger_info, app_config, run_timestamp, start_date, end_date, + full_fleet=True): + """Assemble a devmon-batch/1 document from produce_db's record builders.""" + batch = _normalize({ + "schema": SCHEMA_ID, + "run_timestamp": run_timestamp, + "window": {"start": start_date, "end": end_date}, + "full_fleet": bool(full_fleet), + "producer": {"host": socket.gethostname(), + "release": os.getenv("DEVICEMON_RELEASE") or "unreleased", + "lab_tools_commit": _lab_tools_commit()}, + "monitoring": monitoring, + "logger_info": logger_info, + "app_config": app_config, + }) + batch["batch_id"] = batch_id_of(batch) + return batch + + +def save_batch(batch, directory=None): + """Write the batch where the cron's logs live, before sending: audit copy and replay source.""" + d = Path(directory or os.getenv("DEVICEMON_BATCH_DIR") + or Path.home() / "logs" / "device-monitoring-batches") + d.mkdir(parents=True, exist_ok=True) + stamp = batch["run_timestamp"].replace(" ", "T").replace(":", "") + path = d / f"{stamp}Z-{batch['batch_id'][:12]}.json" + path.write_text(canonical(batch)) + return path + + +def publish(batch, url=None, token=None, attempts=None, backoff=None, sleep=time.sleep): + """POST the batch; retry network errors and 408/425/429/5xx with doubling backoff. + + A 4xx other than those means the batch or token is wrong: retrying unchanged cannot help, + so it raises at once. Re-sending an already-stored batch returns "duplicate" (no-op). + """ + url = (url or os.getenv("DEVICEMON_INGEST_URL") or "").rstrip("/") + token = token or os.getenv("DEVICEMON_INGEST_TOKEN") + if not url or not token: + raise PublishError("DEVICEMON_INGEST_URL and DEVICEMON_INGEST_TOKEN must both be set") + attempts = int(attempts or os.getenv("DEVICEMON_PUBLISH_ATTEMPTS", "5")) + backoff = float(backoff or os.getenv("DEVICEMON_PUBLISH_BACKOFF", "15")) + body = canonical(batch).encode("utf-8") + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json", + "Idempotency-Key": batch["batch_id"]} + last = None + for attempt in range(1, attempts + 1): + try: + r = requests.post(f"{url}/v1/batches", data=body, headers=headers, + timeout=REQUEST_TIMEOUT) + except requests.RequestException as e: + last = f"{type(e).__name__}: {e}" + else: + if r.status_code in (200, 201): + info = r.json() + print(f"--Published batch {batch['batch_id'][:12]}: {info.get('status')} " + f"({len(batch['monitoring'])} nodes, run {batch['run_timestamp']} UTC)--") + return info + if r.status_code not in RETRY_STATUS: + raise PublishError(f"ingest refused the batch: HTTP {r.status_code} {r.text[:300]}") + last = f"HTTP {r.status_code} {r.text[:200]}" + if attempt < attempts: + delay = min(backoff * 2 ** (attempt - 1), 300) + print(f"publish attempt {attempt}/{attempts} failed ({last}); retry in {delay:.0f}s") + sleep(delay) + raise PublishError(f"publish failed after {attempts} attempts: {last}") + + +def publish_file(path, **kwargs): + """Re-send a saved batch (replay, or seeding the test service).""" + return publish(json.loads(Path(path).read_text()), **kwargs) diff --git a/src/rtgs_lab_tools/device_monitoring/web_app/produce_db.py b/src/rtgs_lab_tools/device_monitoring/web_app/produce_db.py index 61909cb..fcc62bc 100644 --- a/src/rtgs_lab_tools/device_monitoring/web_app/produce_db.py +++ b/src/rtgs_lab_tools/device_monitoring/web_app/produce_db.py @@ -12,6 +12,56 @@ from ..message_builder import get_device_info, get_product_slug, get_product_name, get_console_url from ..timezones import utc_stamp +def monitoring_records(analyzed_data_dict): + """One dict per node with the Monitoring columns except monitoring_timestamp (the run's). + + Shared by build_db (direct write) and the publisher (devmon-batch/1), so both paths store + exactly the same values. + """ + records = [] + for node_id, data in analyzed_data_dict.items(): + records.append({ + "node_id": node_id, + "flagged": data.get("flagged", False), + "battery": data.get("battery"), + "system": data.get("system"), + "humidity": data.get("humidity"), + "errors": data.get("errors", []), + # Both written as UTC strings with no offset suffix; see build_db. + "device_timestamp": utc_stamp(data.get("battery_timestamp") or + data.get("system_timestamp") or + data.get("humidity_timestamp")), + "is_missing": data.get("is_missing", False), + "last_heard": utc_stamp(data.get("last_heard"), "%Y-%m-%d %H:%M:%S"), + }) + return records + + +def logger_info_records(analyzed_data_dict): + """One dict per node with the LoggerInfo columns (Particle API lookups happen here).""" + records = [] + for node_id in analyzed_data_dict: + field_name, product_id = get_device_info(node_id) + product_slug = get_product_slug(product_id) + records.append({ + "node_id": node_id, + "field_name": field_name, + "product_name": get_product_name(product_id), + "particle_url": get_console_url(node_id, product_id, product_slug), + }) + return records + + +def app_config_defaults(): + """The config.py defaults the web app serves at /api/config.""" + return { + "battery_voltage_min": BATTERY_VOLTAGE_MIN, + "system_power_max": SYSTEM_POWER_MAX, + "inbox_humidity_max": INBOX_HUMIDITY_MAX, + "critical_errors": CRITICAL_ERRORS, + } + + def init_db(): db.create_all() diff --git a/tests/device_monitoring/test_publisher.py b/tests/device_monitoring/test_publisher.py new file mode 100644 index 0000000..987641c --- /dev/null +++ b/tests/device_monitoring/test_publisher.py @@ -0,0 +1,71 @@ +import json +from pathlib import Path + +import jsonschema +import pytest + +from rtgs_lab_tools.device_monitoring import publisher + +DM = Path(publisher.__file__).resolve().parent + + +def sample(): + monitoring = [{"node_id": "e00fce68aaaa", "device_timestamp": "2026-10-14 08:59", + "flagged": False, "battery": 4.01, "system": 0.18, "humidity": float("nan"), + "errors": [], "is_missing": False, "last_heard": "2026-10-14 08:59:12"}] + logger_info = [{"node_id": "e00fce68aaaa", "field_name": "WinterTurf_Type_A_64", + "product_name": "Winter Turf - v3", + "particle_url": "https://console.particle.io/devices/e00fce68aaaa"}] + config = {"battery_voltage_min": 3.6, "critical_errors": ["SD_ACCESS_FAIL"]} + return publisher.build_batch(monitoring, logger_info, config, "2026-10-14 09:27", + "2026-10-11", "2026-10-14") + + +def test_batch_matches_schema_and_id(): + b = sample() + schema = json.loads((DM / "devmon-batch-1.schema.json").read_text()) + jsonschema.Draft202012Validator(schema).validate(b) + assert b["batch_id"] == publisher.batch_id_of(b) + assert b["monitoring"][0]["humidity"] is None # NaN is not JSON + + +def test_batch_id_is_stable(): + assert sample()["batch_id"] == sample()["batch_id"] + + +class FakeResponse: + def __init__(self, code, body): + self.status_code, self._body, self.text = code, body, json.dumps(body) + + def json(self): + return self._body + + +def test_retries_then_succeeds(monkeypatch): + replies = [FakeResponse(503, {}), FakeResponse(201, {"status": "stored"})] + monkeypatch.setattr(publisher.requests, "post", lambda *a, **k: replies.pop(0)) + slept = [] + out = publisher.publish(sample(), url="https://x", token="t", sleep=slept.append) + assert out["status"] == "stored" and slept == [15.0] + + +def test_bad_request_is_not_retried(monkeypatch): + calls = [] + monkeypatch.setattr(publisher.requests, "post", + lambda *a, **k: calls.append(1) or FakeResponse(400, {"error": "x"})) + with pytest.raises(publisher.PublishError): + publisher.publish(sample(), url="https://x", token="t", sleep=lambda s: None) + assert len(calls) == 1 + + +def test_needs_url_and_token(monkeypatch): + monkeypatch.delenv("DEVICEMON_INGEST_URL", raising=False) + monkeypatch.delenv("DEVICEMON_INGEST_TOKEN", raising=False) + with pytest.raises(publisher.PublishError): + publisher.publish(sample()) + + +def test_cron_uses_the_cli_flag_names(): + script = (DM / "scheduled_device_monitoring.sh").read_text() + for bad in ("--start_date", "--end_date", "--node_ids"): + assert bad not in script From f4d8197139f988a18541d5528d4fb6f8e6d6f30f Mon Sep 17 00:00:00 2001 From: runck014 Date: Wed, 14 Oct 2026 16:00:00 -0500 Subject: [PATCH 3/7] scheduled_device_monitoring.sh: fix date flags, run a pinned release, versioned scrontab The script passed --start_date/--end_date/--node_ids; the CLI defines --start-date/--end-date/--node-ids, so any dated run failed with "No such option". The cron now runs a release tag instead of git pull on whatever branch the clone is on: DEVICEMON_RELEASE=devmon-vX.Y.Z pins it (a one-line edit on MSI deploys), track:devmon-v* follows the newest tag (a tag push deploys). If the release carries a newer copy of this script it re-runs that copy once. Credentials (main file plus ~/.rtgs_creds.d/*.env drop- ins) load before the checkout; the run refuses to start with no publish target instead of writing SQLite inside the clone. The scrontab line lives in scrontab.msi. Co-Authored-By: Claude Opus 5.5 --- .../device_monitoring/README.md | 13 +- .../scheduled_device_monitoring.sh | 119 +++++++++++++----- .../device_monitoring/scrontab.msi | 8 ++ 3 files changed, 107 insertions(+), 33 deletions(-) create mode 100644 src/rtgs_lab_tools/device_monitoring/scrontab.msi diff --git a/src/rtgs_lab_tools/device_monitoring/README.md b/src/rtgs_lab_tools/device_monitoring/README.md index 8f7247a..fb5e784 100644 --- a/src/rtgs_lab_tools/device_monitoring/README.md +++ b/src/rtgs_lab_tools/device_monitoring/README.md @@ -37,7 +37,8 @@ The device monitoring system runs automatically on MSI (Minnesota Supercomputing scrontab -e # Add this line to run monitoring daily at 4:27 AM -27 4 * * 1-5 /home/$USER/code/scripts/scheduled_device_monitoring.sh +# The line lives in scrontab.msi (versioned); install it with +scrontab ~/rtgs-lab-tools-cron/rtgs-lab-tools/src/rtgs_lab_tools/device_monitoring/scrontab.msi ``` #### Script Location and Setup @@ -87,10 +88,12 @@ export LOGGING_DB_USER=your_logging_username export LOGGING_DB_PASSWORD=your_logging_password # Device Monitoring Database (Cloud SQL Postgres - backs the web dashboard) -export DEVICEMON_INSTANCE_CONNECTION_NAME=sustained-edge-501900-p8:us-central1:device-monitoring-database -export DEVICEMON_DB_NAME=device_monitoring -export DEVICEMON_DB_USER=devicemon_app -export DEVICEMON_DB_PASSWORD=your_devicemon_db_password +# Publish target (rtgs-infrastructure device-monitoring-ingest-prod; URL from its +# deploy run summary, token from Secret Manager devicemon-ingest-token) +export DEVICEMON_INGEST_URL=https://device-monitoring-ingest-prod-....run.app +export DEVICEMON_INGEST_TOKEN=your_ingest_token +# The rtgs-lab-tools tag the cron runs (or track:devmon-v* for newest) +export DEVICEMON_RELEASE=devmon-v0.1.0 # Path to a service account key with only the `roles/cloudsql.client` role, # used to authenticate the Cloud SQL connector from MSI (which has no gcloud # login of its own). Get this from whoever manages the GCP project. diff --git a/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh b/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh index 11b7c2a..62e359e 100755 --- a/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh +++ b/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh @@ -14,6 +14,9 @@ export PYTHONUNBUFFERED=1 # flush python output to the log as it happens, so a r set -e # Exit on any error +# Fingerprint of the copy now running; see the release pin below. +SELF_SHA="$(sha1sum < "${BASH_SOURCE[0]}")" + # Parse command line arguments START_DATE="${1:-}" END_DATE="${2:-}" @@ -115,23 +118,83 @@ else log "Module command not found, assuming modules are already available" fi -# Setup repository - clone once, then just update -REPO_DIR="$WORK_DIR/rtgs-lab-tools" -if [ -d "$REPO_DIR" ]; then - log "Repository exists, updating with git pull..." - cd "$REPO_DIR" - git pull || { - log "Git pull failed, repository may be corrupted. Removing and re-cloning..." - cd "$WORK_DIR" - rm -rf rtgs-lab-tools - git clone https://github.com/RTGS-Lab/rtgs-lab-tools.git - cd rtgs-lab-tools - } +# Source RTGS credentials: the main file (if any), then every drop-in in ~/.rtgs_creds.d/ +# (devmon.env holds the publish target; written by the devmon-msi-install script). +CREDS_LOADED=0 +if [ -f "$HOME/.rtgs_credentials" ]; then + source "$HOME/.rtgs_credentials"; CREDS_LOADED=1 + log "Loaded credentials from $HOME/.rtgs_credentials" +elif [ -f "$HOME/.rtgs_creds" ]; then + source "$HOME/.rtgs_creds"; CREDS_LOADED=1 + log "Loaded credentials from $HOME/.rtgs_creds" +fi +for f in "$HOME"/.rtgs_creds.d/*.env; do + [ -f "$f" ] || continue + source "$f"; CREDS_LOADED=1 + log "Loaded credentials from $f" +done +if [ "$CREDS_LOADED" -eq 0 ]; then + log "ERROR: No credentials found: expected ~/.rtgs_credentials, ~/.rtgs_creds or ~/.rtgs_creds.d/*.env" + exit 1 +fi + +# Where the results go. Publishing to device-monitoring-ingest is the supported path; the direct +# Cloud SQL write (DEVICEMON_INSTANCE_CONNECTION_NAME + a service-account key) is deprecated, and +# with neither set the pipeline would silently write a SQLite file inside this clone. +if [ -n "${DEVICEMON_INGEST_URL:-}" ] && [ -n "${DEVICEMON_INGEST_TOKEN:-}" ]; then + log "Results go to the ingest service: $DEVICEMON_INGEST_URL" +elif [ -n "${DEVICEMON_INSTANCE_CONNECTION_NAME:-}" ]; then + log "WARNING: DEVICEMON_INGEST_URL/TOKEN unset; writing Cloud SQL directly (deprecated)" else + log "ERROR: set DEVICEMON_INGEST_URL and DEVICEMON_INGEST_TOKEN in ~/.rtgs_creds" + exit 1 +fi + +# Setup repository and pin the release. The cron runs a tagged release, never a moving branch. +# DEVICEMON_RELEASE (in ~/.rtgs_creds): +# devmon-v0.1.0 pinned: editing this one line on MSI is the deploy +# track:devmon-v* newest tag matching the glob, fetched every run: pushing a tag is the deploy +# (unset) stay on the current checkout, unpinned, with a warning +REPO_DIR="$WORK_DIR/rtgs-lab-tools" +if [ ! -d "$REPO_DIR/.git" ]; then log "Repository doesn't exist, cloning..." - cd "$WORK_DIR" - git clone https://github.com/RTGS-Lab/rtgs-lab-tools.git - cd rtgs-lab-tools + rm -rf "$REPO_DIR" + git clone https://github.com/RTGS-Lab/rtgs-lab-tools.git "$REPO_DIR" +fi +cd "$REPO_DIR" + +select_release() { + if ! git fetch --quiet --tags --force origin; then + log "WARNING: git fetch failed; staying on $(git describe --tags --always)" + return 0 + fi + local want="${DEVICEMON_RELEASE:-}" + if [ -z "$want" ]; then + log "WARNING: DEVICEMON_RELEASE unset; running $(git describe --tags --always) unpinned" + return 0 + fi + if [[ $want == track:* ]]; then + want="$(git tag -l "${want#track:}" --sort=-v:refname | head -n 1)" + [ -n "$want" ] || { log "ERROR: no tag matches ${DEVICEMON_RELEASE#track:}"; return 1; } + fi + git rev-parse -q --verify "refs/tags/$want" >/dev/null \ + || { log "ERROR: release tag $want not found"; return 1; } + if [ "$(git rev-parse HEAD)" != "$(git rev-parse "refs/tags/$want^{commit}")" ]; then + git checkout --quiet --detach "refs/tags/$want" + log "Checked out release $want" + else + log "Release $want already checked out" + fi + export DEVICEMON_RELEASE="$want" +} +select_release || exit 1 + +# If the release carries a different version of this script, run that one instead (once). +NEW_SELF="$REPO_DIR/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh" +if [ -z "${DEVICEMON_REEXEC:-}" ] && [ -f "$NEW_SELF" ] \ + && [ "$(sha1sum < "$NEW_SELF")" != "$SELF_SHA" ]; then + log "Script differs in ${DEVICEMON_RELEASE:-this checkout}; re-running that copy" + exec env DEVICEMON_REEXEC=1 bash "$NEW_SELF" "$@" fi # Only run installation if venv doesn't exist or if it's a fresh clone @@ -162,16 +225,16 @@ else exit 1 fi -# Source RTGS credentials (use generic path) -if [ -f "$HOME/.rtgs_credentials" ]; then - source "$HOME/.rtgs_credentials" - log "Loaded credentials from $HOME/.rtgs_credentials" -elif [ -f "$HOME/.rtgs_creds" ]; then - source "$HOME/.rtgs_creds" - log "Loaded credentials from $HOME/.rtgs_creds" -else - log "ERROR: No credentials file found. Expected $HOME/.rtgs_credentials or $HOME/.rtgs_creds" - exit 1 + +# A new release may bring new dependencies: refresh the editable install once per release. +STAMP="venv/.devmon-installed-release" +if [ "$(cat "$STAMP" 2>/dev/null)" != "${DEVICEMON_RELEASE:-unpinned}" ]; then + log "Installing rtgs-lab-tools for release ${DEVICEMON_RELEASE:-unpinned}" + if command -v uv &> /dev/null; then + uv pip install --quiet -e . && echo "${DEVICEMON_RELEASE:-unpinned}" > "$STAMP" + else + pip install --quiet -e . && echo "${DEVICEMON_RELEASE:-unpinned}" > "$STAMP" + fi fi # Verify rtgs command is available @@ -185,15 +248,15 @@ MONITOR_CMD="rtgs device-monitoring monitor" # Add optional parameters if provided if [ -n "$START_DATE" ]; then - MONITOR_CMD="$MONITOR_CMD --start_date=$START_DATE" + MONITOR_CMD="$MONITOR_CMD --start-date=$START_DATE" fi if [ -n "$END_DATE" ]; then - MONITOR_CMD="$MONITOR_CMD --end_date=$END_DATE" + MONITOR_CMD="$MONITOR_CMD --end-date=$END_DATE" fi if [ -n "$NODE_IDS" ]; then - MONITOR_CMD="$MONITOR_CMD --node_ids=$NODE_IDS" + MONITOR_CMD="$MONITOR_CMD --node-ids=$NODE_IDS" fi if [ "$PROJECT" != "ALL" ]; then diff --git a/src/rtgs_lab_tools/device_monitoring/scrontab.msi b/src/rtgs_lab_tools/device_monitoring/scrontab.msi new file mode 100644 index 0000000..49f875e --- /dev/null +++ b/src/rtgs_lab_tools/device_monitoring/scrontab.msi @@ -0,0 +1,8 @@ +# Versioned scrontab for the MSI device-monitoring cron (rtgs-lab-tools). +# A change here is a PR; it reaches MSI when the cron's account re-installs it: +# scrontab ~/rtgs-lab-tools-cron/rtgs-lab-tools/src/rtgs_lab_tools/device_monitoring/scrontab.msi +# (replaces that account's whole scrontab -- merge by hand if it holds other jobs). +# --time must exceed the script's retry budget (3 x 600 s + 2 x 120 s = 34 min). +#SCRON --time=01:00:00 +#SCRON --job-name=devmon-publish +27 4 * * 1-5 $HOME/rtgs-lab-tools-cron/rtgs-lab-tools/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh From 81d355b0af150c21a1bb0fa79e9567cb0b5b87a5 Mon Sep 17 00:00:00 2001 From: runck014 Date: Wed, 14 Oct 2026 16:00:00 -0500 Subject: [PATCH 4/7] ci: check the device-monitoring publisher on PRs and devmon-v* tags bash -n on the cron script, publisher tests (schema, batch id, retry policy, flag names), and the CLI flags the cron passes. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/devmon-publisher.yml | 43 ++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/devmon-publisher.yml diff --git a/.github/workflows/devmon-publisher.yml b/.github/workflows/devmon-publisher.yml new file mode 100644 index 0000000..1dbfc0b --- /dev/null +++ b/.github/workflows/devmon-publisher.yml @@ -0,0 +1,43 @@ +name: Device monitoring publisher + +# Checks the MSI cron side on every PR that touches it and on every release tag the cron can be +# pinned to (devmon-v*). The cron itself runs on MSI; GitHub cannot reach MSI. + +on: + pull_request: + paths: + - "src/rtgs_lab_tools/device_monitoring/**" + - "tests/device_monitoring/**" + - ".github/workflows/devmon-publisher.yml" + push: + tags: + - "devmon-v*" + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Cron script parses + run: bash -n src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh + + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e . pytest pytest-cov jsonschema + + - name: Publisher tests + run: pytest -q tests/device_monitoring/test_publisher.py + + - name: CLI takes the flags the cron passes + run: | + rtgs device-monitoring monitor --help | grep -q -- '--start-date' + rtgs device-monitoring publish --help >/dev/null From 93e8b6e428a96df8152d5aaae6f2376bc62b3d0f Mon Sep 17 00:00:00 2001 From: runck014 Date: Fri, 16 Oct 2026 11:28:09 -0500 Subject: [PATCH 5/7] device_monitoring cron: run from the lab's shared MSI workspace Defaults to /projects/standard/runck014/shared/device_monitoring (Bryan's scrontab): DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE (credentials/.rtgs_credentials, 660 group runck014), DEVICEMON_BATCH_DIR (logs/batches). umask 007. Per-user ~/.rtgs_credentials and ~/.rtgs_creds remain fallbacks; ~/.rtgs_creds.d is dropped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0149VSJ7dmCkXd4UcUjQbG9R --- .../device_monitoring/README.md | 43 ++++++++++--------- .../scheduled_device_monitoring.sh | 39 ++++++++++------- .../device_monitoring/scrontab.msi | 11 ++--- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/src/rtgs_lab_tools/device_monitoring/README.md b/src/rtgs_lab_tools/device_monitoring/README.md index fb5e784..d15254a 100644 --- a/src/rtgs_lab_tools/device_monitoring/README.md +++ b/src/rtgs_lab_tools/device_monitoring/README.md @@ -38,39 +38,43 @@ scrontab -e # Add this line to run monitoring daily at 4:27 AM # The line lives in scrontab.msi (versioned); install it with -scrontab ~/rtgs-lab-tools-cron/rtgs-lab-tools/src/rtgs_lab_tools/device_monitoring/scrontab.msi +scrontab /projects/standard/runck014/shared/device_monitoring/rtgs-lab-tools-cron/rtgs-lab-tools/src/rtgs_lab_tools/device_monitoring/scrontab.msi ``` #### Script Location and Setup The monitoring script is located at: ``` -/home/$USER/code/scripts/scheduled_device_monitoring.sh +/projects/standard/runck014/shared/device_monitoring/scripts/scheduled_device_monitoring.sh +(the lab's shared workspace; runs from Bryan's scrontab. Paths are overridable: +DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE) ``` This script automatically: - Loads required MSI modules (Python, Git) - Clones/updates the rtgs-lab-tools repository - Sets up the virtual environment -- Loads credentials from the MSI home directory +- Loads credentials from the shared credentials file - Runs the device monitoring -- Logs all activities to `~/logs/device-monitoring-logs/` +- Logs all activities to `/projects/standard/runck014/shared/device_monitoring/logs/` (batches in `logs/batches/`) ### MSI Credentials Configuration #### Credentials File Setup -Create a secure credentials file in your MSI home directory: +The shared credentials file (mode 660, group runck014; per-user `~/.rtgs_credentials` or +`~/.rtgs_creds` are fallbacks): ```bash # Create credentials file with restricted permissions -touch ~/.rtgs_creds -chmod 600 ~/.rtgs_creds +touch /projects/standard/runck014/shared/device_monitoring/credentials/.rtgs_credentials +chmod 660 /projects/standard/runck014/shared/device_monitoring/credentials/.rtgs_credentials +chgrp runck014 /projects/standard/runck014/shared/device_monitoring/credentials/.rtgs_credentials ``` #### Required Credentials -Edit `~/.rtgs_creds` with your actual credentials: +Edit `credentials/.rtgs_credentials` with the actual credentials: ```bash # GEMS Database Configuration @@ -94,10 +98,6 @@ export DEVICEMON_INGEST_URL=https://device-monitoring-ingest-prod-....run.app export DEVICEMON_INGEST_TOKEN=your_ingest_token # The rtgs-lab-tools tag the cron runs (or track:devmon-v* for newest) export DEVICEMON_RELEASE=devmon-v0.1.0 -# Path to a service account key with only the `roles/cloudsql.client` role, -# used to authenticate the Cloud SQL connector from MSI (which has no gcloud -# login of its own). Get this from whoever manages the GCP project. -export GOOGLE_APPLICATION_CREDENTIALS=/path/to/device-monitoring-writer-key.json # Particle Cloud API export PARTICLE_ACCESS_TOKEN=your_particle_token @@ -110,11 +110,10 @@ export GMAIL_RECIPIENT=notification_recipient@umn.edu #### Security Notes -- **File permissions**: The credentials file has `600` permissions (readable only by you) +- **File permissions**: The credentials file has `660` permissions (owner + group runck014) - **No VPN required**: MSI infrastructure has direct access to the database - **Secure storage**: Credentials are stored locally on MSI, not in version control -- **Alternative names**: The script also checks for `~/.rtgs_credentials` as a fallback -- **Service account key**: `GOOGLE_APPLICATION_CREDENTIALS` should also point to a file with `600` permissions, kept outside version control like the credentials file itself +- **Alternative names**: The script falls back to `~/.rtgs_credentials`, then `~/.rtgs_creds` ### MSI Monitoring Workflow @@ -122,10 +121,10 @@ export GMAIL_RECIPIENT=notification_recipient@umn.edu 1. **Before 5:00 AM daily**: SLURM crontab triggers the monitoring script 2. **Environment setup**: Script loads Python and Git modules -3. **Code update**: Repository is updated with `git pull` -4. **Credential loading**: Secure credentials are loaded from `~/.rtgs_creds` +3. **Code update**: The pinned release tag (`DEVICEMON_RELEASE`) is checked out +4. **Credential loading**: Credentials are loaded from `credentials/.rtgs_credentials` 5. **Monitoring execution**: Device monitoring runs across all projects -6. **Logging**: Results and logs are saved to `~/logs/device-monitoring-logs/` +6. **Logging**: Results and logs are saved to `logs/` in the shared workspace 7. **Cleanup**: Virtual environment is properly deactivated #### Log File Management @@ -153,10 +152,14 @@ module load python module load git # Run the monitoring script directly -/home/$USER/code/scripts/scheduled_device_monitoring.sh +/projects/standard/runck014/shared/device_monitoring/scripts/scheduled_device_monitoring.sh +(the lab's shared workspace; runs from Bryan's scrontab. Paths are overridable: +DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE) # Or run specific monitoring with parameters -/home/$USER/code/scripts/scheduled_device_monitoring.sh 2024-01-01 2024-01-31 "e00fce68c7f2c2b634bb4b7d" LCCMR +/projects/standard/runck014/shared/device_monitoring/scripts/scheduled_device_monitoring.sh +(the lab's shared workspace; runs from Bryan's scrontab. Paths are overridable: +DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE) 2024-01-01 2024-01-31 "e00fce68c7f2c2b634bb4b7d" LCCMR ``` ## Python API Usage diff --git a/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh b/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh index 62e359e..2548310 100755 --- a/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh +++ b/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh @@ -11,6 +11,14 @@ export PYTHONUNBUFFERED=1 # flush python output to the log as it happens, so a r # DEVICEMON_MAX_ATTEMPTS how many times to run the report before giving up (default 3) # DEVICEMON_ATTEMPT_TIMEOUT hard wall-clock limit per attempt, seconds (default 1800) # DEVICEMON_RETRY_DELAY pause between attempts, seconds (default 300) +# +# Paths (defaults = the lab's shared MSI workspace, run from Bryan's scrontab): +# DEVICEMON_BASE_DIR /projects/standard/runck014/shared/device_monitoring +# DEVICEMON_WORK_DIR $DEVICEMON_BASE_DIR/rtgs-lab-tools-cron (clone + venv live here) +# DEVICEMON_LOG_DIR $DEVICEMON_BASE_DIR/logs +# DEVICEMON_CREDS_FILE $DEVICEMON_BASE_DIR/credentials/.rtgs_credentials +# (mode 660, group runck014; holds DEVICEMON_INGEST_URL/TOKEN) +# DEVICEMON_BATCH_DIR $DEVICEMON_LOG_DIR/batches (each batch saved before it is sent) set -e # Exit on any error @@ -36,9 +44,14 @@ MAX_ATTEMPTS="${DEVICEMON_MAX_ATTEMPTS:-3}" ATTEMPT_TIMEOUT="${DEVICEMON_ATTEMPT_TIMEOUT:-600}" RETRY_DELAY="${DEVICEMON_RETRY_DELAY:-120}" -# Set up working directory and log file -WORK_DIR="$HOME/rtgs-lab-tools-cron" -LOG_DIR="$HOME/logs/device-monitoring-logs" +# Set up working directory and log file. Group-shared workspace: files stay group-readable, +# never world-readable. +umask 007 +BASE_DIR="${DEVICEMON_BASE_DIR:-/projects/standard/runck014/shared/device_monitoring}" +WORK_DIR="${DEVICEMON_WORK_DIR:-$BASE_DIR/rtgs-lab-tools-cron}" +LOG_DIR="${DEVICEMON_LOG_DIR:-$BASE_DIR/logs}" +CREDS_FILE="${DEVICEMON_CREDS_FILE:-$BASE_DIR/credentials/.rtgs_credentials}" +export DEVICEMON_BATCH_DIR="${DEVICEMON_BATCH_DIR:-$LOG_DIR/batches}" LOG_FILE="$LOG_DIR/device_monitoring_$(date +%Y%m%d_%H%M%S).log" LOCK_FILE="$WORK_DIR/device_monitoring.lock" @@ -118,23 +131,17 @@ else log "Module command not found, assuming modules are already available" fi -# Source RTGS credentials: the main file (if any), then every drop-in in ~/.rtgs_creds.d/ -# (devmon.env holds the publish target; written by the devmon-msi-install script). +# Source RTGS credentials: the shared file (DEVICEMON_CREDS_FILE); the per-user files are a +# fallback for running from a personal account. CREDS_LOADED=0 -if [ -f "$HOME/.rtgs_credentials" ]; then - source "$HOME/.rtgs_credentials"; CREDS_LOADED=1 - log "Loaded credentials from $HOME/.rtgs_credentials" -elif [ -f "$HOME/.rtgs_creds" ]; then - source "$HOME/.rtgs_creds"; CREDS_LOADED=1 - log "Loaded credentials from $HOME/.rtgs_creds" -fi -for f in "$HOME"/.rtgs_creds.d/*.env; do +for f in "$CREDS_FILE" "$HOME/.rtgs_credentials" "$HOME/.rtgs_creds"; do [ -f "$f" ] || continue source "$f"; CREDS_LOADED=1 log "Loaded credentials from $f" + break done if [ "$CREDS_LOADED" -eq 0 ]; then - log "ERROR: No credentials found: expected ~/.rtgs_credentials, ~/.rtgs_creds or ~/.rtgs_creds.d/*.env" + log "ERROR: No credentials found: expected $CREDS_FILE (or ~/.rtgs_credentials, ~/.rtgs_creds)" exit 1 fi @@ -146,12 +153,12 @@ if [ -n "${DEVICEMON_INGEST_URL:-}" ] && [ -n "${DEVICEMON_INGEST_TOKEN:-}" ]; t elif [ -n "${DEVICEMON_INSTANCE_CONNECTION_NAME:-}" ]; then log "WARNING: DEVICEMON_INGEST_URL/TOKEN unset; writing Cloud SQL directly (deprecated)" else - log "ERROR: set DEVICEMON_INGEST_URL and DEVICEMON_INGEST_TOKEN in ~/.rtgs_creds" + log "ERROR: set DEVICEMON_INGEST_URL and DEVICEMON_INGEST_TOKEN in $CREDS_FILE" exit 1 fi # Setup repository and pin the release. The cron runs a tagged release, never a moving branch. -# DEVICEMON_RELEASE (in ~/.rtgs_creds): +# DEVICEMON_RELEASE (in the credentials file): # devmon-v0.1.0 pinned: editing this one line on MSI is the deploy # track:devmon-v* newest tag matching the glob, fetched every run: pushing a tag is the deploy # (unset) stay on the current checkout, unpinned, with a warning diff --git a/src/rtgs_lab_tools/device_monitoring/scrontab.msi b/src/rtgs_lab_tools/device_monitoring/scrontab.msi index 49f875e..a0421e3 100644 --- a/src/rtgs_lab_tools/device_monitoring/scrontab.msi +++ b/src/rtgs_lab_tools/device_monitoring/scrontab.msi @@ -1,8 +1,9 @@ -# Versioned scrontab for the MSI device-monitoring cron (rtgs-lab-tools). -# A change here is a PR; it reaches MSI when the cron's account re-installs it: -# scrontab ~/rtgs-lab-tools-cron/rtgs-lab-tools/src/rtgs_lab_tools/device_monitoring/scrontab.msi -# (replaces that account's whole scrontab -- merge by hand if it holds other jobs). +# Versioned scrontab for the device-monitoring cron (rtgs-lab-tools), run from Bryan's MSI +# account out of the lab's shared workspace. A change here is a PR; it reaches MSI when +# re-installed: scrontab (replaces the account's whole scrontab -- merge by hand +# if it holds other jobs). scripts/scheduled_device_monitoring.sh may be a stale copy: the +# script re-runs the pinned release's own copy when they differ. # --time must exceed the script's retry budget (3 x 600 s + 2 x 120 s = 34 min). #SCRON --time=01:00:00 #SCRON --job-name=devmon-publish -27 4 * * 1-5 $HOME/rtgs-lab-tools-cron/rtgs-lab-tools/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh +27 4 * * 1-5 /projects/standard/runck014/shared/device_monitoring/scripts/scheduled_device_monitoring.sh From 9e1fa28f4f9dce4bc2e71a4328c26a2e087137a8 Mon Sep 17 00:00:00 2001 From: runck014 Date: Fri, 16 Oct 2026 12:05:29 -0500 Subject: [PATCH 6/7] device_monitoring: fetch the ingest token from Secret Manager each run The publisher reads devicemon-ingest-token (prod URL) or devicemon-ingest-token-test (test URL) from Secret Manager at run time with a service-account key (DEVICEMON_SA_KEY, default ~/.config/devmon/sa.json) and holds it in memory only, so rotating the token never touches MSI. The key must be a regular file owned by the running user, mode 600; anything else fails closed. The token is sent only to the known ingest hosts, and the secret must match the host. DEVICEMON_INGEST_TOKEN stays as an explicit test override. - new `rtgs device-monitoring check-ingest`: fetch the token, call /health and /v1/status, print the secret version and HTTP statuses (exit 2 setup, 3 transient) - save_batch creates files with O_EXCL|O_NOFOLLOW (the batch dir is group-writable) --- src/rtgs_lab_tools/device_monitoring/cli.py | 41 +++++ .../device_monitoring/ingest_token.py | 171 ++++++++++++++++++ .../device_monitoring/publisher.py | 38 +++- tests/device_monitoring/test_ingest_token.py | 155 ++++++++++++++++ tests/device_monitoring/test_publisher.py | 32 +++- 5 files changed, 426 insertions(+), 11 deletions(-) create mode 100644 src/rtgs_lab_tools/device_monitoring/ingest_token.py create mode 100644 tests/device_monitoring/test_ingest_token.py diff --git a/src/rtgs_lab_tools/device_monitoring/cli.py b/src/rtgs_lab_tools/device_monitoring/cli.py index 72d1237..80bc59b 100644 --- a/src/rtgs_lab_tools/device_monitoring/cli.py +++ b/src/rtgs_lab_tools/device_monitoring/cli.py @@ -105,5 +105,46 @@ def publish_cmd(batch_file): publish_file(batch_file) +@device_monitoring_cli.command("check-ingest") +def check_ingest_cmd(): + """Fetch the ingest token from Secret Manager and call /health and /v1/status. + + Prints the secret version used and the HTTP statuses, never the token. Exit 0 ok, + 2 setup is wrong (key file, IAM, URL), 3 transient (network, 5xx): worth a retry. + """ + import os + + import requests + + from . import ingest_token + + url = os.getenv("DEVICEMON_INGEST_URL", "") + try: + base, _ = ingest_token.check_ingest_url(url) + secret = ingest_token.secret_for(base) + token, version = ingest_token.fetch_from_secret_manager(secret) + except ingest_token.TokenError as e: + click.echo(f"check-ingest: FAILED: {e}", err=True) + sys.exit(3 if e.transient else 2) + click.echo(f"check-ingest: token from {version}") + try: + h = requests.get(f"{base}/health", timeout=(10, 30)) + s = requests.get(f"{base}/v1/status", timeout=(10, 30), + headers={"Authorization": f"Bearer {token}"}) + except requests.RequestException as e: + click.echo(f"check-ingest: FAILED: {base} unreachable: {type(e).__name__}", err=True) + sys.exit(3) + finally: + del token + click.echo(f"check-ingest: {base}/health HTTP {h.status_code}; /v1/status HTTP " + f"{s.status_code} {s.text[:200] if s.status_code == 200 else ''}".rstrip()) + if s.status_code == 401: + click.echo("check-ingest: FAILED: the service rejects this token (it pins secret " + "versions; redeploy it after adding a version)", err=True) + sys.exit(2) + if h.status_code != 200 or s.status_code != 200: + sys.exit(3 if max(h.status_code, s.status_code) >= 500 else 2) + + if __name__ == "__main__": device_monitoring_cli() diff --git a/src/rtgs_lab_tools/device_monitoring/ingest_token.py b/src/rtgs_lab_tools/device_monitoring/ingest_token.py new file mode 100644 index 0000000..f0ad3e8 --- /dev/null +++ b/src/rtgs_lab_tools/device_monitoring/ingest_token.py @@ -0,0 +1,171 @@ +""" +Overview: + - Fetches the device-monitoring-ingest bearer token from Google Secret Manager at run time, + so rotating the token never touches MSI. The token lives only in this process's memory: + it is never logged, printed, written to a batch file, or passed on a command line. +Credentials: + - A service-account JSON key owned by the running user, mode 600 (or 400): + DEVICEMON_SA_KEY (default ~/.config/devmon/sa.json). The account + (device-monitoring-msi@rtgslab) can read the two token secrets and nothing else. + - The key is used explicitly; GOOGLE_APPLICATION_CREDENTIALS is never set, so nothing else + in rtgs-lab-tools picks it up as Application Default Credentials. +Environment: + DEVICEMON_SA_KEY service-account key file (default ~/.config/devmon/sa.json) + DEVICEMON_TOKEN_SECRET secret name; default follows DEVICEMON_INGEST_URL: + the test service -> devicemon-ingest-token-test, + otherwise devicemon-ingest-token + DEVICEMON_GCP_PROJECT project holding the secret (default rtgslab) + DEVICEMON_INGEST_TOKEN explicit override for tests only; the MSI cron unsets it +""" + +import base64 +import json +import os +import re +import stat +import sys +from pathlib import Path +from urllib.parse import urlparse + +SM_SCOPE = "https://www.googleapis.com/auth/cloud-platform" +SM_URL = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/latest:access" +DEFAULT_PROJECT = "rtgslab" +DEFAULT_KEY = "~/.config/devmon/sa.json" +PROD_SECRET = "devicemon-ingest-token" +TEST_SECRET = "devicemon-ingest-token-test" +REQUEST_TIMEOUT = (10, 30) + +# The only hosts the bearer token is ever sent to, and the secret each one accepts. A URL from +# the group-writable credentials file cannot redirect the token anywhere else. +INGEST_HOSTS = { + "device-monitoring-ingest-prod-685278787484.us-central1.run.app": PROD_SECRET, + "device-monitoring-ingest-prod-ieix5g464a-uc.a.run.app": PROD_SECRET, + "device-monitoring-ingest-test-685278787484.us-central1.run.app": TEST_SECRET, +} +_SECRET_NAME = re.compile(r"^[A-Za-z0-9_-]{1,255}$") + + +class TokenError(RuntimeError): + """The ingest token could not be obtained. The message never contains a secret. + + transient=True: a network or server-side failure worth retrying later; False: setup is + wrong (key file, IAM, names) and retrying unchanged cannot help. + """ + + def __init__(self, message, transient=False): + super().__init__(message) + self.transient = transient + + +def check_ingest_url(url): + """Return (base_url, secret_for_host); refuse anything but https to a known ingest host.""" + u = urlparse((url or "").strip()) + if u.scheme != "https" or u.hostname not in INGEST_HOSTS or u.port not in (None, 443): + raise TokenError(f"refusing to send the ingest token to {url!r}: not one of the known " + f"device-monitoring-ingest hosts (https only): " + f"{', '.join(sorted(INGEST_HOSTS))}") + return f"https://{u.hostname}", INGEST_HOSTS[u.hostname] + + +def secret_for(url=None, env=None): + """Secret name for this run: DEVICEMON_TOKEN_SECRET, else the one the URL's host accepts.""" + env = os.environ if env is None else env + host_secret = check_ingest_url(url)[1] if url else None + name = (env.get("DEVICEMON_TOKEN_SECRET") or host_secret or PROD_SECRET).strip() + if not _SECRET_NAME.match(name): + raise TokenError(f"DEVICEMON_TOKEN_SECRET {name!r} is not a valid secret name") + if host_secret and name != host_secret: + raise TokenError(f"secret {name} does not belong to {url}; that host takes {host_secret}") + return name + + +def read_key_file(path): + """Load the service-account key, failing closed unless it is ours, a regular file, and 600.""" + p = Path(os.path.expanduser(path)) + try: + fd = os.open(p, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except FileNotFoundError: + raise TokenError(f"service-account key {p} not found. Copy it from the machine that ran " + f"devmon-msi-sa-setup.sh, then chmod 600 it") from None + except OSError as e: + raise TokenError(f"cannot open service-account key {p}: {e.strerror} " + f"(a symlink is refused)") from None + try: + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode): + raise TokenError(f"service-account key {p} is not a regular file") + if st.st_uid != os.geteuid(): + raise TokenError(f"service-account key {p} is owned by uid {st.st_uid}, not by the " + f"running user (uid {os.geteuid()}); refusing it") + mode = stat.S_IMODE(st.st_mode) + if mode not in (0o600, 0o400): + raise TokenError(f"service-account key {p} has mode {mode:03o}; it must be 600 " + f"(run: chmod 600 {p})") + with os.fdopen(fd, "rb") as f: + fd = None + raw = f.read(65536) + finally: + if fd is not None: + os.close(fd) + try: + info = json.loads(raw) + except ValueError: + raise TokenError(f"service-account key {p} is not valid JSON") from None + if not isinstance(info, dict) or info.get("type") != "service_account": + raise TokenError(f"{p} is not a service-account key (type != service_account)") + return info + + +def _session(info): + from google.auth.transport.requests import AuthorizedSession + from google.oauth2 import service_account + + creds = service_account.Credentials.from_service_account_info(info, scopes=[SM_SCOPE]) + return AuthorizedSession(creds) + + +def fetch_from_secret_manager(secret, key_path=None, project=None, session_factory=_session): + """Return (token, version_name). version_name ('.../versions/3') is safe to log.""" + key_path = key_path or os.getenv("DEVICEMON_SA_KEY") or DEFAULT_KEY + project = project or os.getenv("DEVICEMON_GCP_PROJECT") or DEFAULT_PROJECT + if not _SECRET_NAME.match(secret) or not _SECRET_NAME.match(project): + raise TokenError("invalid secret or project name") + info = read_key_file(key_path) + url = SM_URL.format(project=project, secret=secret) + try: + r = session_factory(info).get(url, timeout=REQUEST_TIMEOUT) + except Exception as e: # auth refresh or network; neither message carries the secret + raise TokenError(f"Secret Manager request for {secret} failed: {type(e).__name__}: " + f"{str(e)[:200]}", transient=True) from None + if r.status_code != 200: + try: + err = r.json().get("error", {}) + why = f"{err.get('status', '')} {err.get('message', '')[:200]}".strip() + except Exception: + why = "" + hint = {403: "the service account cannot read this secret (IAM)", + 404: "no such secret, or no enabled version", + 400: "latest version is disabled or destroyed"}.get(r.status_code, "") + raise TokenError(f"Secret Manager refused {project}/{secret}: HTTP {r.status_code} " + f"{why} {('-- ' + hint) if hint else ''}".rstrip(), + transient=r.status_code == 429 or r.status_code >= 500) + body = r.json() + try: + token = base64.b64decode(body["payload"]["data"]).decode("utf-8").strip() + except Exception: + raise TokenError(f"Secret Manager returned no usable payload for {secret}") from None + if not token: + raise TokenError(f"secret {secret} is empty") + return token, body.get("name", f"projects/{project}/secrets/{secret}/versions/?") + + +def load_token(url=None, env=None, fetch=fetch_from_secret_manager): + """The bearer token for this run. DEVICEMON_INGEST_TOKEN wins, for tests only.""" + env = os.environ if env is None else env + override = env.get("DEVICEMON_INGEST_TOKEN") + if override: + print("WARNING: using DEVICEMON_INGEST_TOKEN from the environment (test override), " + "not Secret Manager", file=sys.stderr) + return override.strip() + token, _version = fetch(secret_for(url, env)) + return token diff --git a/src/rtgs_lab_tools/device_monitoring/publisher.py b/src/rtgs_lab_tools/device_monitoring/publisher.py index 9b1e1fc..3b59e1c 100644 --- a/src/rtgs_lab_tools/device_monitoring/publisher.py +++ b/src/rtgs_lab_tools/device_monitoring/publisher.py @@ -3,15 +3,16 @@ - Publishes one monitoring run to device-monitoring-ingest (rtgs-infrastructure, services/device-monitoring-ingest) instead of writing Cloud SQL directly. - Why: GEMS is reachable only from UMN/MSI, so the cron on MSI stays the puller and the - cloud side only receives. MSI holds a bearer token for one endpoint instead of a - service-account key and the database password. + cloud side only receives. The bearer token is fetched from Secret Manager each run + (ingest_token.py) and held in memory only; MSI holds a key that can read that one secret. Payload: - devmon-batch/1, defined by devmon-batch-1.schema.json next to this file. It is a copy of rtgs-infrastructure services/device-monitoring-ingest/schema/devmon-batch-1.schema.json; change both together. Environment: DEVICEMON_INGEST_URL base URL of the ingest service (deploy run summary) - DEVICEMON_INGEST_TOKEN bearer token (Secret Manager devicemon-ingest-token) + DEVICEMON_SA_KEY / DEVICEMON_TOKEN_SECRET see ingest_token.py (token source) + DEVICEMON_INGEST_TOKEN test override only; normally unset (token from Secret Manager) DEVICEMON_BATCH_DIR where each batch is saved before sending (default ~/logs/device-monitoring-batches) DEVICEMON_RELEASE release tag the cron checked out (recorded in the batch) @@ -29,6 +30,8 @@ import requests +from . import ingest_token + SCHEMA_ID = "devmon-batch/1" SCHEMA_PATH = Path(__file__).resolve().parent / "devmon-batch-1.schema.json" RETRY_STATUS = {408, 425, 429, 500, 502, 503, 504} @@ -101,9 +104,20 @@ def save_batch(batch, directory=None): or Path.home() / "logs" / "device-monitoring-batches") d.mkdir(parents=True, exist_ok=True) stamp = batch["run_timestamp"].replace(" ", "T").replace(":", "") - path = d / f"{stamp}Z-{batch['batch_id'][:12]}.json" - path.write_text(canonical(batch)) - return path + data = canonical(batch).encode("utf-8") + # The batch dir is group-writable on MSI: create the file exclusively and never follow a + # symlink someone else planted under the expected name. + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + for n in range(100): + path = d / f"{stamp}Z-{batch['batch_id'][:12]}{'' if n == 0 else f'-{n}'}.json" + try: + fd = os.open(path, flags, 0o640) + except FileExistsError: + continue + with os.fdopen(fd, "wb") as f: + f.write(data) + return path + raise PublishError(f"could not create a batch file in {d}") def publish(batch, url=None, token=None, attempts=None, backoff=None, sleep=time.sleep): @@ -113,9 +127,15 @@ def publish(batch, url=None, token=None, attempts=None, backoff=None, sleep=time so it raises at once. Re-sending an already-stored batch returns "duplicate" (no-op). """ url = (url or os.getenv("DEVICEMON_INGEST_URL") or "").rstrip("/") - token = token or os.getenv("DEVICEMON_INGEST_TOKEN") - if not url or not token: - raise PublishError("DEVICEMON_INGEST_URL and DEVICEMON_INGEST_TOKEN must both be set") + if not url: + raise PublishError("DEVICEMON_INGEST_URL must be set") + if not token: + try: + if not os.getenv("DEVICEMON_INGEST_TOKEN"): + url = ingest_token.check_ingest_url(url)[0] + token = ingest_token.load_token(url) + except ingest_token.TokenError as e: + raise PublishError(f"no ingest token: {e}") from None attempts = int(attempts or os.getenv("DEVICEMON_PUBLISH_ATTEMPTS", "5")) backoff = float(backoff or os.getenv("DEVICEMON_PUBLISH_BACKOFF", "15")) body = canonical(batch).encode("utf-8") diff --git a/tests/device_monitoring/test_ingest_token.py b/tests/device_monitoring/test_ingest_token.py new file mode 100644 index 0000000..26548af --- /dev/null +++ b/tests/device_monitoring/test_ingest_token.py @@ -0,0 +1,155 @@ +import base64 +import json +import os + +import pytest + +from rtgs_lab_tools.device_monitoring import ingest_token as it + +TEST_URL = "https://device-monitoring-ingest-test-685278787484.us-central1.run.app" +PROD_URL = "https://device-monitoring-ingest-prod-685278787484.us-central1.run.app" +SECRET_VALUE = "s3cr3t-token-value" + + +def key_file(tmp_path, mode=0o600, body=None): + p = tmp_path / "sa.json" + p.write_text(json.dumps(body or {"type": "service_account", "project_id": "rtgslab"})) + os.chmod(p, mode) + return p + + +class Resp: + def __init__(self, code, body): + self.status_code, self._body = code, body + + def json(self): + return self._body + + +class Session: + """Stands in for google.auth AuthorizedSession: records the URL, returns a canned reply.""" + + def __init__(self, reply): + self.reply, self.urls = reply, [] + + def __call__(self, info): + assert info["type"] == "service_account" + return self + + def get(self, url, timeout): + self.urls.append(url) + if isinstance(self.reply, Exception): + raise self.reply + return self.reply + + +def ok_reply(version=3): + return Resp(200, {"name": f"projects/685278787484/secrets/x/versions/{version}", + "payload": {"data": base64.b64encode(SECRET_VALUE.encode()).decode()}}) + + +def test_fetch_latest_version(tmp_path): + s = Session(ok_reply()) + token, version = it.fetch_from_secret_manager("devicemon-ingest-token-test", + key_path=key_file(tmp_path), session_factory=s) + assert token == SECRET_VALUE and version.endswith("/versions/3") + assert s.urls == ["https://secretmanager.googleapis.com/v1/projects/rtgslab/secrets/" + "devicemon-ingest-token-test/versions/latest:access"] + + +@pytest.mark.parametrize("mode", [0o640, 0o644, 0o660, 0o700]) +def test_key_mode_must_be_600(tmp_path, mode): + with pytest.raises(it.TokenError, match="must be 600"): + it.fetch_from_secret_manager("devicemon-ingest-token", key_path=key_file(tmp_path, mode), + session_factory=Session(ok_reply())) + + +def test_missing_key(tmp_path): + with pytest.raises(it.TokenError, match="not found"): + it.fetch_from_secret_manager("devicemon-ingest-token", key_path=tmp_path / "nope.json") + + +def test_symlinked_key_refused(tmp_path): + link = tmp_path / "link.json" + link.symlink_to(key_file(tmp_path)) + with pytest.raises(it.TokenError, match="symlink"): + it.fetch_from_secret_manager("devicemon-ingest-token", key_path=link, + session_factory=Session(ok_reply())) + + +def test_key_owned_by_someone_else(tmp_path, monkeypatch): + p = key_file(tmp_path) + monkeypatch.setattr(it.os, "geteuid", lambda: os.stat(p).st_uid + 1) + with pytest.raises(it.TokenError, match="not by the running user"): + it.fetch_from_secret_manager("devicemon-ingest-token", key_path=p, + session_factory=Session(ok_reply())) + + +def test_not_a_service_account_key(tmp_path): + with pytest.raises(it.TokenError, match="not a service-account key"): + it.fetch_from_secret_manager("devicemon-ingest-token", + key_path=key_file(tmp_path, body={"type": "user"}), + session_factory=Session(ok_reply())) + + +@pytest.mark.parametrize("code,transient", [(403, False), (404, False), (503, True)]) +def test_secret_manager_errors_never_echo_a_value(tmp_path, code, transient): + reply = Resp(code, {"error": {"status": "X", "message": "denied"}}) + with pytest.raises(it.TokenError) as e: + it.fetch_from_secret_manager("devicemon-ingest-token", key_path=key_file(tmp_path), + session_factory=Session(reply)) + assert f"HTTP {code}" in str(e.value) and e.value.transient is transient + assert SECRET_VALUE not in str(e.value) + + +def test_network_error_is_transient(tmp_path): + with pytest.raises(it.TokenError) as e: + it.fetch_from_secret_manager("devicemon-ingest-token", key_path=key_file(tmp_path), + session_factory=Session(ConnectionError("down"))) + assert e.value.transient + + +def test_secret_follows_url_and_pairs(): + assert it.secret_for(TEST_URL, env={}) == "devicemon-ingest-token-test" + assert it.secret_for(PROD_URL, env={}) == "devicemon-ingest-token" + with pytest.raises(it.TokenError, match="does not belong"): + it.secret_for(PROD_URL, env={"DEVICEMON_TOKEN_SECRET": "devicemon-ingest-token-test"}) + with pytest.raises(it.TokenError, match="not a valid secret name"): + it.secret_for(None, env={"DEVICEMON_TOKEN_SECRET": "../x"}) + + +@pytest.mark.parametrize("url", ["http://device-monitoring-ingest-test-685278787484" + ".us-central1.run.app", "https://attacker.run.app", + "https://device-monitoring-ingest-test-685278787484" + ".us-central1.run.app.evil.com", ""]) +def test_unknown_hosts_refused(url): + with pytest.raises(it.TokenError, match="refusing"): + it.check_ingest_url(url) + + +def test_load_token_uses_secret_manager_and_override(capsys): + seen = [] + tok = it.load_token(TEST_URL, env={}, fetch=lambda s: seen.append(s) or ("abc", "v1")) + assert tok == "abc" and seen == ["devicemon-ingest-token-test"] + assert it.load_token(TEST_URL, env={"DEVICEMON_INGEST_TOKEN": "t"}, + fetch=lambda s: pytest.fail("fetched")) == "t" + out = capsys.readouterr() + assert "abc" not in out.out + out.err and "test override" in out.err + + +def test_publish_end_to_end_with_mocked_secret_manager(tmp_path, monkeypatch): + from rtgs_lab_tools.device_monitoring import publisher + from tests.device_monitoring.test_publisher import FakeResponse, sample + + monkeypatch.delenv("DEVICEMON_INGEST_TOKEN", raising=False) + monkeypatch.delenv("DEVICEMON_TOKEN_SECRET", raising=False) + monkeypatch.setenv("DEVICEMON_SA_KEY", str(key_file(tmp_path))) + monkeypatch.setattr(it, "_session", Session(ok_reply())) + monkeypatch.setattr(it.fetch_from_secret_manager, "__defaults__", + (None, None, Session(ok_reply()))) + sent = {} + monkeypatch.setattr(publisher.requests, "post", lambda url, data, headers, timeout: + sent.update(url=url, auth=headers["Authorization"]) + or FakeResponse(201, {"status": "stored"})) + publisher.publish(sample(), url=TEST_URL + "/") + assert sent == {"url": TEST_URL + "/v1/batches", "auth": f"Bearer {SECRET_VALUE}"} diff --git a/tests/device_monitoring/test_publisher.py b/tests/device_monitoring/test_publisher.py index 987641c..53072d6 100644 --- a/tests/device_monitoring/test_publisher.py +++ b/tests/device_monitoring/test_publisher.py @@ -58,13 +58,41 @@ def test_bad_request_is_not_retried(monkeypatch): assert len(calls) == 1 -def test_needs_url_and_token(monkeypatch): +def test_needs_url(monkeypatch): monkeypatch.delenv("DEVICEMON_INGEST_URL", raising=False) monkeypatch.delenv("DEVICEMON_INGEST_TOKEN", raising=False) - with pytest.raises(publisher.PublishError): + with pytest.raises(publisher.PublishError, match="DEVICEMON_INGEST_URL"): publisher.publish(sample()) +def test_no_token_source_fails_closed(monkeypatch, tmp_path): + monkeypatch.delenv("DEVICEMON_INGEST_TOKEN", raising=False) + monkeypatch.setenv("DEVICEMON_SA_KEY", str(tmp_path / "missing.json")) + monkeypatch.setattr(publisher.requests, "post", lambda *a, **k: pytest.fail("posted")) + with pytest.raises(publisher.PublishError, match="not found"): + publisher.publish(sample(), url="https://device-monitoring-ingest-test-685278787484" + ".us-central1.run.app") + + +def test_token_never_sent_to_unknown_host(monkeypatch): + monkeypatch.delenv("DEVICEMON_INGEST_TOKEN", raising=False) + monkeypatch.setattr(publisher.requests, "post", lambda *a, **k: pytest.fail("posted")) + with pytest.raises(publisher.PublishError, match="refusing"): + publisher.publish(sample(), url="https://evil.example.run.app") + + +def test_save_batch_is_exclusive_and_ignores_planted_symlink(tmp_path): + b = sample() + target = tmp_path / "victim" + target.write_text("keep") + stamp = b["run_timestamp"].replace(" ", "T").replace(":", "") + (tmp_path / f"{stamp}Z-{b['batch_id'][:12]}.json").symlink_to(target) + p = publisher.save_batch(b, tmp_path) + assert target.read_text() == "keep" + assert p.name.endswith("-1.json") and json.loads(p.read_text())["batch_id"] == b["batch_id"] + assert "Bearer" not in p.read_text() + + def test_cron_uses_the_cli_flag_names(): script = (DM / "scheduled_device_monitoring.sh").read_text() for bad in ("--start_date", "--end_date", "--node_ids"): From a8790738ca197d8f5db1b4d8b6327e060625dc72 Mon Sep 17 00:00:00 2001 From: runck014 Date: Fri, 16 Oct 2026 12:05:29 -0500 Subject: [PATCH 7/7] MSI cron: credentials as data, tag+SHA pin, refuse shared code, safe logs - the shared credentials file is parsed (KEY=VALUE, allowlist), never sourced; *_DIR, *_FILE, PATH, LD_*, PYTHON*, GOOGLE_*, DEVICEMON_RELEASE and the token are refused from it - release pin from ~/.config/devmon/release (" "), both checked; git runs with core.hooksPath=/dev/null; no fetch --tags --force; track: mode removed - refuses to run if this script, its dir, the checkout or the venv is group/world-writable or not owned by the running user; umask 027 - log file created by mktemp and held open on fd 3; lock in ~/.cache/devmon - check-ingest preflight before the pipeline; DEVICEMON_INGEST_TOKEN unset in the cron - --check-creds lists accepted/rejected names (value lengths only) - CI runs all device_monitoring tests --- .github/workflows/devmon-publisher.yml | 3 +- .../device_monitoring/README.md | 149 +++----- .../scheduled_device_monitoring.sh | 358 ++++++++++-------- .../device_monitoring/scrontab.msi | 1 + tests/device_monitoring/test_cron_script.py | 136 +++++++ 5 files changed, 409 insertions(+), 238 deletions(-) create mode 100644 tests/device_monitoring/test_cron_script.py diff --git a/.github/workflows/devmon-publisher.yml b/.github/workflows/devmon-publisher.yml index 1dbfc0b..c20620a 100644 --- a/.github/workflows/devmon-publisher.yml +++ b/.github/workflows/devmon-publisher.yml @@ -35,9 +35,10 @@ jobs: pip install -e . pytest pytest-cov jsonschema - name: Publisher tests - run: pytest -q tests/device_monitoring/test_publisher.py + run: pytest -q --no-cov tests/device_monitoring - name: CLI takes the flags the cron passes run: | rtgs device-monitoring monitor --help | grep -q -- '--start-date' rtgs device-monitoring publish --help >/dev/null + rtgs device-monitoring check-ingest --help >/dev/null diff --git a/src/rtgs_lab_tools/device_monitoring/README.md b/src/rtgs_lab_tools/device_monitoring/README.md index d15254a..2832e6e 100644 --- a/src/rtgs_lab_tools/device_monitoring/README.md +++ b/src/rtgs_lab_tools/device_monitoring/README.md @@ -43,123 +43,94 @@ scrontab /projects/standard/runck014/shared/device_monitoring/rtgs-lab-tools-cro #### Script Location and Setup -The monitoring script is located at: +The monitoring script runs from the lab's shared workspace on Bryan's scrontab: ``` /projects/standard/runck014/shared/device_monitoring/scripts/scheduled_device_monitoring.sh -(the lab's shared workspace; runs from Bryan's scrontab. Paths are overridable: -DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE) ``` +Paths are overridable from the scrontab environment (never from the credentials file): +DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE, +DEVICEMON_RELEASE_FILE, DEVICEMON_SA_KEY, DEVICEMON_TOKEN_SECRET. -This script automatically: -- Loads required MSI modules (Python, Git) -- Clones/updates the rtgs-lab-tools repository -- Sets up the virtual environment -- Loads credentials from the shared credentials file -- Runs the device monitoring -- Logs all activities to `/projects/standard/runck014/shared/device_monitoring/logs/` (batches in `logs/batches/`) - -### MSI Credentials Configuration - -#### Credentials File Setup +Ownership (the script refuses to run if the code side is group- or world-writable): +``` +scripts/, rtgs-lab-tools-cron/ (checkout + venv) Bryan, 2750 dirs, 640/750 files +logs/, logs/batches/ Bryan, group runck014, 2770 (the one + group-writable place) +credentials/.rtgs_credentials 660, group runck014 -- data, never sourced +~/.config/devmon/ (700): sa.json (600), release (600) Bryan's home, not shared +``` -The shared credentials file (mode 660, group runck014; per-user `~/.rtgs_credentials` or -`~/.rtgs_creds` are fallbacks): +Each run: +- loads MSI modules (Python, Git) +- parses the credentials file against an allowlist (no `source`) +- checks out the release pinned in `~/.config/devmon/release` (` `; both + checked; git runs with hooks disabled; an existing tag is never force-moved) +- fetches the ingest token from Secret Manager with `~/.config/devmon/sa.json` + (`rtgs device-monitoring check-ingest` preflight), runs the monitor, publishes the batch +- logs to `logs/device_monitoring__.log`; the lock is in `~/.cache/devmon/` -```bash -# Create credentials file with restricted permissions -touch /projects/standard/runck014/shared/device_monitoring/credentials/.rtgs_credentials -chmod 660 /projects/standard/runck014/shared/device_monitoring/credentials/.rtgs_credentials -chgrp runck014 /projects/standard/runck014/shared/device_monitoring/credentials/.rtgs_credentials -``` +### MSI Credentials Configuration -#### Required Credentials +#### Shared credentials file (data only) -Edit `credentials/.rtgs_credentials` with the actual credentials: +`credentials/.rtgs_credentials` holds KEY=VALUE lines (an `export ` prefix and one pair of +outer quotes are accepted; `$(...)` and `$VAR` are taken literally). Accepted names: +DEVICEMON_INGEST_URL, PARTICLE_ACCESS_TOKEN, DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD, +LOGGING_DB_HOST, LOGGING_DB_PORT, LOGGING_DB_NAME, LOGGING_DB_USER, LOGGING_DB_PASSWORD, +POSTGRES_LOGGING_STATUS, GMAIL_USER, GMAIL_APP_PASSWORD, GMAIL_RECIPIENT. Everything else is +ignored and named in the log -- in particular DEVICEMON_INGEST_TOKEN (now from Secret +Manager), DEVICEMON_RELEASE, *_DIR, *_FILE, PATH, LD_*, PYTHON*, GOOGLE_*. ```bash # GEMS Database Configuration -export DB_HOST=sensing-0.msi.umn.edu -export DB_PORT=5433 -export DB_NAME=gems -export DB_USER=your_username -export DB_PASSWORD=your_password - -# Logging Database (optional) -export LOGGING_DB_HOST=34.170.80.6 -export LOGGING_DB_PORT=5432 -export LOGGING_DB_NAME=logs -export LOGGING_DB_USER=your_logging_username -export LOGGING_DB_PASSWORD=your_logging_password - -# Device Monitoring Database (Cloud SQL Postgres - backs the web dashboard) -# Publish target (rtgs-infrastructure device-monitoring-ingest-prod; URL from its -# deploy run summary, token from Secret Manager devicemon-ingest-token) -export DEVICEMON_INGEST_URL=https://device-monitoring-ingest-prod-....run.app -export DEVICEMON_INGEST_TOKEN=your_ingest_token -# The rtgs-lab-tools tag the cron runs (or track:devmon-v* for newest) -export DEVICEMON_RELEASE=devmon-v0.1.0 +DB_HOST=sensing-0.msi.umn.edu +DB_PORT=5433 +DB_NAME=gems +DB_USER=your_username +DB_PASSWORD=your_password + +# Publish target: device-monitoring-ingest-prod (or -test). The token is NOT here. +DEVICEMON_INGEST_URL=https://device-monitoring-ingest-prod-685278787484.us-central1.run.app # Particle Cloud API -export PARTICLE_ACCESS_TOKEN=your_particle_token +PARTICLE_ACCESS_TOKEN=your_particle_token # Email Notifications (optional) -export GMAIL_USER=your_email@gmail.com -export GMAIL_APP_PASSWORD="your_gmail_app_password" -export GMAIL_RECIPIENT=notification_recipient@umn.edu +GMAIL_USER=your_email@gmail.com +GMAIL_APP_PASSWORD="your_gmail_app_password" +GMAIL_RECIPIENT=notification_recipient@umn.edu ``` -#### Security Notes - -- **File permissions**: The credentials file has `660` permissions (owner + group runck014) -- **No VPN required**: MSI infrastructure has direct access to the database -- **Secure storage**: Credentials are stored locally on MSI, not in version control -- **Alternative names**: The script falls back to `~/.rtgs_credentials`, then `~/.rtgs_creds` +Check what a run would accept (names and value lengths only, never values): +```bash +scripts/scheduled_device_monitoring.sh --check-creds credentials/.rtgs_credentials +``` -### MSI Monitoring Workflow +#### Ingest token (Secret Manager) -#### Daily Execution Process +The token is read each run from `devicemon-ingest-token` (prod URL) or +`devicemon-ingest-token-test` (test URL) in project rtgslab, as the service account +`device-monitoring-msi@rtgslab.iam.gserviceaccount.com`, whose only grant is +secretAccessor on those two secrets. Its key is `~/.config/devmon/sa.json`, owned by the +running user, mode 600 -- anything else is refused. The token is sent only to the known +ingest hosts and is never logged or written to disk. -1. **Before 5:00 AM daily**: SLURM crontab triggers the monitoring script -2. **Environment setup**: Script loads Python and Git modules -3. **Code update**: The pinned release tag (`DEVICEMON_RELEASE`) is checked out -4. **Credential loading**: Credentials are loaded from `credentials/.rtgs_credentials` -5. **Monitoring execution**: Device monitoring runs across all projects -6. **Logging**: Results and logs are saved to `logs/` in the shared workspace -7. **Cleanup**: Virtual environment is properly deactivated +#### Release pin -#### Log File Management +`~/.config/devmon/release`, one line: `devmon-v0.1.0 <40-hex commit sha of that tag>`. +Changing this line is the deploy. -Monitoring logs are stored in: -``` -~/logs/device-monitoring-logs/device_monitoring_YYYYMMDD_HHMMSS.log -``` +### Log File Management -Each log file contains: -- Execution timestamps -- Git repository updates -- Credential loading status -- Monitoring command output -- Error messages and debugging information -- Cleanup status +Logs: `logs/device_monitoring_YYYYMMDD_HHMMSS_.log`; batches: +`logs/batches/Z-.json` (contain no credentials). #### Manual Execution -You can also run monitoring manually on MSI: - ```bash -# Load required modules -module load python -module load git - -# Run the monitoring script directly -/projects/standard/runck014/shared/device_monitoring/scripts/scheduled_device_monitoring.sh -(the lab's shared workspace; runs from Bryan's scrontab. Paths are overridable: -DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE) - -# Or run specific monitoring with parameters /projects/standard/runck014/shared/device_monitoring/scripts/scheduled_device_monitoring.sh -(the lab's shared workspace; runs from Bryan's scrontab. Paths are overridable: -DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE) 2024-01-01 2024-01-31 "e00fce68c7f2c2b634bb4b7d" LCCMR +# or with parameters +.../scripts/scheduled_device_monitoring.sh 2024-01-01 2024-01-31 "e00fce68c7f2c2b634bb4b7d" LCCMR ``` ## Python API Usage diff --git a/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh b/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh index 2548310..97eeef5 100755 --- a/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh +++ b/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh @@ -1,29 +1,99 @@ #!/bin/bash -export PYTHONUTF8=1 # ensure proper encoding so that this script can print emojis without error -export PYTHONUNBUFFERED=1 # flush python output to the log as it happens, so a run that dies mid-way still leaves a usable log -# Scheduled Device Monitoring Script for CRON +# Scheduled Device Monitoring Script for CRON (MSI scrontab, Bryan's account) # Based on .github/workflows/daily-device-monitoring.yml # # Usage: ./scheduled_device_monitoring.sh [start_date] [end_date] [node_ids] [project] -# All parameters are optional and will use defaults if not provided +# ./scheduled_device_monitoring.sh --check-creds +# parse a credentials file the way a run would and list the accepted / rejected +# names (never values), then exit +# All run parameters are optional and will use defaults if not provided. +# +# Trust model. The workspace is shared with the lab group, so only logs/ is group-writable. +# Everything that decides what code runs is Bryan's alone: +# - this script, its directory, the checkout and the venv: owned by the running user and not +# group- or world-writable (dirs 2750, files 640/750). The script refuses to run otherwise. +# - the release pin: ~/.config/devmon/release, " <40-hex commit>", owned by the running +# user, not group/world-writable. Both the tag and its commit are checked. +# - the ingest token: fetched from Secret Manager each run with ~/.config/devmon/sa.json (600) +# by the Python side; it is never in the environment, a file, or a command line. +# - the shared credentials file is DATA: KEY=VALUE lines parsed against an allowlist, never +# sourced. Paths, PATH, LD_*, PYTHON*, the release and the token are refused from it. # -# Tunable via the environment: +# Tunable via the environment (scrontab or the caller, not the credentials file): # DEVICEMON_MAX_ATTEMPTS how many times to run the report before giving up (default 3) -# DEVICEMON_ATTEMPT_TIMEOUT hard wall-clock limit per attempt, seconds (default 1800) -# DEVICEMON_RETRY_DELAY pause between attempts, seconds (default 300) +# DEVICEMON_ATTEMPT_TIMEOUT hard wall-clock limit per attempt, seconds (default 600) +# DEVICEMON_RETRY_DELAY pause between attempts, seconds (default 120) +# DEVICEMON_SA_KEY service-account key (default ~/.config/devmon/sa.json) +# DEVICEMON_TOKEN_SECRET secret name (default follows the URL: prod or -test) +# DEVICEMON_RELEASE_FILE release pin (default ~/.config/devmon/release) # -# Paths (defaults = the lab's shared MSI workspace, run from Bryan's scrontab): +# Paths (defaults = the lab's shared MSI workspace): # DEVICEMON_BASE_DIR /projects/standard/runck014/shared/device_monitoring # DEVICEMON_WORK_DIR $DEVICEMON_BASE_DIR/rtgs-lab-tools-cron (clone + venv live here) -# DEVICEMON_LOG_DIR $DEVICEMON_BASE_DIR/logs +# DEVICEMON_LOG_DIR $DEVICEMON_BASE_DIR/logs (2770: the one shared dir) # DEVICEMON_CREDS_FILE $DEVICEMON_BASE_DIR/credentials/.rtgs_credentials -# (mode 660, group runck014; holds DEVICEMON_INGEST_URL/TOKEN) -# DEVICEMON_BATCH_DIR $DEVICEMON_LOG_DIR/batches (each batch saved before it is sent) +# DEVICEMON_BATCH_DIR $DEVICEMON_LOG_DIR/batches (2770) +# lock file ~/.cache/devmon/device_monitoring.lock (Bryan-owned) set -e # Exit on any error +export PYTHONUTF8=1 # ensure proper encoding so that this script can print emojis without error +export PYTHONUNBUFFERED=1 # flush python output to the log as it happens + +# Names the shared credentials file may set. Everything else is refused (and named in the log). +CREDS_ALLOW=" DEVICEMON_INGEST_URL PARTICLE_ACCESS_TOKEN DB_HOST DB_PORT DB_NAME DB_USER \ +DB_PASSWORD LOGGING_DB_HOST LOGGING_DB_PORT LOGGING_DB_NAME LOGGING_DB_USER LOGGING_DB_PASSWORD \ +POSTGRES_LOGGING_STATUS GMAIL_USER GMAIL_APP_PASSWORD GMAIL_RECIPIENT " +CREDS_ACCEPTED="" +CREDS_REJECTED="" + +# parse_creds : export allowlisted KEY=VALUE lines. No eval, no source, no expansion: +# a value is taken literally after stripping one pair of matching outer quotes. +parse_creds() { + local file="$1" line name value n=0 + CREDS_ACCEPTED=""; CREDS_REJECTED="" + while IFS= read -r line || [ -n "$line" ]; do + n=$((n + 1)) + line="${line%$'\r'}" + [[ $line =~ ^[[:space:]]*(#|$) ]] && continue + if [[ $line =~ ^[[:space:]]*(export[[:space:]]+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$ ]]; then + name="${BASH_REMATCH[2]}"; value="${BASH_REMATCH[3]}" + else + CREDS_REJECTED+=" line$n(not-KEY=VALUE)"; continue + fi + if [[ $value =~ ^\"(.*)\"[[:space:]]*$ ]] || [[ $value =~ ^\'(.*)\'[[:space:]]*$ ]]; then + value="${BASH_REMATCH[1]}" + else + value="${value%%[[:space:]]#*}" # trailing comment on an unquoted value + value="${value%"${value##*[![:space:]]}"}" # trailing whitespace + fi + case "$name" in + DEVICEMON_INGEST_TOKEN) CREDS_REJECTED+=" $name(now-from-Secret-Manager:delete-it)" + continue ;; + *_DIR|*_FILE|PATH|LD_*|PYTHON*|DEVICEMON_*|GOOGLE_*|BASH*|ENV|HOME|IFS) + if [ "$name" != DEVICEMON_INGEST_URL ]; then + CREDS_REJECTED+=" $name(forbidden)"; continue + fi ;; + esac + if [[ $CREDS_ALLOW != *" $name "* ]]; then + CREDS_REJECTED+=" $name(not-allowlisted)"; continue + fi + export "$name=$value" + CREDS_ACCEPTED+=" $name" + done < "$file" +} + +if [ "${1:-}" = "--check-creds" ]; then + [ -r "${2:-}" ] || { echo "usage: $0 --check-creds " >&2; exit 2; } + ( parse_creds "$2" + for v in $CREDS_ACCEPTED; do val="${!v}"; echo "accepted $v len=${#val}"; done + for v in $CREDS_REJECTED; do echo "rejected $v"; done ) + exit 0 +fi + # Fingerprint of the copy now running; see the release pin below. -SELF_SHA="$(sha1sum < "${BASH_SOURCE[0]}")" +SELF="$(readlink -f "${BASH_SOURCE[0]}")" +SELF_SHA="$(sha1sum < "$SELF")" # Parse command line arguments START_DATE="${1:-}" @@ -39,44 +109,44 @@ PROJECT="${4:-ALL}" # log anything -- the moment #SCRON --time is reached. The worst case below is # MAX_ATTEMPTS*ATTEMPT_TIMEOUT + (MAX_ATTEMPTS-1)*RETRY_DELAY = 34 minutes, so # --time must be comfortably above that (01:00:00 is the intended pairing). -# Raising these values without raising --time makes the retries unreachable. MAX_ATTEMPTS="${DEVICEMON_MAX_ATTEMPTS:-3}" ATTEMPT_TIMEOUT="${DEVICEMON_ATTEMPT_TIMEOUT:-600}" RETRY_DELAY="${DEVICEMON_RETRY_DELAY:-120}" -# Set up working directory and log file. Group-shared workspace: files stay group-readable, -# never world-readable. -umask 007 +# Code and venv: group may read, never write. Logs inherit the setgid group of logs/. +umask 027 BASE_DIR="${DEVICEMON_BASE_DIR:-/projects/standard/runck014/shared/device_monitoring}" WORK_DIR="${DEVICEMON_WORK_DIR:-$BASE_DIR/rtgs-lab-tools-cron}" LOG_DIR="${DEVICEMON_LOG_DIR:-$BASE_DIR/logs}" CREDS_FILE="${DEVICEMON_CREDS_FILE:-$BASE_DIR/credentials/.rtgs_credentials}" +RELEASE_FILE="${DEVICEMON_RELEASE_FILE:-$HOME/.config/devmon/release}" export DEVICEMON_BATCH_DIR="${DEVICEMON_BATCH_DIR:-$LOG_DIR/batches}" -LOG_FILE="$LOG_DIR/device_monitoring_$(date +%Y%m%d_%H%M%S).log" -LOCK_FILE="$WORK_DIR/device_monitoring.lock" +LOCK_DIR="$HOME/.cache/devmon" +LOCK_FILE="$LOCK_DIR/device_monitoring.lock" +REPO_DIR="$WORK_DIR/rtgs-lab-tools" +# The token only ever comes from Secret Manager here. +unset DEVICEMON_INGEST_TOKEN DEVICEMON_RELEASE GOOGLE_APPLICATION_CREDENTIALS -# Create directories if they don't exist -mkdir -p "$WORK_DIR" -mkdir -p "$LOG_DIR" +mkdir -p "$LOG_DIR" "$DEVICEMON_BATCH_DIR" +mkdir -p -m 700 "$LOCK_DIR" + +# Unpredictable log name in the shared dir, created exclusively by mktemp, then held open on +# fd 3 for the whole run: a file swapped in under the same name later is never written to. +LOG_FILE="$(mktemp "$LOG_DIR/device_monitoring_$(date +%Y%m%d_%H%M%S)_XXXXXX.log")" +exec 3>>"$LOG_FILE" +chmod 640 "$LOG_FILE" -# Function to log messages log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S %Z')] $1" | tee -a "$LOG_FILE" + local msg="[$(date '+%Y-%m-%d %H:%M:%S %Z')] $1" + echo "$msg" + echo "$msg" >&3 } -# Simple cleanup function - just deactivate venv cleanup() { - # Deactivate venv if active - if [ -n "$VIRTUAL_ENV" ]; then - deactivate || true - log "Deactivated virtual environment" - fi - log "Cleanup completed - keeping persistent directory for reuse" + log "Script finished" } -# Record why we stopped. Previously cleanup() was defined but never wired up to -# a trap, so a script killed by a signal left no trace at all in the log. -# SIGKILL still cannot be caught, but everything else now says so on the way out. +# Record why we stopped. SIGKILL still cannot be caught, but everything else says so. on_signal() { log "ERROR: Received SIG$1 - aborting device monitoring" exit 128 @@ -86,16 +156,29 @@ trap 'on_signal INT' INT trap 'on_signal HUP' HUP trap cleanup EXIT +# fail_if_shared ...: refuse paths not owned by us or writable by group/other. +# Symlinks are skipped (their own mode is always 777); their targets are checked where they +# live inside the tree. +fail_if_shared() { + local p bad + for p in "$@"; do + [ -e "$p" ] || continue + bad="$(find "$p" ! -type l \( -perm /022 -o ! -uid "$(id -u)" \) -print -quit 2>/dev/null)" + if [ -n "$bad" ]; then + log "ERROR: $bad is group/world-writable or not owned by $(id -un); refusing to run." + log " Fix: chmod -R g-w,o-w '$p' (dirs 2750, files 640/750), owner $(id -un)" + exit 1 + fi + done +} + log "Starting daily device monitoring" log "Work directory: $WORK_DIR" -log "Log directory: $LOG_DIR" +log "Log file: $LOG_FILE" log "Parameters: start_date=$START_DATE, end_date=$END_DATE, node_ids=$NODE_IDS, project=$PROJECT" RETRY_BUDGET=$(( MAX_ATTEMPTS * ATTEMPT_TIMEOUT + (MAX_ATTEMPTS - 1) * RETRY_DELAY )) log "Retry policy: up to $MAX_ATTEMPTS attempts, ${ATTEMPT_TIMEOUT}s limit each, ${RETRY_DELAY}s between (worst case ${RETRY_BUDGET}s)" -# Under SLURM, say up front how much room we actually have. A job time limit -# below the retry budget means later attempts can never run, and the kill -# arrives as a SIGKILL that leaves no trace in this log. if [ -n "$SLURM_JOB_ID" ]; then log "Running as SLURM job $SLURM_JOB_ID" if command -v squeue &> /dev/null; then @@ -104,9 +187,13 @@ if [ -n "$SLURM_JOB_ID" ]; then fi fi -# Refuse to start if yesterday's run is somehow still alive. Two concurrent -# runs would write competing rows for the same nodes, and a hung run holding -# stale database connections is not something to pile onto. +# This script and the directory holding it decide what runs: they must be ours alone. +[ -O "$SELF" ] || { log "ERROR: $SELF is not owned by $(id -un)"; exit 1; } +fail_if_shared "$SELF" +find "$(dirname "$SELF")" -maxdepth 0 -perm /022 | grep -q . \ + && { log "ERROR: $(dirname "$SELF") is group/world-writable; refusing to run"; exit 1; } + +# One run at a time. The lock lives in Bryan's home so no one else can hold or replace it. if command -v flock &> /dev/null; then exec 9>"$LOCK_FILE" if ! flock -n 9; then @@ -118,11 +205,6 @@ else log "Warning: flock not available, cannot guard against overlapping runs" fi -# Change to work directory -cd "$WORK_DIR" -log "Changed to work directory: $(pwd)" - -# Load required modules (if running on HPC/cluster environment) log "Loading required modules" if command -v module &> /dev/null; then module load python || log "Warning: Could not load python module" @@ -131,97 +213,92 @@ else log "Module command not found, assuming modules are already available" fi -# Source RTGS credentials: the shared file (DEVICEMON_CREDS_FILE); the per-user files are a -# fallback for running from a personal account. +# Credentials: the shared file (DEVICEMON_CREDS_FILE) as data; per-user files are a fallback +# for running from a personal account. CREDS_LOADED=0 for f in "$CREDS_FILE" "$HOME/.rtgs_credentials" "$HOME/.rtgs_creds"; do [ -f "$f" ] || continue - source "$f"; CREDS_LOADED=1 - log "Loaded credentials from $f" + parse_creds "$f"; CREDS_LOADED=1 + log "Credentials from $f: accepted${CREDS_ACCEPTED:- nothing}" + [ -z "$CREDS_REJECTED" ] || log "WARNING: ignored in $f:$CREDS_REJECTED" break done if [ "$CREDS_LOADED" -eq 0 ]; then log "ERROR: No credentials found: expected $CREDS_FILE (or ~/.rtgs_credentials, ~/.rtgs_creds)" exit 1 fi - -# Where the results go. Publishing to device-monitoring-ingest is the supported path; the direct -# Cloud SQL write (DEVICEMON_INSTANCE_CONNECTION_NAME + a service-account key) is deprecated, and -# with neither set the pipeline would silently write a SQLite file inside this clone. -if [ -n "${DEVICEMON_INGEST_URL:-}" ] && [ -n "${DEVICEMON_INGEST_TOKEN:-}" ]; then - log "Results go to the ingest service: $DEVICEMON_INGEST_URL" -elif [ -n "${DEVICEMON_INSTANCE_CONNECTION_NAME:-}" ]; then - log "WARNING: DEVICEMON_INGEST_URL/TOKEN unset; writing Cloud SQL directly (deprecated)" -else - log "ERROR: set DEVICEMON_INGEST_URL and DEVICEMON_INGEST_TOKEN in $CREDS_FILE" +if [ -z "${DEVICEMON_INGEST_URL:-}" ]; then + log "ERROR: set DEVICEMON_INGEST_URL in $CREDS_FILE" + exit 1 +fi +log "Results go to the ingest service: $DEVICEMON_INGEST_URL" + +# Release pin: " " in a file only Bryan can write. +[ -f "$RELEASE_FILE" ] && [ -O "$RELEASE_FILE" ] \ + || { log "ERROR: release pin $RELEASE_FILE missing or not owned by $(id -un)"; exit 1; } +find "$RELEASE_FILE" -maxdepth 0 -perm /022 | grep -q . \ + && { log "ERROR: $RELEASE_FILE is group/world-writable"; exit 1; } +read -r PIN_TAG PIN_SHA _ < "$RELEASE_FILE" || true +if ! [[ $PIN_TAG =~ ^devmon-v[0-9A-Za-z._-]+$ && $PIN_SHA =~ ^[0-9a-f]{40}$ ]]; then + log "ERROR: $RELEASE_FILE must hold one line: <40-hex commit sha>" exit 1 fi -# Setup repository and pin the release. The cron runs a tagged release, never a moving branch. -# DEVICEMON_RELEASE (in the credentials file): -# devmon-v0.1.0 pinned: editing this one line on MSI is the deploy -# track:devmon-v* newest tag matching the glob, fetched every run: pushing a tag is the deploy -# (unset) stay on the current checkout, unpinned, with a warning -REPO_DIR="$WORK_DIR/rtgs-lab-tools" +# Git never runs repository hooks, and never force-moves a tag it already has. +G() { git -c core.hooksPath=/dev/null -c protocol.file.allow=never "$@"; } + +mkdir -p "$WORK_DIR" +fail_if_shared "$WORK_DIR" if [ ! -d "$REPO_DIR/.git" ]; then + if [ -e "$REPO_DIR" ]; then + log "ERROR: $REPO_DIR exists but is not a git checkout; move it aside and re-run" + exit 1 + fi log "Repository doesn't exist, cloning..." - rm -rf "$REPO_DIR" - git clone https://github.com/RTGS-Lab/rtgs-lab-tools.git "$REPO_DIR" + G clone --quiet --no-checkout https://github.com/RTGS-Lab/rtgs-lab-tools.git "$REPO_DIR" fi +fail_if_shared "$REPO_DIR" cd "$REPO_DIR" +log "Changed to repository: $(pwd)" -select_release() { - if ! git fetch --quiet --tags --force origin; then - log "WARNING: git fetch failed; staying on $(git describe --tags --always)" - return 0 - fi - local want="${DEVICEMON_RELEASE:-}" - if [ -z "$want" ]; then - log "WARNING: DEVICEMON_RELEASE unset; running $(git describe --tags --always) unpinned" - return 0 - fi - if [[ $want == track:* ]]; then - want="$(git tag -l "${want#track:}" --sort=-v:refname | head -n 1)" - [ -n "$want" ] || { log "ERROR: no tag matches ${DEVICEMON_RELEASE#track:}"; return 1; } - fi - git rev-parse -q --verify "refs/tags/$want" >/dev/null \ - || { log "ERROR: release tag $want not found"; return 1; } - if [ "$(git rev-parse HEAD)" != "$(git rev-parse "refs/tags/$want^{commit}")" ]; then - git checkout --quiet --detach "refs/tags/$want" - log "Checked out release $want" - else - log "Release $want already checked out" - fi - export DEVICEMON_RELEASE="$want" -} -select_release || exit 1 +if ! G rev-parse -q --verify "refs/tags/$PIN_TAG" >/dev/null; then + G fetch --quiet --no-tags origin "refs/tags/$PIN_TAG:refs/tags/$PIN_TAG" \ + || { log "ERROR: could not fetch tag $PIN_TAG"; exit 1; } +fi +TAG_SHA="$(G rev-parse "refs/tags/$PIN_TAG^{commit}")" +if [ "$TAG_SHA" != "$PIN_SHA" ]; then + log "ERROR: tag $PIN_TAG is $TAG_SHA here but the pin says $PIN_SHA; refusing to run." + log " If the tag was legitimately re-cut, update $RELEASE_FILE (and delete the local tag)." + exit 1 +fi +if [ "$(G rev-parse HEAD 2>/dev/null)" != "$PIN_SHA" ]; then + G checkout --quiet --detach "$PIN_SHA" + log "Checked out release $PIN_TAG ($PIN_SHA)" +else + log "Release $PIN_TAG ($PIN_SHA) already checked out" +fi +export DEVICEMON_RELEASE="$PIN_TAG" +fail_if_shared "$REPO_DIR" # If the release carries a different version of this script, run that one instead (once). NEW_SELF="$REPO_DIR/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh" if [ -z "${DEVICEMON_REEXEC:-}" ] && [ -f "$NEW_SELF" ] \ && [ "$(sha1sum < "$NEW_SELF")" != "$SELF_SHA" ]; then - log "Script differs in ${DEVICEMON_RELEASE:-this checkout}; re-running that copy" + log "Script differs in $PIN_TAG; re-running that copy" + exec 3>&- 9>&- exec env DEVICEMON_REEXEC=1 bash "$NEW_SELF" "$@" fi -# Only run installation if venv doesn't exist or if it's a fresh clone if [ ! -d "venv" ]; then log "Virtual environment not found, running installation" - bash install.sh -else - log "Virtual environment exists, skipping installation" + bash install.sh >&3 2>&1 fi - -# Check if virtual environment was created if [ ! -d "venv" ]; then log "ERROR: Virtual environment not found after installation" exit 1 fi +fail_if_shared "venv" -# Activate virtual environment. Check both layouts: venv puts the activate -# script in bin/ on Linux and Scripts/ on Windows, and a venv rebuilt by -# install.sh on the cluster would land in bin/ and silently break the hardcoded -# path under `set -e`. log "Activating virtual environment" if [ -f "venv/bin/activate" ]; then source venv/bin/activate @@ -232,71 +309,63 @@ else exit 1 fi - -# A new release may bring new dependencies: refresh the editable install once per release. +# A new release may bring new dependencies: refresh the editable install once per commit. STAMP="venv/.devmon-installed-release" -if [ "$(cat "$STAMP" 2>/dev/null)" != "${DEVICEMON_RELEASE:-unpinned}" ]; then - log "Installing rtgs-lab-tools for release ${DEVICEMON_RELEASE:-unpinned}" +if [ "$(cat "$STAMP" 2>/dev/null)" != "$PIN_TAG $PIN_SHA" ]; then + log "Installing rtgs-lab-tools for release $PIN_TAG" if command -v uv &> /dev/null; then - uv pip install --quiet -e . && echo "${DEVICEMON_RELEASE:-unpinned}" > "$STAMP" + uv pip install --quiet -e . >&3 2>&1 else - pip install --quiet -e . && echo "${DEVICEMON_RELEASE:-unpinned}" > "$STAMP" + pip install --quiet -e . >&3 2>&1 fi + echo "$PIN_TAG $PIN_SHA" > "$STAMP" + fail_if_shared "venv" fi -# Verify rtgs command is available if ! command -v rtgs &> /dev/null; then log "ERROR: rtgs command not found in virtual environment" exit 1 fi -# Build the monitoring command -MONITOR_CMD="rtgs device-monitoring monitor" - -# Add optional parameters if provided -if [ -n "$START_DATE" ]; then - MONITOR_CMD="$MONITOR_CMD --start-date=$START_DATE" -fi - -if [ -n "$END_DATE" ]; then - MONITOR_CMD="$MONITOR_CMD --end-date=$END_DATE" -fi - -if [ -n "$NODE_IDS" ]; then - MONITOR_CMD="$MONITOR_CMD --node-ids=$NODE_IDS" +# Fail fast on setup problems (key file, IAM, URL) before the 80-second pipeline runs. +# Exit 2 = setup is wrong; 3 = transient, so carry on and let the attempts retry. +set +e +rtgs device-monitoring check-ingest >&3 2>&1 +CHECK=$? +set -e +if [ "$CHECK" -eq 2 ]; then + log "ERROR: ingest preflight failed (setup); see the check-ingest lines above in $LOG_FILE" + exit 1 +elif [ "$CHECK" -ne 0 ]; then + log "WARNING: ingest preflight failed (exit $CHECK); continuing, publish retries later" +else + log "Ingest preflight ok" fi -if [ "$PROJECT" != "ALL" ]; then - MONITOR_CMD="$MONITOR_CMD --project=$PROJECT" -fi +MONITOR_ARGS=(device-monitoring monitor) +[ -z "$START_DATE" ] || MONITOR_ARGS+=("--start-date=$START_DATE") +[ -z "$END_DATE" ] || MONITOR_ARGS+=("--end-date=$END_DATE") +[ -z "$NODE_IDS" ] || MONITOR_ARGS+=("--node-ids=$NODE_IDS") +[ "$PROJECT" = "ALL" ] || MONITOR_ARGS+=("--project=$PROJECT") # Pin one timestamp for the whole day and keep it across retries, so a retried -# report replaces the rows of the attempt it is repeating rather than adding a -# second near-identical entry to the web app's list of monitoring timestamps. -# -u because the pipeline stores UTC throughout: device timestamps come from -# GEMS publish_time (UTC), and mixing in the cluster's local Central time is -# what made "last connected" read five hours ahead of the report that found it. +# report replaces the rows of the attempt it is repeating. UTC throughout. export DEVICEMON_RUN_TIMESTAMP="$(date -u '+%Y-%m-%d %H:%M')" log "Monitoring run timestamp: $DEVICEMON_RUN_TIMESTAMP UTC" -# Cap each attempt so a hang becomes a failure the retry loop can act on. -# --kill-after sends SIGKILL if the process ignores the initial SIGTERM. +TIMEOUT_PREFIX=() if command -v timeout &> /dev/null; then - TIMEOUT_PREFIX="timeout --kill-after=60 $ATTEMPT_TIMEOUT" + TIMEOUT_PREFIX=(timeout --kill-after=60 "$ATTEMPT_TIMEOUT") else log "Warning: timeout not available, attempts will run unbounded" - TIMEOUT_PREFIX="" fi EXIT_CODE=1 ATTEMPT=1 - while [ "$ATTEMPT" -le "$MAX_ATTEMPTS" ]; do - log "Attempt $ATTEMPT/$MAX_ATTEMPTS - running command: $MONITOR_CMD" - - # set +e so a failed attempt does not abort the script before it can retry. + log "Attempt $ATTEMPT/$MAX_ATTEMPTS - running: rtgs ${MONITOR_ARGS[*]}" set +e - $TIMEOUT_PREFIX $MONITOR_CMD >> "$LOG_FILE" 2>&1 + "${TIMEOUT_PREFIX[@]}" rtgs "${MONITOR_ARGS[@]}" >&3 2>&1 EXIT_CODE=$? set -e @@ -304,28 +373,21 @@ while [ "$ATTEMPT" -le "$MAX_ATTEMPTS" ]; do log "Device monitoring completed successfully on attempt $ATTEMPT" break fi - # 124 = timeout sent SIGTERM; 137 = it had to escalate to SIGKILL. if [ "$EXIT_CODE" -eq 124 ] || [ "$EXIT_CODE" -eq 137 ]; then log "ERROR: Attempt $ATTEMPT hit the ${ATTEMPT_TIMEOUT}s limit and was killed (exit $EXIT_CODE)" else log "ERROR: Attempt $ATTEMPT failed with exit code $EXIT_CODE" fi - if [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; then log "Retrying in ${RETRY_DELAY}s" sleep "$RETRY_DELAY" fi - ATTEMPT=$((ATTEMPT + 1)) done if [ "$EXIT_CODE" -ne 0 ]; then log "ERROR: Device monitoring failed on all $MAX_ATTEMPTS attempts - no report for today" fi - -# Virtual environment is deactivated by cleanup() on exit. log "Script completed with exit code $EXIT_CODE" - -# Exit with the same code as the monitoring command exit $EXIT_CODE diff --git a/src/rtgs_lab_tools/device_monitoring/scrontab.msi b/src/rtgs_lab_tools/device_monitoring/scrontab.msi index a0421e3..f3fb87d 100644 --- a/src/rtgs_lab_tools/device_monitoring/scrontab.msi +++ b/src/rtgs_lab_tools/device_monitoring/scrontab.msi @@ -3,6 +3,7 @@ # re-installed: scrontab (replaces the account's whole scrontab -- merge by hand # if it holds other jobs). scripts/scheduled_device_monitoring.sh may be a stale copy: the # script re-runs the pinned release's own copy when they differ. +# Needs ~/.config/devmon/release and ~/.config/devmon/sa.json (600) in the account's home. # --time must exceed the script's retry budget (3 x 600 s + 2 x 120 s = 34 min). #SCRON --time=01:00:00 #SCRON --job-name=devmon-publish diff --git a/tests/device_monitoring/test_cron_script.py b/tests/device_monitoring/test_cron_script.py new file mode 100644 index 0000000..d7e4a22 --- /dev/null +++ b/tests/device_monitoring/test_cron_script.py @@ -0,0 +1,136 @@ +"""The MSI cron script: credentials parsing and the refuse-to-run checks.""" +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from rtgs_lab_tools.device_monitoring import publisher + +SCRIPT = Path(publisher.__file__).resolve().parent / "scheduled_device_monitoring.sh" +GIT_ENV = {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t"} + + +def check_creds(tmp_path, text): + f = tmp_path / "creds" + f.write_text(text) + out = subprocess.run(["bash", str(SCRIPT), "--check-creds", str(f)], capture_output=True, + text=True, check=True, cwd=tmp_path).stdout + return out.split("\n") + + +def test_parser_allowlist(tmp_path): + marker = tmp_path / "pwned" + lines = check_creds(tmp_path, f"""# comment +export DEVICEMON_INGEST_URL="https://device-monitoring-ingest-test-685278787484.us-central1.run.app" +DB_HOST=gems.example.org # trailing comment +DB_PASSWORD='pa ss$(touch {marker})' +PARTICLE_ACCESS_TOKEN=abc$(touch {marker}) +PATH=/tmp/evil +LD_PRELOAD=/tmp/evil.so +PYTHONPATH=/tmp/evil +DEVICEMON_WORK_DIR=/tmp/evil +DEVICEMON_RELEASE=devmon-v9 +DEVICEMON_INGEST_TOKEN=leaked +GOOGLE_APPLICATION_CREDENTIALS=/tmp/k.json +SOMETHING_ELSE=1 +$(touch {marker}) +""") + assert not marker.exists() # nothing was executed + url = "https://device-monitoring-ingest-test-685278787484.us-central1.run.app" + assert f"accepted DEVICEMON_INGEST_URL len={len(url)}" in lines + assert f"accepted DB_HOST len={len('gems.example.org')}" in lines + assert f"accepted DB_PASSWORD len={len('pa ss$(touch ' + str(marker) + ')')}" in lines + assert any(l.startswith("accepted PARTICLE_ACCESS_TOKEN") for l in lines) + rejected = " ".join(l for l in lines if l.startswith("rejected")) + for name in ("PATH", "LD_PRELOAD", "PYTHONPATH", "DEVICEMON_WORK_DIR", "DEVICEMON_RELEASE", + "DEVICEMON_INGEST_TOKEN", "GOOGLE_APPLICATION_CREDENTIALS", "SOMETHING_ELSE"): + assert f"rejected {name}(" in rejected + assert "not-KEY=VALUE" in rejected + assert "leaked" not in "".join(lines) and "pa ss" not in "".join(lines) + + +def test_no_set_x_or_source_of_creds(): + text = SCRIPT.read_text() + assert "set -x" not in text and "bash -x" not in text + assert 'source "$f"' not in text and "fetch --tags --force" not in text + assert "core.hooksPath=/dev/null" in text + + +@pytest.fixture +def ws(tmp_path): + """A fake shared workspace + home; the script copy is ours and 750.""" + home, base = tmp_path / "home", tmp_path / "shared" + (home / ".config/devmon").mkdir(parents=True) + (base / "scripts").mkdir(parents=True) + (base / "credentials").mkdir() + (base / "logs").mkdir() + os.chmod(base / "logs", 0o2770) + script = base / "scripts/scheduled_device_monitoring.sh" + shutil.copy(SCRIPT, script) + os.chmod(script, 0o750) + os.chmod(base / "scripts", 0o2750) + (base / "credentials/.rtgs_credentials").write_text( + "DEVICEMON_INGEST_URL=https://device-monitoring-ingest-test-685278787484" + ".us-central1.run.app\n") + # a pinned "release": a local repo with one tagged commit and a hook that must never run + repo = base / "rtgs-lab-tools-cron/rtgs-lab-tools" + repo.mkdir(parents=True) + env = {**os.environ, **GIT_ENV} + subprocess.run(["git", "init", "-q", str(repo)], check=True, env=env) + (repo / "install.sh").write_text("exit 0\n") + subprocess.run(["git", "-C", str(repo), "add", "."], check=True, env=env) + subprocess.run(["git", "-C", str(repo), "commit", "-qm", "r"], check=True, env=env) + subprocess.run(["git", "-C", str(repo), "tag", "devmon-v0.0.1"], check=True, env=env) + hook = repo / ".git/hooks/post-checkout" + hook.write_text(f"#!/bin/sh\ntouch {tmp_path}/hook-ran\n") + os.chmod(hook, 0o750) + sha = subprocess.run(["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, + text=True, check=True).stdout.strip() + subprocess.run(["chmod", "-R", "g-w,o-w", str(base / "rtgs-lab-tools-cron")], check=True) + os.chmod(base / "rtgs-lab-tools-cron", 0o2750) + return {"home": home, "base": base, "script": script, "sha": sha, "tmp": tmp_path} + + +def run(ws, release=None): + if release is not None: + f = ws["home"] / ".config/devmon/release" + f.write_text(release) + os.chmod(f, 0o600) + env = {"HOME": str(ws["home"]), "PATH": os.environ["PATH"], + "DEVICEMON_BASE_DIR": str(ws["base"]), "DEVICEMON_INGEST_TOKEN": "must-be-unset"} + return subprocess.run(["bash", str(ws["script"])], capture_output=True, text=True, env=env, + cwd=ws["tmp"], timeout=60) + + +def test_refuses_group_writable_script(ws): + os.chmod(ws["script"], 0o770) + r = run(ws, f"devmon-v0.0.1 {ws['sha']}\n") + assert r.returncode == 1 and "group/world-writable" in r.stdout + + +def test_refuses_group_writable_repo(ws): + os.chmod(ws["base"] / "rtgs-lab-tools-cron/rtgs-lab-tools", 0o2770) + r = run(ws, f"devmon-v0.0.1 {ws['sha']}\n") + assert r.returncode == 1 and "group/world-writable" in r.stdout + + +def test_refuses_missing_or_bad_release_pin(ws): + assert "release pin" in run(ws).stdout + assert "must hold one line" in run(ws, "devmon-v0.0.1\n").stdout + r = run(ws, "devmon-v0.0.1 " + "0" * 40 + "\n") + assert r.returncode == 1 and "the pin says" in r.stdout + + +def test_pinned_release_checks_out_without_hooks(ws): + r = run(ws, f"devmon-v0.0.1 {ws['sha']}\n") + # gets past the pin and stops at the (fake) venv, having run no repository hook + assert f"devmon-v0.0.1 ({ws['sha']})" in r.stdout + assert "Virtual environment not found after installation" in r.stdout + assert not (ws["tmp"] / "hook-ran").exists() + logs = list((ws["base"] / "logs").glob("device_monitoring_*.log")) + assert len(logs) == 1 and len(logs[0].name.split("_")[-1]) == len("XXXXXX.log") + assert (logs[0].stat().st_mode & 0o777) == 0o640 + assert (ws["home"] / ".cache/devmon/device_monitoring.lock").exists()