From 70bb47a3a5cb27c879f8a6a9a9736f2a055e0419 Mon Sep 17 00:00:00 2001 From: runck014 Date: Fri, 16 Oct 2026 13:33:21 -0500 Subject: [PATCH 1/2] rtgs_lab_tools: resolve top-level names lazily (PEP 562) Importing any subpackage runs the package __init__ first, and it eagerly imported visualization (matplotlib), device_configuration, sensing_data and the parsers. Every tool, including the MSI device-monitoring cron, paid for all of them. The same public names now resolve on first use; `from rtgs_lab_tools import X` is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0149VSJ7dmCkXd4UcUjQbG9R --- src/rtgs_lab_tools/__init__.py | 56 ++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/src/rtgs_lab_tools/__init__.py b/src/rtgs_lab_tools/__init__.py index 3f06ceb6..954110a9 100644 --- a/src/rtgs_lab_tools/__init__.py +++ b/src/rtgs_lab_tools/__init__.py @@ -4,34 +4,44 @@ __author__ = "RTGS Lab" __email__ = "rtgs@umn.edu" -# Core infrastructure -from .core import Config, DatabaseManager -from .core.exceptions import APIError, DatabaseError, ValidationError -from .core.postgres_logger import PostgresLogger -from .data_parser.parsers.data_parser import DataV2Parser +# Public names resolve on first use (PEP 562), not at import. Importing any subpackage imports +# this file first, so eager imports here made every tool -- including the MSI device-monitoring +# cron -- load matplotlib, the Particle client and the parsers whether it used them or not. +_LAZY = { + "Config": ".core", + "DatabaseManager": ".core", + "APIError": ".core.exceptions", + "DatabaseError": ".core.exceptions", + "ValidationError": ".core.exceptions", + "PostgresLogger": ".core.postgres_logger", + "DataV2Parser": ".data_parser.parsers.data_parser", + "ParserFactory": ".data_parser.parsers.factory", + "ParticleClient": ".device_configuration", + "ParticleConfigUpdater": ".device_configuration", + "extract_data": ".sensing_data", + "get_raw_data": ".sensing_data", + "list_available_projects": ".sensing_data", + "create_multi_parameter_plot": ".visualization", + "create_time_series_plot": ".visualization", + "detect_data_type": ".visualization", + "load_and_prepare_data": ".visualization", + # Climate data: download_GEE_raster stays out (slow import), as before. +} -# Data parsing functions -from .data_parser.parsers.factory import ParserFactory -# Device management -from .device_configuration import ParticleClient, ParticleConfigUpdater +def __getattr__(name): + module = _LAZY.get(name) + if module is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + import importlib -# High-level data extraction functions -from .sensing_data import extract_data, get_raw_data, list_available_projects + value = getattr(importlib.import_module(module, __name__), name) + globals()[name] = value # cache: later lookups skip __getattr__ + return value -# Visualization functions -from .visualization import ( - create_multi_parameter_plot, - create_time_series_plot, - detect_data_type, - load_and_prepare_data, -) -# Climate data -# from .gridded_data import download_GEE_raster # Commented out to avoid slow imports - - -# Error analysis +def __dir__(): + return sorted(set(globals()) | set(_LAZY)) __all__ = [ From 036070ab469afcc773e7e8e1515692fa81f4c06a Mon Sep 17 00:00:00 2001 From: runck014 Date: Fri, 16 Oct 2026 13:33:21 -0500 Subject: [PATCH 2/2] device_monitoring cron: lean venv on a pinned Python >= 3.10 The first MSI run spent 24 min in install.sh (`uv pip install -e ".[all]"`: streamlit, pyarrow, netcdf4, mypy, ...) and stalled; on agate a bare `module load python` also left /usr/bin/python3 = 3.6.8 on PATH. - requirements-cron.txt: the 11 packages the publisher path imports (pins from uv.lock). - The script loads DEVICEMON_PYTHON_MODULE (default python3/3.12.4_anaconda2024.06-1_libmamba) or takes DEVICEMON_PYTHON, picks python3/python >= 3.10 by absolute path, and stops with a message otherwise. It builds venv/ with `$PY -m venv`, then `pip install -r requirements-cron.txt` + `pip install --no-deps -e .`, once per release. It never runs install.sh or the base/extra dependencies. A venv on an old Python is refused. - The cron runs `venv/bin/python -m rtgs_lab_tools.device_monitoring.cli` (a missing module is a traceback, not "No such command"); `rtgs device-monitoring` still works on full installs. - CI installs exactly requirements-cron.txt, so a new import fails the PR, not the 04:27 run. - Tests: an old interpreter is refused before any venv; the venv build runs only the two lean pip commands, is not group-writable, runs check-ingest then monitor through the module CLI, and a second run of the same release installs nothing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0149VSJ7dmCkXd4UcUjQbG9R --- .github/workflows/devmon-publisher.yml | 12 ++- .../device_monitoring/README.md | 12 ++- .../device_monitoring/requirements-cron.txt | 20 +++++ .../scheduled_device_monitoring.sh | 90 +++++++++++++------ tests/device_monitoring/test_cron_script.py | 85 +++++++++++++++++- 5 files changed, 181 insertions(+), 38 deletions(-) create mode 100644 src/rtgs_lab_tools/device_monitoring/requirements-cron.txt diff --git a/.github/workflows/devmon-publisher.yml b/.github/workflows/devmon-publisher.yml index c20620a1..ab6c28a0 100644 --- a/.github/workflows/devmon-publisher.yml +++ b/.github/workflows/devmon-publisher.yml @@ -29,16 +29,22 @@ jobs: - name: Cron script parses run: bash -n src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh - - name: Install + # Exactly what the MSI cron installs: if the publisher path gains an import that is not in + # requirements-cron.txt, this job fails instead of the 04:27 run. + - name: Install (lean, as the cron does) run: | python -m pip install --upgrade pip - pip install -e . pytest pytest-cov jsonschema + pip install -r src/rtgs_lab_tools/device_monitoring/requirements-cron.txt + pip install --no-deps -e . + pip install pytest jsonschema - name: Publisher tests - run: pytest -q --no-cov tests/device_monitoring + run: pytest -q -o addopts= tests/device_monitoring - name: CLI takes the flags the cron passes run: | rtgs device-monitoring monitor --help | grep -q -- '--start-date' rtgs device-monitoring publish --help >/dev/null rtgs device-monitoring check-ingest --help >/dev/null + python -m rtgs_lab_tools.device_monitoring.cli monitor --help | grep -q -- '--no-email' + python -m rtgs_lab_tools.device_monitoring.cli check-ingest --help >/dev/null diff --git a/src/rtgs_lab_tools/device_monitoring/README.md b/src/rtgs_lab_tools/device_monitoring/README.md index 2832e6e0..9f0908da 100644 --- a/src/rtgs_lab_tools/device_monitoring/README.md +++ b/src/rtgs_lab_tools/device_monitoring/README.md @@ -49,7 +49,8 @@ The monitoring script runs from the lab's shared workspace on Bryan's scrontab: ``` Paths are overridable from the scrontab environment (never from the credentials file): DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE, -DEVICEMON_RELEASE_FILE, DEVICEMON_SA_KEY, DEVICEMON_TOKEN_SECRET. +DEVICEMON_RELEASE_FILE, DEVICEMON_SA_KEY, DEVICEMON_TOKEN_SECRET, DEVICEMON_PYTHON_MODULE +(default `python3/3.12.4_anaconda2024.06-1_libmamba`), DEVICEMON_PYTHON (absolute path). Ownership (the script refuses to run if the code side is group- or world-writable): ``` @@ -61,12 +62,17 @@ credentials/.rtgs_credentials 660, group runck014 -- data, ``` Each run: -- loads MSI modules (Python, Git) +- loads MSI modules (Python, Git) and picks a Python >= 3.10 by absolute path; an older one + (agate's `/usr/bin/python3` is 3.6.8) stops the run with a message, before any install +- builds `venv/` lean on first run and once per release: `pip install -r + device_monitoring/requirements-cron.txt` then `pip install --no-deps -e .` -- never + `install.sh` or the package's full base dependencies (minutes, not 25+) - parses the credentials file against an allowlist (no `source`) - checks out the release pinned in `~/.config/devmon/release` (` `; both checked; git runs with hooks disabled; an existing tag is never force-moved) - fetches the ingest token from Secret Manager with `~/.config/devmon/sa.json` - (`rtgs device-monitoring check-ingest` preflight), runs the monitor, publishes the batch + (`check-ingest` preflight), runs the monitor, publishes the batch -- all through + `venv/bin/python -m rtgs_lab_tools.device_monitoring.cli`, not the `rtgs` router - logs to `logs/device_monitoring__.log`; the lock is in `~/.cache/devmon/` ### MSI Credentials Configuration diff --git a/src/rtgs_lab_tools/device_monitoring/requirements-cron.txt b/src/rtgs_lab_tools/device_monitoring/requirements-cron.txt new file mode 100644 index 00000000..38229d7c --- /dev/null +++ b/src/rtgs_lab_tools/device_monitoring/requirements-cron.txt @@ -0,0 +1,20 @@ +# What the MSI device-monitoring cron needs, and nothing else. scheduled_device_monitoring.sh +# builds its venv with: python -m pip install -r requirements-cron.txt +# python -m pip install --no-deps -e . +# The package's base dependencies (earthengine, geopandas, streamlit, pyarrow, matplotlib, ...) +# serve the other tools and are deliberately not installed for the cron. +# Pins follow uv.lock. Verified on Python 3.12 against tests/device_monitoring and a run of +# check-ingest + monitor with the network and database faked. Add a line here when the +# publisher path gains an import; CI installs exactly this file (devmon-publisher.yml). +# (requirements.txt beside this file is the web app's, not the cron's.) +click==8.2.1 # CLI +requests==2.32.4 # ingest POST, Particle lookups +google-auth==2.40.3 # service-account key -> Secret Manager (ingest token) +sqlalchemy==2.0.41 # GEMS raw-data query (<2.1: keep postgresql:// on psycopg2) +psycopg2-binary==2.9.10 # driver for the postgresql:// GEMS URL +pandas==2.3.1 # parse / analyze (brings numpy) +python-dotenv==1.1.1 # core.config, message_builder +yagmail==0.15.293 # notification e-mail +chardet<6 # yagmail->premailer->cssutils pulls chardet 7, which requests warns on +flask==3.1.3 # device_monitoring.core imports web_app.models (record builders) +flask-sqlalchemy==3.1.1 # same diff --git a/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh b/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh index 97eeef53..d3401f93 100755 --- a/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh +++ b/src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh @@ -26,6 +26,11 @@ # DEVICEMON_SA_KEY service-account key (default ~/.config/devmon/sa.json) # DEVICEMON_TOKEN_SECRET secret name (default follows the URL: prod or -test) # DEVICEMON_RELEASE_FILE release pin (default ~/.config/devmon/release) +# DEVICEMON_PYTHON_MODULE module that provides Python >= 3.10 (default +# python3/3.12.4_anaconda2024.06-1_libmamba; agate's bare +# `module load python` left /usr/bin/python3 = 3.6.8 on PATH) +# DEVICEMON_PYTHON interpreter to build the venv with (absolute path); skips the +# module lookup # # Paths (defaults = the lab's shared MSI workspace): # DEVICEMON_BASE_DIR /projects/standard/runck014/shared/device_monitoring @@ -206,13 +211,41 @@ else fi log "Loading required modules" +PYTHON_MODULE="${DEVICEMON_PYTHON_MODULE:-python3/3.12.4_anaconda2024.06-1_libmamba}" if command -v module &> /dev/null; then - module load python || log "Warning: Could not load python module" + if [ -z "${DEVICEMON_PYTHON:-}" ]; then + module load "$PYTHON_MODULE" || log "Warning: Could not load module $PYTHON_MODULE" + fi module load git || log "Warning: Could not load git module" else log "Module command not found, assuming modules are already available" fi +# The interpreter the venv is built from, by absolute path. A module may put python3, or only +# python, on PATH; take the first that is new enough. Anything older fails here, not 20 minutes +# into a pip run that builds sdists for an interpreter the code cannot run on. +py_ok() { "$1" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null; } +py_ver() { "$1" -c 'import sys; print(sys.version.split()[0])' 2>/dev/null || echo "?"; } +PY="" +if [ -n "${DEVICEMON_PYTHON:-}" ]; then + PY="$DEVICEMON_PYTHON" + py_ok "$PY" || { log "ERROR: DEVICEMON_PYTHON=$PY is Python $(py_ver "$PY"); need >= 3.10" + exit 1; } +else + for cand in python3 python; do + c="$(command -v "$cand" 2>/dev/null)" || continue + if py_ok "$c"; then PY="$c"; break; fi + log "Skipping $c: Python $(py_ver "$c") (need >= 3.10)" + done + if [ -z "$PY" ]; then + log "ERROR: no Python >= 3.10 on PATH after 'module load $PYTHON_MODULE'." + log " Check 'module avail python', then set DEVICEMON_PYTHON_MODULE (or" + log " DEVICEMON_PYTHON=/abs/path/python3) in the scrontab line." + exit 1 + fi +fi +log "Python for the venv: $PY ($(py_ver "$PY"))" + # Credentials: the shared file (DEVICEMON_CREDS_FILE) as data; per-user files are a fallback # for running from a personal account. CREDS_LOADED=0 @@ -289,48 +322,47 @@ if [ -z "${DEVICEMON_REEXEC:-}" ] && [ -f "$NEW_SELF" ] \ exec env DEVICEMON_REEXEC=1 bash "$NEW_SELF" "$@" fi -if [ ! -d "venv" ]; then - log "Virtual environment not found, running installation" - bash install.sh >&3 2>&1 +# The cron's venv holds only what the publisher needs (requirements-cron.txt), never the +# package's full base dependencies, install.sh, or the dev/all extras. +REQS="src/rtgs_lab_tools/device_monitoring/requirements-cron.txt" +VPY="$REPO_DIR/venv/bin/python" +if [ ! -f "$REQS" ]; then + log "ERROR: $REQS is not in release $PIN_TAG; cannot build the venv" + exit 1 fi if [ ! -d "venv" ]; then - log "ERROR: Virtual environment not found after installation" + log "Virtual environment not found, creating it with $PY" + "$PY" -m venv venv >&3 2>&1 +fi +if [ ! -x "$VPY" ]; then + log "ERROR: Virtual environment not found after installation ($VPY missing)" exit 1 fi -fail_if_shared "venv" - -log "Activating virtual environment" -if [ -f "venv/bin/activate" ]; then - source venv/bin/activate -elif [ -f "venv/Scripts/activate" ]; then - source venv/Scripts/activate -else - log "ERROR: No activate script found in venv/bin or venv/Scripts" +if ! py_ok "$VPY"; then + log "ERROR: venv/ runs Python $(py_ver "$VPY"); need >= 3.10. Move $REPO_DIR/venv aside" + log " and re-run: the venv is rebuilt with $PY." exit 1 fi +fail_if_shared "venv" -# A new release may bring new dependencies: refresh the editable install once per commit. +# A new release may bring new dependencies: refresh once per commit, still lean. STAMP="venv/.devmon-installed-release" if [ "$(cat "$STAMP" 2>/dev/null)" != "$PIN_TAG $PIN_SHA" ]; then - log "Installing rtgs-lab-tools for release $PIN_TAG" - if command -v uv &> /dev/null; then - uv pip install --quiet -e . >&3 2>&1 - else - pip install --quiet -e . >&3 2>&1 - fi + log "Installing the device-monitoring requirements for release $PIN_TAG" + "$VPY" -m pip install --quiet --disable-pip-version-check -r "$REQS" >&3 2>&1 + "$VPY" -m pip install --quiet --disable-pip-version-check --no-deps -e . >&3 2>&1 echo "$PIN_TAG $PIN_SHA" > "$STAMP" fail_if_shared "venv" fi -if ! command -v rtgs &> /dev/null; then - log "ERROR: rtgs command not found in virtual environment" - exit 1 -fi +# Device monitoring's own CLI, not the `rtgs` router: a missing dependency shows up as a +# traceback naming the module instead of "No such command". +DEVMON=("$VPY" -m rtgs_lab_tools.device_monitoring.cli) # Fail fast on setup problems (key file, IAM, URL) before the 80-second pipeline runs. # Exit 2 = setup is wrong; 3 = transient, so carry on and let the attempts retry. set +e -rtgs device-monitoring check-ingest >&3 2>&1 +"${DEVMON[@]}" check-ingest >&3 2>&1 CHECK=$? set -e if [ "$CHECK" -eq 2 ]; then @@ -342,7 +374,7 @@ else log "Ingest preflight ok" fi -MONITOR_ARGS=(device-monitoring monitor) +MONITOR_ARGS=(monitor) [ -z "$START_DATE" ] || MONITOR_ARGS+=("--start-date=$START_DATE") [ -z "$END_DATE" ] || MONITOR_ARGS+=("--end-date=$END_DATE") [ -z "$NODE_IDS" ] || MONITOR_ARGS+=("--node-ids=$NODE_IDS") @@ -363,9 +395,9 @@ fi EXIT_CODE=1 ATTEMPT=1 while [ "$ATTEMPT" -le "$MAX_ATTEMPTS" ]; do - log "Attempt $ATTEMPT/$MAX_ATTEMPTS - running: rtgs ${MONITOR_ARGS[*]}" + log "Attempt $ATTEMPT/$MAX_ATTEMPTS - running: device_monitoring.cli ${MONITOR_ARGS[*]}" set +e - "${TIMEOUT_PREFIX[@]}" rtgs "${MONITOR_ARGS[@]}" >&3 2>&1 + "${TIMEOUT_PREFIX[@]}" "${DEVMON[@]}" "${MONITOR_ARGS[@]}" >&3 2>&1 EXIT_CODE=$? set -e diff --git a/tests/device_monitoring/test_cron_script.py b/tests/device_monitoring/test_cron_script.py index d7e4a22e..1d3d09e1 100644 --- a/tests/device_monitoring/test_cron_script.py +++ b/tests/device_monitoring/test_cron_script.py @@ -94,13 +94,14 @@ def ws(tmp_path): return {"home": home, "base": base, "script": script, "sha": sha, "tmp": tmp_path} -def run(ws, release=None): +def run(ws, release=None, **extra_env): if release is not None: f = ws["home"] / ".config/devmon/release" f.write_text(release) os.chmod(f, 0o600) env = {"HOME": str(ws["home"]), "PATH": os.environ["PATH"], - "DEVICEMON_BASE_DIR": str(ws["base"]), "DEVICEMON_INGEST_TOKEN": "must-be-unset"} + "DEVICEMON_BASE_DIR": str(ws["base"]), "DEVICEMON_INGEST_TOKEN": "must-be-unset", + **extra_env} return subprocess.run(["bash", str(ws["script"])], capture_output=True, text=True, env=env, cwd=ws["tmp"], timeout=60) @@ -128,9 +129,87 @@ def test_pinned_release_checks_out_without_hooks(ws): r = run(ws, f"devmon-v0.0.1 {ws['sha']}\n") # gets past the pin and stops at the (fake) venv, having run no repository hook assert f"devmon-v0.0.1 ({ws['sha']})" in r.stdout - assert "Virtual environment not found after installation" in r.stdout + assert "requirements-cron.txt is not in release devmon-v0.0.1" in r.stdout assert not (ws["tmp"] / "hook-ran").exists() logs = list((ws["base"] / "logs").glob("device_monitoring_*.log")) assert len(logs) == 1 and len(logs[0].name.split("_")[-1]) == len("XXXXXX.log") assert (logs[0].stat().st_mode & 0o777) == 0o640 assert (ws["home"] / ".cache/devmon/device_monitoring.lock").exists() + + +# A stand-in interpreter: logs every call, answers the version probes, and makes a "venv" whose +# python is itself. Lets the tests follow the install and run commands without pip or a network. +STUB_PY = """#!/bin/bash +echo "$0 $*" >> "$STUB_LOG" +case "$1" in + -c) case "$2" in *print*) echo "${STUB_VERSION:-3.12.4}" ;; + *) [ "${STUB_VERSION:-3.12.4}" != 3.6.8 ] ;; esac ;; + -m) [ "$2" = venv ] && mkdir -p "$3/bin" && cp "$0" "$3/bin/python"; exit 0 ;; +esac +""" + + +def stub_python(ws): + py = ws["tmp"] / "stubpy" + py.write_text(STUB_PY) + os.chmod(py, 0o750) + return py + + +def release_with_requirements(ws): + """Tag devmon-v0.0.2: the fake release plus a requirements-cron.txt.""" + repo = ws["base"] / "rtgs-lab-tools-cron/rtgs-lab-tools" + env = {**os.environ, **GIT_ENV} + os.chmod(repo, 0o750) + reqs = repo / "src/rtgs_lab_tools/device_monitoring/requirements-cron.txt" + reqs.parent.mkdir(parents=True) + reqs.write_text("click==8.2.1\n") + subprocess.run(["git", "-C", str(repo), "add", "."], check=True, env=env) + subprocess.run(["git", "-C", str(repo), "commit", "-qm", "r2"], check=True, env=env) + subprocess.run(["git", "-C", str(repo), "tag", "devmon-v0.0.2"], check=True, env=env) + subprocess.run(["chmod", "-R", "g-w,o-w", str(repo)], check=True) + return subprocess.run(["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, + text=True, check=True).stdout.strip() + + +def test_refuses_python_older_than_310(ws): + sha = release_with_requirements(ws) + log = ws["tmp"] / "py.log" + r = run(ws, f"devmon-v0.0.2 {sha}\n", DEVICEMON_PYTHON=str(stub_python(ws)), + STUB_LOG=str(log), STUB_VERSION="3.6.8") + assert r.returncode == 1 and "is Python 3.6.8; need >= 3.10" in r.stdout + assert "-m venv" not in log.read_text() + assert not (ws["base"] / "rtgs-lab-tools-cron/rtgs-lab-tools/venv").exists() + + +def test_builds_lean_venv_and_runs_the_devmon_cli(ws): + sha = release_with_requirements(ws) + log = ws["tmp"] / "py.log" + r = run(ws, f"devmon-v0.0.2 {sha}\n", DEVICEMON_PYTHON=str(stub_python(ws)), + STUB_LOG=str(log)) + assert r.returncode == 0, r.stdout + r.stderr + calls = [c.split(" ", 1)[1] for c in log.read_text().splitlines()] + pip = [c for c in calls if c.startswith("-m pip")] + assert len(pip) == 2 + assert pip[0].endswith("-r src/rtgs_lab_tools/device_monitoring/requirements-cron.txt") + assert pip[1].endswith("--no-deps -e .") + assert "-m venv venv" in calls + runs = [c for c in calls if c.startswith("-m rtgs_lab_tools.device_monitoring.cli")] + assert runs[0].endswith("check-ingest") and runs[1].startswith( + "-m rtgs_lab_tools.device_monitoring.cli monitor") + venv = ws["base"] / "rtgs-lab-tools-cron/rtgs-lab-tools/venv" + bad = subprocess.run(["find", str(venv), "-perm", "/022"], capture_output=True, + text=True).stdout + assert bad == "" # umask 027: group may read, never write + assert (venv / ".devmon-installed-release").read_text().strip() == f"devmon-v0.0.2 {sha}" + # the next run of the same release installs nothing + log.write_text("") + r = run(ws, f"devmon-v0.0.2 {sha}\n", DEVICEMON_PYTHON=str(stub_python(ws)), + STUB_LOG=str(log)) + assert r.returncode == 0 and "-m pip" not in log.read_text() + + +def test_never_runs_install_sh_or_full_extras(): + text = SCRIPT.read_text() + assert "bash install.sh" not in text and "[dev]" not in text and "[all]" not in text + assert "pip install --quiet -e ." not in text # the full base dependencies