devmon: lean cron install (11 pinned packages, Python >= 3.10 check)

.github/workflows/devmon-publisher.yml

@@ -29,16 +29,22 @@ jobs:
       - name: Cron script parses
         run: bash -n src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh
 
-      - name: Install
+      # Exactly what the MSI cron installs: if the publisher path gains an import that is not in
+      # requirements-cron.txt, this job fails instead of the 04:27 run.
+      - name: Install (lean, as the cron does)
         run: |
           python -m pip install --upgrade pip
-          pip install -e . pytest pytest-cov jsonschema
+          pip install -r src/rtgs_lab_tools/device_monitoring/requirements-cron.txt
+          pip install --no-deps -e .
+          pip install pytest jsonschema
 
       - name: Publisher tests
-        run: pytest -q --no-cov tests/device_monitoring
+        run: pytest -q -o addopts= tests/device_monitoring
 
       - name: CLI takes the flags the cron passes
         run: |
           rtgs device-monitoring monitor --help | grep -q -- '--start-date'
           rtgs device-monitoring publish --help >/dev/null
           rtgs device-monitoring check-ingest --help >/dev/null
+          python -m rtgs_lab_tools.device_monitoring.cli monitor --help | grep -q -- '--no-email'
+          python -m rtgs_lab_tools.device_monitoring.cli check-ingest --help >/dev/null

src/rtgs_lab_tools/__init__.py

@@ -4,34 +4,44 @@
 __author__ = "RTGS Lab"
 __email__ = "[email protected]"
 
-# Core infrastructure
-from .core import Config, DatabaseManager
-from .core.exceptions import APIError, DatabaseError, ValidationError
-from .core.postgres_logger import PostgresLogger
-from .data_parser.parsers.data_parser import DataV2Parser
+# Public names resolve on first use (PEP 562), not at import. Importing any subpackage imports
+# this file first, so eager imports here made every tool -- including the MSI device-monitoring
+# cron -- load matplotlib, the Particle client and the parsers whether it used them or not.
+_LAZY = {
+    "Config": ".core",
+    "DatabaseManager": ".core",
+    "APIError": ".core.exceptions",
+    "DatabaseError": ".core.exceptions",
+    "ValidationError": ".core.exceptions",
+    "PostgresLogger": ".core.postgres_logger",
+    "DataV2Parser": ".data_parser.parsers.data_parser",
+    "ParserFactory": ".data_parser.parsers.factory",
+    "ParticleClient": ".device_configuration",
+    "ParticleConfigUpdater": ".device_configuration",
+    "extract_data": ".sensing_data",
+    "get_raw_data": ".sensing_data",
+    "list_available_projects": ".sensing_data",
+    "create_multi_parameter_plot": ".visualization",
+    "create_time_series_plot": ".visualization",
+    "detect_data_type": ".visualization",
+    "load_and_prepare_data": ".visualization",
+    # Climate data: download_GEE_raster stays out (slow import), as before.
+}
 
-# Data parsing functions
-from .data_parser.parsers.factory import ParserFactory
 
-# Device management
-from .device_configuration import ParticleClient, ParticleConfigUpdater
+def __getattr__(name):
+    module = _LAZY.get(name)
+    if module is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    import importlib
 
-# High-level data extraction functions
-from .sensing_data import extract_data, get_raw_data, list_available_projects
+    value = getattr(importlib.import_module(module, __name__), name)
+    globals()[name] = value  # cache: later lookups skip __getattr__
+    return value
 
-# Visualization functions
-from .visualization import (
-    create_multi_parameter_plot,
-    create_time_series_plot,
-    detect_data_type,
-    load_and_prepare_data,
-)
 
-# Climate data
-# from .gridded_data import download_GEE_raster  # Commented out to avoid slow imports
-
-
-# Error analysis
+def __dir__():
+    return sorted(set(globals()) | set(_LAZY))
 
 
 __all__ = [

src/rtgs_lab_tools/device_monitoring/README.md

@@ -49,7 +49,8 @@ The monitoring script runs from the lab's shared workspace on Bryan's scrontab:
 ```
 Paths are overridable from the scrontab environment (never from the credentials file):
 DEVICEMON_BASE_DIR, DEVICEMON_WORK_DIR, DEVICEMON_LOG_DIR, DEVICEMON_CREDS_FILE,
-DEVICEMON_RELEASE_FILE, DEVICEMON_SA_KEY, DEVICEMON_TOKEN_SECRET.
+DEVICEMON_RELEASE_FILE, DEVICEMON_SA_KEY, DEVICEMON_TOKEN_SECRET, DEVICEMON_PYTHON_MODULE
+(default `python3/3.12.4_anaconda2024.06-1_libmamba`), DEVICEMON_PYTHON (absolute path).
 
 Ownership (the script refuses to run if the code side is group- or world-writable):
 ```
@@ -61,12 +62,17 @@ credentials/.rtgs_credentials                      660, group runck014 -- data,
 ```
 
 Each run:
-- loads MSI modules (Python, Git)
+- loads MSI modules (Python, Git) and picks a Python >= 3.10 by absolute path; an older one
+  (agate's `/usr/bin/python3` is 3.6.8) stops the run with a message, before any install
+- builds `venv/` lean on first run and once per release: `pip install -r
+  device_monitoring/requirements-cron.txt` then `pip install --no-deps -e .` -- never
+  `install.sh` or the package's full base dependencies (minutes, not 25+)
 - parses the credentials file against an allowlist (no `source`)
 - checks out the release pinned in `~/.config/devmon/release` (`<tag> <commit sha>`; both
   checked; git runs with hooks disabled; an existing tag is never force-moved)
 - fetches the ingest token from Secret Manager with `~/.config/devmon/sa.json`
-  (`rtgs device-monitoring check-ingest` preflight), runs the monitor, publishes the batch
+  (`check-ingest` preflight), runs the monitor, publishes the batch -- all through
+  `venv/bin/python -m rtgs_lab_tools.device_monitoring.cli`, not the `rtgs` router
 - logs to `logs/device_monitoring_<stamp>_<random>.log`; the lock is in `~/.cache/devmon/`
 
 ### MSI Credentials Configuration

src/rtgs_lab_tools/device_monitoring/requirements-cron.txt

@@ -0,0 +1,20 @@
+# What the MSI device-monitoring cron needs, and nothing else. scheduled_device_monitoring.sh
+# builds its venv with:  python -m pip install -r requirements-cron.txt
+#                        python -m pip install --no-deps -e .
+# The package's base dependencies (earthengine, geopandas, streamlit, pyarrow, matplotlib, ...)
+# serve the other tools and are deliberately not installed for the cron.
+# Pins follow uv.lock. Verified on Python 3.12 against tests/device_monitoring and a run of
+# check-ingest + monitor with the network and database faked. Add a line here when the
+# publisher path gains an import; CI installs exactly this file (devmon-publisher.yml).
+# (requirements.txt beside this file is the web app's, not the cron's.)
+click==8.2.1                # CLI
+requests==2.32.4            # ingest POST, Particle lookups
+google-auth==2.40.3         # service-account key -> Secret Manager (ingest token)
+sqlalchemy==2.0.41          # GEMS raw-data query (<2.1: keep postgresql:// on psycopg2)
+psycopg2-binary==2.9.10     # driver for the postgresql:// GEMS URL
+pandas==2.3.1               # parse / analyze (brings numpy)
+python-dotenv==1.1.1        # core.config, message_builder
+yagmail==0.15.293           # notification e-mail
+chardet<6                   # yagmail->premailer->cssutils pulls chardet 7, which requests warns on
+flask==3.1.3                # device_monitoring.core imports web_app.models (record builders)
+flask-sqlalchemy==3.1.1     # same

src/rtgs_lab_tools/device_monitoring/scheduled_device_monitoring.sh

@@ -26,6 +26,11 @@
 #   DEVICEMON_SA_KEY           service-account key (default ~/.config/devmon/sa.json)
 #   DEVICEMON_TOKEN_SECRET     secret name (default follows the URL: prod or -test)
 #   DEVICEMON_RELEASE_FILE     release pin (default ~/.config/devmon/release)
+#   DEVICEMON_PYTHON_MODULE    module that provides Python >= 3.10 (default
+#                              python3/3.12.4_anaconda2024.06-1_libmamba; agate's bare
+#                              `module load python` left /usr/bin/python3 = 3.6.8 on PATH)
+#   DEVICEMON_PYTHON           interpreter to build the venv with (absolute path); skips the
+#                              module lookup
 #
 # Paths (defaults = the lab's shared MSI workspace):
 #   DEVICEMON_BASE_DIR    /projects/standard/runck014/shared/device_monitoring
@@ -206,13 +211,41 @@ else
 fi
 
 log "Loading required modules"
+PYTHON_MODULE="${DEVICEMON_PYTHON_MODULE:-python3/3.12.4_anaconda2024.06-1_libmamba}"
 if command -v module &> /dev/null; then
-    module load python || log "Warning: Could not load python module"
+    if [ -z "${DEVICEMON_PYTHON:-}" ]; then
+        module load "$PYTHON_MODULE" || log "Warning: Could not load module $PYTHON_MODULE"
+    fi
     module load git || log "Warning: Could not load git module"
 else
     log "Module command not found, assuming modules are already available"
 fi
 
+# The interpreter the venv is built from, by absolute path. A module may put python3, or only
+# python, on PATH; take the first that is new enough. Anything older fails here, not 20 minutes
+# into a pip run that builds sdists for an interpreter the code cannot run on.
+py_ok() { "$1" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null; }
+py_ver() { "$1" -c 'import sys; print(sys.version.split()[0])' 2>/dev/null || echo "?"; }
+PY=""
+if [ -n "${DEVICEMON_PYTHON:-}" ]; then
+    PY="$DEVICEMON_PYTHON"
+    py_ok "$PY" || { log "ERROR: DEVICEMON_PYTHON=$PY is Python $(py_ver "$PY"); need >= 3.10"
+                     exit 1; }
+else
+    for cand in python3 python; do
+        c="$(command -v "$cand" 2>/dev/null)" || continue
+        if py_ok "$c"; then PY="$c"; break; fi
+        log "Skipping $c: Python $(py_ver "$c") (need >= 3.10)"
+    done
+    if [ -z "$PY" ]; then
+        log "ERROR: no Python >= 3.10 on PATH after 'module load $PYTHON_MODULE'."
+        log "       Check 'module avail python', then set DEVICEMON_PYTHON_MODULE (or"
+        log "       DEVICEMON_PYTHON=/abs/path/python3) in the scrontab line."
+        exit 1
+    fi
+fi
+log "Python for the venv: $PY ($(py_ver "$PY"))"
+
 # Credentials: the shared file (DEVICEMON_CREDS_FILE) as data; per-user files are a fallback
 # for running from a personal account.
 CREDS_LOADED=0
@@ -289,48 +322,47 @@ if [ -z "${DEVICEMON_REEXEC:-}" ] && [ -f "$NEW_SELF" ] \
     exec env DEVICEMON_REEXEC=1 bash "$NEW_SELF" "$@"
 fi
 
-if [ ! -d "venv" ]; then
-    log "Virtual environment not found, running installation"
-    bash install.sh >&3 2>&1
+# The cron's venv holds only what the publisher needs (requirements-cron.txt), never the
+# package's full base dependencies, install.sh, or the dev/all extras.
+REQS="src/rtgs_lab_tools/device_monitoring/requirements-cron.txt"
+VPY="$REPO_DIR/venv/bin/python"
+if [ ! -f "$REQS" ]; then
+    log "ERROR: $REQS is not in release $PIN_TAG; cannot build the venv"
+    exit 1
 fi
 if [ ! -d "venv" ]; then
-    log "ERROR: Virtual environment not found after installation"
+    log "Virtual environment not found, creating it with $PY"
+    "$PY" -m venv venv >&3 2>&1
+fi
+if [ ! -x "$VPY" ]; then
+    log "ERROR: Virtual environment not found after installation ($VPY missing)"
     exit 1
 fi
-fail_if_shared "venv"
-
-log "Activating virtual environment"
-if [ -f "venv/bin/activate" ]; then
-    source venv/bin/activate
-elif [ -f "venv/Scripts/activate" ]; then
-    source venv/Scripts/activate
-else
-    log "ERROR: No activate script found in venv/bin or venv/Scripts"
+if ! py_ok "$VPY"; then
+    log "ERROR: venv/ runs Python $(py_ver "$VPY"); need >= 3.10. Move $REPO_DIR/venv aside"
+    log "       and re-run: the venv is rebuilt with $PY."
     exit 1
 fi
+fail_if_shared "venv"
 
-# A new release may bring new dependencies: refresh the editable install once per commit.
+# A new release may bring new dependencies: refresh once per commit, still lean.
 STAMP="venv/.devmon-installed-release"
 if [ "$(cat "$STAMP" 2>/dev/null)" != "$PIN_TAG $PIN_SHA" ]; then
-    log "Installing rtgs-lab-tools for release $PIN_TAG"
-    if command -v uv &> /dev/null; then
-        uv pip install --quiet -e . >&3 2>&1
-    else
-        pip install --quiet -e . >&3 2>&1
-    fi
+    log "Installing the device-monitoring requirements for release $PIN_TAG"
+    "$VPY" -m pip install --quiet --disable-pip-version-check -r "$REQS" >&3 2>&1
+    "$VPY" -m pip install --quiet --disable-pip-version-check --no-deps -e . >&3 2>&1
     echo "$PIN_TAG $PIN_SHA" > "$STAMP"
     fail_if_shared "venv"
 fi
 
-if ! command -v rtgs &> /dev/null; then
-    log "ERROR: rtgs command not found in virtual environment"
-    exit 1
-fi
+# Device monitoring's own CLI, not the `rtgs` router: a missing dependency shows up as a
+# traceback naming the module instead of "No such command".
+DEVMON=("$VPY" -m rtgs_lab_tools.device_monitoring.cli)
 
 # Fail fast on setup problems (key file, IAM, URL) before the 80-second pipeline runs.
 # Exit 2 = setup is wrong; 3 = transient, so carry on and let the attempts retry.
 set +e
-rtgs device-monitoring check-ingest >&3 2>&1
+"${DEVMON[@]}" check-ingest >&3 2>&1
 CHECK=$?
 set -e
 if [ "$CHECK" -eq 2 ]; then
@@ -342,7 +374,7 @@ else
     log "Ingest preflight ok"
 fi
 
-MONITOR_ARGS=(device-monitoring monitor)
+MONITOR_ARGS=(monitor)
 [ -z "$START_DATE" ] || MONITOR_ARGS+=("--start-date=$START_DATE")
 [ -z "$END_DATE" ] || MONITOR_ARGS+=("--end-date=$END_DATE")
 [ -z "$NODE_IDS" ] || MONITOR_ARGS+=("--node-ids=$NODE_IDS")
@@ -363,9 +395,9 @@ fi
 EXIT_CODE=1
 ATTEMPT=1
 while [ "$ATTEMPT" -le "$MAX_ATTEMPTS" ]; do
-    log "Attempt $ATTEMPT/$MAX_ATTEMPTS - running: rtgs ${MONITOR_ARGS[*]}"
+    log "Attempt $ATTEMPT/$MAX_ATTEMPTS - running: device_monitoring.cli ${MONITOR_ARGS[*]}"
     set +e
-    "${TIMEOUT_PREFIX[@]}" rtgs "${MONITOR_ARGS[@]}" >&3 2>&1
+    "${TIMEOUT_PREFIX[@]}" "${DEVMON[@]}" "${MONITOR_ARGS[@]}" >&3 2>&1
     EXIT_CODE=$?
     set -e
 

tests/device_monitoring/test_cron_script.py

@@ -94,13 +94,14 @@ def ws(tmp_path):
     return {"home": home, "base": base, "script": script, "sha": sha, "tmp": tmp_path}
 
 
-def run(ws, release=None):
+def run(ws, release=None, **extra_env):
     if release is not None:
         f = ws["home"] / ".config/devmon/release"
         f.write_text(release)
         os.chmod(f, 0o600)
     env = {"HOME": str(ws["home"]), "PATH": os.environ["PATH"],
-           "DEVICEMON_BASE_DIR": str(ws["base"]), "DEVICEMON_INGEST_TOKEN": "must-be-unset"}
+           "DEVICEMON_BASE_DIR": str(ws["base"]), "DEVICEMON_INGEST_TOKEN": "must-be-unset",
+           **extra_env}
     return subprocess.run(["bash", str(ws["script"])], capture_output=True, text=True, env=env,
                           cwd=ws["tmp"], timeout=60)
 
@@ -128,9 +129,87 @@ def test_pinned_release_checks_out_without_hooks(ws):
     r = run(ws, f"devmon-v0.0.1 {ws['sha']}\n")
     # gets past the pin and stops at the (fake) venv, having run no repository hook
     assert f"devmon-v0.0.1 ({ws['sha']})" in r.stdout
-    assert "Virtual environment not found after installation" in r.stdout
+    assert "requirements-cron.txt is not in release devmon-v0.0.1" in r.stdout
     assert not (ws["tmp"] / "hook-ran").exists()
     logs = list((ws["base"] / "logs").glob("device_monitoring_*.log"))
     assert len(logs) == 1 and len(logs[0].name.split("_")[-1]) == len("XXXXXX.log")
     assert (logs[0].stat().st_mode & 0o777) == 0o640
     assert (ws["home"] / ".cache/devmon/device_monitoring.lock").exists()
+
+
+# A stand-in interpreter: logs every call, answers the version probes, and makes a "venv" whose
+# python is itself. Lets the tests follow the install and run commands without pip or a network.
+STUB_PY = """#!/bin/bash
+echo "$0 $*" >> "$STUB_LOG"
+case "$1" in
+  -c) case "$2" in *print*) echo "${STUB_VERSION:-3.12.4}" ;;
+                   *) [ "${STUB_VERSION:-3.12.4}" != 3.6.8 ] ;; esac ;;
+  -m) [ "$2" = venv ] && mkdir -p "$3/bin" && cp "$0" "$3/bin/python"; exit 0 ;;
+esac
+"""
+
+
+def stub_python(ws):
+    py = ws["tmp"] / "stubpy"
+    py.write_text(STUB_PY)
+    os.chmod(py, 0o750)
+    return py
+
+
+def release_with_requirements(ws):
+    """Tag devmon-v0.0.2: the fake release plus a requirements-cron.txt."""
+    repo = ws["base"] / "rtgs-lab-tools-cron/rtgs-lab-tools"
+    env = {**os.environ, **GIT_ENV}
+    os.chmod(repo, 0o750)
+    reqs = repo / "src/rtgs_lab_tools/device_monitoring/requirements-cron.txt"
+    reqs.parent.mkdir(parents=True)
+    reqs.write_text("click==8.2.1\n")
+    subprocess.run(["git", "-C", str(repo), "add", "."], check=True, env=env)
+    subprocess.run(["git", "-C", str(repo), "commit", "-qm", "r2"], check=True, env=env)
+    subprocess.run(["git", "-C", str(repo), "tag", "devmon-v0.0.2"], check=True, env=env)
+    subprocess.run(["chmod", "-R", "g-w,o-w", str(repo)], check=True)
+    return subprocess.run(["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True,
+                          text=True, check=True).stdout.strip()
+
+
+def test_refuses_python_older_than_310(ws):
+    sha = release_with_requirements(ws)
+    log = ws["tmp"] / "py.log"
+    r = run(ws, f"devmon-v0.0.2 {sha}\n", DEVICEMON_PYTHON=str(stub_python(ws)),
+            STUB_LOG=str(log), STUB_VERSION="3.6.8")
+    assert r.returncode == 1 and "is Python 3.6.8; need >= 3.10" in r.stdout
+    assert "-m venv" not in log.read_text()
+    assert not (ws["base"] / "rtgs-lab-tools-cron/rtgs-lab-tools/venv").exists()
+
+
+def test_builds_lean_venv_and_runs_the_devmon_cli(ws):
+    sha = release_with_requirements(ws)
+    log = ws["tmp"] / "py.log"
+    r = run(ws, f"devmon-v0.0.2 {sha}\n", DEVICEMON_PYTHON=str(stub_python(ws)),
+            STUB_LOG=str(log))
+    assert r.returncode == 0, r.stdout + r.stderr
+    calls = [c.split(" ", 1)[1] for c in log.read_text().splitlines()]
+    pip = [c for c in calls if c.startswith("-m pip")]
+    assert len(pip) == 2
+    assert pip[0].endswith("-r src/rtgs_lab_tools/device_monitoring/requirements-cron.txt")
+    assert pip[1].endswith("--no-deps -e .")
+    assert "-m venv venv" in calls
+    runs = [c for c in calls if c.startswith("-m rtgs_lab_tools.device_monitoring.cli")]
+    assert runs[0].endswith("check-ingest") and runs[1].startswith(
+        "-m rtgs_lab_tools.device_monitoring.cli monitor")
+    venv = ws["base"] / "rtgs-lab-tools-cron/rtgs-lab-tools/venv"
+    bad = subprocess.run(["find", str(venv), "-perm", "/022"], capture_output=True,
+                         text=True).stdout
+    assert bad == ""                                  # umask 027: group may read, never write
+    assert (venv / ".devmon-installed-release").read_text().strip() == f"devmon-v0.0.2 {sha}"
+    # the next run of the same release installs nothing
+    log.write_text("")
+    r = run(ws, f"devmon-v0.0.2 {sha}\n", DEVICEMON_PYTHON=str(stub_python(ws)),
+            STUB_LOG=str(log))
+    assert r.returncode == 0 and "-m pip" not in log.read_text()
+
+
+def test_never_runs_install_sh_or_full_extras():
+    text = SCRIPT.read_text()
+    assert "bash install.sh" not in text and "[dev]" not in text and "[all]" not in text
+    assert "pip install --quiet -e ." not in text     # the full base dependencies