Update the sea ice sensitivity diagnostic for ESMValTool v2.15

.github/actions/build-container/action.yml

@@ -10,7 +10,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: docker/metadata-action@v6.0.0
+    - uses: docker/metadata-action@v6.2.0
       id: metadata
       with:
         images: ghcr.io/${{ github.repository_owner }}/${{ inputs.container-name }}
@@ -19,7 +19,7 @@ runs:
           type=ref,event=tag
           # set latest tag for default branch
           type=raw,value=main,enable={{is_default_branch}}
-    - uses: docker/build-push-action@v7.1.0
+    - uses: docker/build-push-action@v7.3.0
       id: push
       with:
         cache-from: type=gha
@@ -32,7 +32,7 @@ runs:
         labels: ${{ steps.metadata.outputs.labels }}
     - name: Attest to REF image
       if: ${{ ! github.event.pull_request.head.repo.fork }}
-      uses: actions/attest@v4.1.0
+      uses: actions/attest@v4.2.2
       with:
         subject-name: ghcr.io/${{ github.repository_owner }}/${{ inputs.container-name }}
         subject-digest: ${{ steps.push.outputs.digest }}

.github/workflows/deploy.yaml

@@ -33,7 +33,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/[email protected]
         with:
-          version: "0.11.6"
+          version: "0.12.7"
           python-version: "3.12"
       - name: Verify installable
         # TODO: this step fails we fix https://github.com/Climate-REF/climate-ref/issues/217

.github/workflows/regression-diff-report.yaml

@@ -0,0 +1,95 @@
+# Build and host the HTML diff of the regression baselines changed on a branch.
+#
+# Called by the mint workflow after it pushes regenerated manifests, and dispatched by hand
+# after a manual mint. Reads native blobs from the public store and writes only to the
+# reports bucket, so it needs the reports token and nothing else.
+name: Regression baselines (diff report)
+
+on:
+  workflow_call:
+    inputs:
+      base:
+        description: "Branch to diff against"
+        type: string
+        default: "main"
+  workflow_dispatch:
+    inputs:
+      base:
+        description: "Branch to diff against"
+        required: false
+        default: "main"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  report:
+    if: github.repository == 'Climate-REF/climate-ref'
+    runs-on: ubuntu-latest
+    environment: baseline-reports
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out the branch head
+        uses: actions/checkout@v7
+        with:
+          # A called workflow sees the dispatching commit, not the one the mint job pushed.
+          ref: ${{ github.ref_name }}
+          fetch-depth: 0
+      - name: Fetch the base branch
+        env:
+          BASE: ${{ inputs.base }}
+        run: git fetch --no-tags origin "+refs/heads/${BASE}:refs/remotes/origin/${BASE}"
+      - uses: ./.github/actions/setup
+        with:
+          python-version: "3.13"
+      - name: Find the pull request
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          pr="$(gh pr list --head "${GITHUB_REF_NAME}" --state open --json number --jq '.[0].number')"
+          sha="$(git rev-parse --short=12 HEAD)"
+          if [ -n "${pr}" ]; then
+            echo "prefix=${pr}/${sha}" >> "${GITHUB_OUTPUT}"
+          else
+            echo "No open pull request for ${GITHUB_REF_NAME}. Uploading under branch/ and skipping the comment."
+            echo "prefix=branch/${GITHUB_REF_NAME}/${sha}" >> "${GITHUB_OUTPUT}"
+          fi
+          echo "number=${pr}" >> "${GITHUB_OUTPUT}"
+      - name: Build and upload the report
+        env:
+          REF_REPORT_STORE_ACCESS_KEY_ID: ${{ secrets.R2_REPORTS_ACCESS_KEY_ID }}
+          REF_REPORT_STORE_SECRET_ACCESS_KEY: ${{ secrets.R2_REPORTS_SECRET_ACCESS_KEY }}
+          BASE: ${{ inputs.base }}
+          PREFIX: ${{ steps.pr.outputs.prefix }}
+        run: |
+          uv run ref test-cases diff \
+            --base "origin/${BASE}" \
+            --html-dir report \
+            --upload "${PREFIX}" \
+            --comment-output comment.md
+          cat comment.md >> "${GITHUB_STEP_SUMMARY}"
+      - name: Post or update the sticky comment
+        # One comment per pull request: the marker identifies the one to edit on a later run.
+        if: steps.pr.outputs.number != ''
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR: ${{ steps.pr.outputs.number }}
+        run: |
+          marker='<!-- climate-ref-baseline-diff -->'
+          existing="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate \
+            --jq "map(select(.body | contains(\"${marker}\"))) | .[0].id // empty")"
+          if [ -n "${existing}" ]; then
+            gh api --method PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${existing}" \
+              -F [email protected]
+          else
+            gh api --method POST "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" \
+              -F [email protected]
+          fi

.github/workflows/regression-mint.yaml

@@ -36,7 +36,7 @@ on:
         default: false
 
 # `contents` to commit the regenerated manifest/bundle back to the dispatched branch,
-# `pull-requests` to post the baseline diff on the branch's open pull request.
+# `pull-requests` so the called diff-report workflow can post on the branch's open pull request.
 permissions:
   contents: write
   pull-requests: write
@@ -70,7 +70,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v7
         with:
-          # Full history so the baseline diff can find the merge-base with the default branch.
+          # Full history so the mint can resolve the committed baselines against the default branch.
           fetch-depth: 0
       - uses: ./.github/actions/setup
         with:
@@ -114,13 +114,6 @@ jobs:
           if [ "${BUMP_VERSION}" = "true" ]; then args+=(--bump-version); fi
           if [ "${DRY_RUN}" = "true" ]; then args+=(--dry-run); fi
           uv run ref test-cases mint "${args[@]}"
-      - name: Fetch the default branch for the baseline diff
-        if: ${{ !inputs.dry_run }}
-        env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-        run: |
-          git fetch --no-tags origin \
-            "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}"
       - name: Commit regenerated baselines
         if: ${{ !inputs.dry_run }}
         env:
@@ -160,37 +153,10 @@ jobs:
           done
           echo "::error::failed to push regenerated baselines after retries"
           exit 1
-      - name: Report the baseline diff
-        # Runs after the commit so HEAD carries the regenerated manifests.
-        # Reporting only, so a failure here must not fail an otherwise successful mint.
-        if: ${{ !inputs.dry_run }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-        run: |
-          # Two reports: the full one is small enough to keep as an artefact whatever the mint's
-          # size, while the capped one fits the comment and job-summary limits.
-          uv run python scripts/ci/mint_diff.py \
-            --base "origin/${DEFAULT_BRANCH}" \
-            --output mint-diff.md \
-            --comment-output mint-diff-comment.md
 
-          # The job summary always gets a report, whether or not a pull request exists.
-          cat mint-diff-comment.md >> "${GITHUB_STEP_SUMMARY}"
-
-          pr="$(gh pr list --head "${GITHUB_REF_NAME}" --state open --json number --jq '.[0].number')"
-          if [ -z "${pr}" ]; then
-            echo "No open pull request for ${GITHUB_REF_NAME}; the report is in the job summary only."
-            exit 0
-          fi
-          gh pr comment "${pr}" --body-file mint-diff-comment.md
-      - name: Upload the full baseline diff
-        # The comment is capped at GitHub's 65 KB limit, so keep the untruncated report too.
-        if: ${{ !inputs.dry_run }}
-        continue-on-error: true
-        uses: actions/upload-artifact@v7
-        with:
-          name: mint-diff
-          path: mint-diff.md
-          if-no-files-found: ignore
+  report:
+    needs: mint
+    if: ${{ !inputs.dry_run }}
+    uses: ./.github/workflows/regression-diff-report.yaml
+    with:
+      base: ${{ github.event.repository.default_branch }}

Makefile

@@ -5,6 +5,9 @@
 # compatible so we're not too worried
 TEMP_FILE := $(shell mktemp)
 
+# Workers used by the parallel test run.
+PYTEST_WORKERS ?= 4
+
 # A helper script to get short descriptions of each target in the Makefile
 define PRINT_HELP_PYSCRIPT
 import re, sys
@@ -164,7 +167,7 @@ test-quick: clean  ## run all the tests at once
 	# A new resource_intensive test has to live under a path listed here.
 	uv run \
 		pytest tests packages \
-		-r a -v  --cov-report=term -n auto \
+		-r a -v  --cov-report=term -n $(PYTEST_WORKERS) \
 		-m "not resource_intensive"
 	uv run \
 		pytest packages/climate-ref-core/tests/integration packages/climate-ref/tests/integration \

changelog/877.trivial.md

@@ -0,0 +1,2 @@
+Updated the development type checker to mypy 2.
+The declared requirement now matches the lockfile, so a fresh resolve cannot pick a mypy 1 release that disagrees with CI.

changelog/897.fix.md

@@ -0,0 +1,2 @@
+Dataset discovery now walks the whole directory tree instead of stopping 10 directories below the root.
+The CMIP7 DRS nests files deeper than that, so ingesting a `MIP-DRS7` root found no files at all.

changelog/898.feature.md

@@ -0,0 +1,17 @@
+Ingests the obs4REF collection under its own `obs4ref` source type.
+The `instance_id` for `obs4ref` datasets now starts with `obs4REF`.
+The diagnostics keep asking for obs4MIPs data.
+Where a dataset is ingested from both, the newest version is used, and the obs4MIPs copy wins a tie.
+Publishing a dataset to obs4MIPs therefore takes over from the registry copy without any change to the REF.
+A newer registry version is still preferred over a stale published one, so nothing reruns for no reason.
+
+`ref doctor` gains three checks: `misfiled-obs4ref`, `superseded-obs4ref` and `unsolvable-diagnostics`.
+Data ingested with `--source-type obs4mips` in earlier releases still solves,
+but re-ingesting it with `--source-type obs4ref` is recommended.
+
+A data requirement can now declare the collections that may supply it,
+through the new `fallback_source_types` field on `DataRequirement`.
+A dataset present under `source_type` still wins,
+and the data is delivered under `source_type` whichever collection it came from.
+The solver, `ref doctor` and the generated reference data documentation all read the declaration.
+No provider declares a fallback yet, so nothing changes behaviourally.

changelog/899.fix.md

@@ -0,0 +1 @@
+Updated the sea ice sensitivity diagnostic for the recipe changes in ESMValTool v2.15.

changelog/908.fix.md

@@ -0,0 +1,3 @@
+Removed the `immutable` flag from read-only SQLite connections.
+The flag told SQLite to skip locking, which did not hold when a background run was writing
+to the same database. The connection is still opened read-only.

changelog/910.feature.md

@@ -0,0 +1,6 @@
+`ref test-cases diff` renders the regression baselines changed on a branch as a local HTML report.
+Images appear old and new side by side, with a button that overlays them.
+Text outputs get a coloured diff,
+and NetCDF and other binaries are listed with a size delta and a link to each blob.
+`--html-dir` chooses where the report is written,
+and `--no-fetch` skips blob downloads and reports sizes only.

changelog/911.fix.md

@@ -0,0 +1,2 @@
+PMP executions are now capped at two threads each.
+Without a limit the scientific stack sized its thread pools from the machine's core count.

changelog/912.feature.md

@@ -0,0 +1,4 @@
+`ref test-cases diff --upload <prefix>` publishes the report to a public object store,
+and `--comment-output <path>` writes the pull request comment markdown that links into it.
+
+Reports are stored in their own bucket, separate from the baselines.

changelog/913.feature.md

@@ -0,0 +1,4 @@
+Summarise the changes to `.nc` files in the `ref test-cases diff` report.
+Each file gets a collapsible listing of its ncdump-style header and a table with one row per
+data variable carrying min, max, mean, NaN count, the largest absolute and relative difference,
+and the number of cells that differ.

changelog/915.feature.md

@@ -0,0 +1 @@
+Regression baseline diffs are now published as an HTML report linked from a short pull request comment, with images shown side by side and NetCDF outputs summarised per variable.

docs/api-surface.md

@@ -34,6 +34,7 @@ The primary module providers interact with.
 - Override `Diagnostic.build_execution_result(definition) -> ExecutionResult`
 - Set `Diagnostic.data_requirements`, `facets`, `slug`, `name`
 - Set `Diagnostic.test_data_spec` for test case support
+- Set `DataRequirement.fallback_source_types` to name the collections that may supply a requirement
 
 ---
 

docs/background/architecture.md

@@ -365,7 +365,7 @@ Key characteristics:
 
 - The backend is a **read-only consumer** of the REF database.
   It opens the database without running migrations
-  (SQLite can be opened immutable so the state volume can be mounted read-only)
+  (SQLite is opened with `mode=ro`, so the state volume can be mounted read-only)
   and reports the results the compute engine has already produced.
 - It depends on the `climate-ref` library directly for configuration, models and the provider registry.
   The database schema remains owned by `climate-ref` migrations.

docs/background/regression-baselines.md

@@ -250,14 +250,15 @@ the `--json` output drives CI's dispatch of the `replay` jobs.
 
 ## Continuous integration
 
-The lifecycle commands are wired into three GitHub Action workflows.
+The lifecycle commands are wired into four GitHub Action workflows.
 The minting process requires credentials to upload data.
 Since this is a public project we have to be careful about when this is run to not leak these credentials.
 
 | Workflow | Trigger | Credentials | What it does |
 | --- | --- | --- | --- |
 | `regression-pr-gate.yaml` | every pull request | none | Runs the coupling gate, then `replay`s every case it routes to `replay`. |
 | `regression-mint.yaml` | manual dispatch | R2 write | `mint`s native baselines and commits the regenerated manifest back to the branch. |
+| `regression-diff-report.yaml` | called by the mint + manual | R2 write (reports bucket) | Builds the HTML diff of the changed baselines and links it from a sticky pull-request comment. |
 | `regression-drift.yaml` | nightly + manual | none | `replay`s every baseline to catch silent drift. |
 
 ### PR gate (`regression-pr-gate.yaml`)
@@ -301,6 +302,7 @@ Dispatch it on the feature branch that should receive the new baseline:
 the job runs `mint`, and commits the regenerated `manifest.json` (and committed bundle) back to that branch,
 so the change is reviewed through its pull request and no developer ever needs write credentials.
 A `dry_run` input previews without uploading or committing, and the job refuses to run on the default branch.
+Once the commit is pushed, the mint calls `regression-diff-report.yaml` to publish the diff of what it changed.
 
 !!! warning "The mint commit does not re-trigger the PR gate"
     The mint job pushes with the default `GITHUB_TOKEN`, and GitHub deliberately does not start new workflow runs for such pushes.
@@ -317,6 +319,47 @@ A `dry_run` input previews without uploading or committing, and the job refuses
     The endpoint and bucket default to the production R2 account
     (`REF_NATIVE_STORE_S3_ENDPOINT_URL` / `REF_NATIVE_STORE_BUCKET` override them).
 
+### Diff report (`regression-diff-report.yaml`)
+
+Reviewing a minted baseline means looking at what actually moved, which a JSON manifest does not show.
+So this workflow builds an HTML report of every test case whose baseline changed against the base branch:
+
+- Images are shown two-up, the old beside the new.
+- Text outputs get a coloured line diff.
+- A changed NetCDF gets a header diff plus one row of statistics per variable, shaded only where the values moved.
+
+The report is uploaded to the public reports bucket and served at
+`https://reports.baselines.climate-ref.org/<pr>/<sha>/index.html`.
+The workflow then posts a short comment on the branch's open pull request linking to it.
+The comment carries a hidden marker, so a later run edits that same comment rather than stacking a new one.
+A run on a branch with no open pull request uploads under `branch/<name>/<sha>/` and writes the summary to the job log only.
+
+The mint workflow calls this automatically.
+After minting on a workstation and pushing the result by hand, dispatch it yourself:
+
+```bash
+gh workflow run regression-diff-report.yaml --ref <branch>
+```
+
+The same report can be built locally, without any credentials:
+
+```bash
+uv run ref test-cases diff --base origin/main --html-dir out/
+```
+
+!!! note "Required repository configuration"
+    Create a `baseline-reports` Environment (Settings -> Environments) with **no required reviewers**,
+    so a mint run does not stop for a second approval, and add two secrets to it
+    holding an R2 token scoped to the `ref-baselines-reports` bucket:
+
+    - `R2_REPORTS_ACCESS_KEY_ID` -> `REF_REPORT_STORE_ACCESS_KEY_ID`
+    - `R2_REPORTS_SECRET_ACCESS_KEY` -> `REF_REPORT_STORE_SECRET_ACCESS_KEY`
+
+    The reports live in their own bucket because an R2 token cannot be scoped write-only or to a prefix.
+    A token that could write reports into the baselines bucket could also overwrite a baseline.
+    The endpoint, bucket and public URL default to the production R2 account
+    (`REF_REPORT_STORE_S3_ENDPOINT_URL` / `REF_REPORT_STORE_BUCKET` / `REF_REPORT_STORE_URL` override them).
+
 ### Nightly drift (`regression-drift.yaml`)
 
 A scheduled (and manually dispatchable) job `replay`s every referenced native blob against the committed bundle.

docs/development.md

@@ -62,7 +62,7 @@ For development, we use a consistent set of decimated sample data.
 ```bash
 make fetch-test-data
 uv run ref datasets ingest --source-type cmip6 $PWD/tests/test-data/sample-data/CMIP6/
-uv run ref datasets ingest --source-type obs4mips $PWD/tests/test-data/sample-data/obs4REF/
+uv run ref datasets ingest --source-type obs4ref $PWD/tests/test-data/sample-data/obs4REF/
 ```
 
 This is enough to run a `solve` against.