Feature/dashboard update 2026 07 18

.Rprofile

@@ -0,0 +1,30 @@
+# CLMS Technical Library — local PDF preview font setup.
+#
+# Makes the bundled Typst fonts (_meta/theme/typst-fonts/) discoverable so that
+# rendering a DOCS/*.qmd to PDF — e.g. via the RStudio "Render" button — uses the
+# CLMS fonts (Lato, Liberation Sans, JetBrains Mono) without installing anything
+# system-wide. Quarto's typst format has no font-path option, so we set Typst's
+# native TYPST_FONT_PATHS env var here; RStudio runs this file on session start,
+# and the quarto render it launches inherits the variable.
+#
+# Preview only — the CI build (build-docs.sh) does not use this.
+local({
+  font_dir <- normalizePath(
+    file.path(getwd(), "_meta", "theme", "typst-fonts"),
+    mustWork = FALSE
+  )
+  if (dir.exists(font_dir)) {
+    existing <- Sys.getenv("TYPST_FONT_PATHS")
+    paths <- if (nzchar(existing)) {
+      strsplit(existing, .Platform$path.sep, fixed = TRUE)[[1]]
+    } else {
+      character(0)
+    }
+    if (!(font_dir %in% paths)) {
+      Sys.setenv(
+        TYPST_FONT_PATHS = paste(c(font_dir, paths), collapse = .Platform$path.sep)
+      )
+      message("CLMS: TYPST_FONT_PATHS set to bundled fonts for local PDF preview.")
+    }
+  }
+})

.github/non_browsable_doc_map.json

@@ -15,6 +15,16 @@
       "source": "Copernicus_Land_Data_Store_CLDS/Survey_Evaluation_v1.qmd",
       "base": "dr3yejtzxollhygq0lwfgjxf755jvzyly2lemmvpgiqrzf5own2074qpdivug54c",
       "url": "/dr3yejtzxollhygq0lwfgjxf755jvzyly2lemmvpgiqrzf5own2074qpdivug54c.html"
+    },
+    {
+      "source": "CDSE_Migration/CLMS_CDSE_Migration_Status.qmd",
+      "base": "54ab2b07e82f82143780f24e2704ef211394c00216e83839e7c1dc42ba8e3a91",
+      "url": "/54ab2b07e82f82143780f24e2704ef211394c00216e83839e7c1dc42ba8e3a91.html"
+    },
+    {
+      "source": "CDSE_Migration/CLMS_CDSE_Migration_Dashboard.qmd",
+      "base": "a3e44009d6b7375f60a58b11278cbe079d93407249c0c5e1c97f00e12e98c09a",
+      "url": "/a3e44009d6b7375f60a58b11278cbe079d93407249c0c5e1c97f00e12e98c09a.html"
     }
   ]
 }

.github/runners/Dockerfile.quarto-doc-builder

@@ -55,7 +55,7 @@ RUN arch=$(dpkg --print-architecture) && \
 #                 files.upload(file=), see gemini_client.py)
 # tiktoken     -> token counting for the LLM rate limiter
 RUN pip install --no-cache-dir --break-system-packages \
-    panflute PyYAML google-genai==2.6.0 tiktoken==0.13.0 && \
+    panflute>=2.3.1 PyYAML google-genai==2.6.0 tiktoken==0.13.0 && \
     python3 -m pip cache purge 2>/dev/null || true && \
     rm -rf /root/.cache /tmp/* /var/tmp/*
 

.github/scripts/ai/update_versions_and_changelogs.py

@@ -816,9 +816,8 @@ def _do_call():
         print("=" * 70)
         print(result_text)
         print("=" * 70)
-        raise Exception(
-            f"Batch {batch_num}/{total_batches} failed: Invalid JSON response"
-        )
+        # Empty -> batch_with_retry splits and retries the halves instead of aborting.
+        return {}
     except Exception as e:
         print(f"\n❌ ERROR: Batch {batch_num}/{total_batches} processing failed")
         print(f"    Error: {e}")
@@ -862,6 +861,8 @@ def _process(sub_batch, _i=i):
 
 def calculate_new_version(current_version, bump_type, major_from_filename):
     """Calculate new version based on bump type"""
+    # First published version is 1.0.0, so a _v0 filename floors to major 1.
+    major_from_filename = max(major_from_filename, 1)
     try:
         parts = current_version.split(".")
         major = int(parts[0])
@@ -947,7 +948,7 @@ def initialize_first_release(all_files):
             print(f"[ERROR] {filepath}: {e}")
             continue
 
-        initial_version = f"{major_version}.0.0"
+        initial_version = f"{max(major_version, 1)}.0.0"
 
         update_qmd_version_only(filepath, initial_version)
 
@@ -1073,7 +1074,7 @@ def main():
         file_info[filepath] = {
             "major_version": major_version,
             "current_version": versions_metadata.get(filepath, {}).get(
-                "current_version", f"{major_version}.0.0"
+                "current_version", f"{max(major_version, 1)}.0.0"
             ),
         }
 

.github/scripts/build/build-docs.sh

@@ -31,6 +31,11 @@ step() {
   _STEP_PREV=$now; _STEP_NAME="$*"
 }
 
+# Snapshot the pristine source before the build mutates it, so a local build is
+# easy to undo. After testing, restore your working tree with:
+#   rm -rf DOCS origin_DOCS && mv source_DOCS DOCS
+rm -rf source_DOCS && cp -rp DOCS source_DOCS
+
 # Apply cached intros/keywords before the rename - the cache is keyed by original path.
 echo "Injecting cached intros & keywords (no API)..."
 python3 .github/scripts/build/apply_cached_intros.py DOCS
@@ -73,6 +78,13 @@ python3 ../.github/scripts/build/fill_version.py .
 echo "Balancing table column widths..."
 python3 ../.github/scripts/qmd-tools/fix_table_colwidths.py .
 
+# Promote bare "Table N:"/"Figure N:" captions (left as plain text by the
+# converters) into .tbl-caption divs / image alt text, so they render as styled
+# captions instead of body text. Runs after the grid->pipe conversion above so a
+# caption next to a (now pipe) table is recognised. Build-copy only; idempotent.
+echo "Promoting bare table/figure captions..."
+python3 ../.github/scripts/qmd-tools/promote_bare_captions.py .
+
 # Bake image descriptions into the qmd source now, so the render doesn't re-hash
 # every image once per format (see the script). This was a Lua filter.
 echo "Baking image descriptions into qmd source..."
@@ -83,9 +95,12 @@ python3 ../.github/scripts/build/inject_image_descriptions.py .
 cp _quarto-no-headers.yml _quarto.yml
 
 step "[3/6] Rendering all documents (HTML + Typst PDF + gfm) in one pass..."
-# Render every format in the config - html (site), typst (PDFs), gfm
-# (the .llms.md sidecars). One pass over the files instead of one per format.
+# # Temporary move out of docs before render to avoid Jupyter engine selections crashes
+# echo "	 [BUILD BYPASS] Temporarily moving Ice products out of the build context..."
+# mv products/products_Algorithm_theoretical_basis_document_-_High_Resolution_Ice_products_Europe.qmd ../origin_DOCS/
 quarto_render --no-clean
+# echo "	 [BUILD BYPASS] Restoring Ice products..."
+# mv ../origin_DOCS/products_Algorithm_theoretical_basis_document_-_High_Resolution_Ice_products_Europe.qmd products/
 
 # Back up sitemap.xml and llms.txt - the index.qmd renders below regenerate
 # them, and we want to keep the values from this first render.
@@ -99,8 +114,10 @@ python3 ../.github/scripts/build/generate_index_all.py
 step "[5/6] Rendering index.qmd files..."
 mv _quarto.yml _quarto_not_used.yml
 mv _quarto-index.yml _quarto.yml
+# Serial (-P1): parallel renders raced on shared _site files (sitemap/search/
+# listings) and intermittently failed the whole build with xargs exit 123.
 find ./ -type f -name index.qmd -print0 | \
-  xargs -0 -P4 -I{} \
+  xargs -0 -P1 -I{} \
   bash -c 'quarto render "$1" --profile index --to html --no-clean --quiet 2>&1 | grep -v -e "Unknown meta key .* specified in a metadata Shortcode" -e "^Output created:"; exit ${PIPESTATUS[0]}' _ {}
 mv _quarto.yml _quarto-index.yml
 cp _quarto_not_used.yml _quarto.yml && rm _quarto_not_used.yml

.github/scripts/build/clean_llms_md.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Strip residual block-level HTML wrappers from generated .llms.md sidecars.
+
+The gfm writer + the simplify_tables_gfm.lua filter already linearize tables and
+remove styling, but Quarto injects crossref float wrappers (`<div id="tbl-…">` …
+`</div>`) and the occasional layout `<div style="overflow-x:…">` AFTER the pandoc
+filters run, so they can't be removed at filter level. These bare wrapper lines are
+pure noise in the plain-text companion the RAG ingests. This post-render pass removes
+standalone block-level `<div>`/`</div>` lines (inline tags like <sub>/<sup> in prose
+are left intact).
+
+Idempotent. Usage:
+    python3 clean_llms_md.py _site            # walk *.llms.md under a directory
+    python3 clean_llms_md.py a.llms.md b.md    # specific files
+"""
+
+import re
+import sys
+from pathlib import Path
+
+# A line that is ONLY an opening/closing <div …> (optionally indented). Block-level
+# wrapper noise — never matches inline tags mid-prose.
+_BARE_DIV = re.compile(r"^[ \t]*</?div\b[^>]*>[ \t]*$")
+
+
+def clean_text(text: str) -> str:
+    out = [ln for ln in text.split("\n") if not _BARE_DIV.match(ln)]
+    cleaned = "\n".join(out)
+    # collapse the blank-line runs a removed wrapper can leave behind (3+ → 2)
+    cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
+    return cleaned
+
+
+def clean_file(path: Path) -> bool:
+    original = path.read_text(encoding="utf-8")
+    cleaned = clean_text(original)
+    if cleaned != original:
+        path.write_text(cleaned, encoding="utf-8")
+        return True
+    return False
+
+
+def _iter_targets(args):
+    for a in args:
+        p = Path(a)
+        if p.is_dir():
+            yield from p.rglob("*.llms.md")
+        elif p.exists():
+            yield p
+
+
+def main(argv) -> int:
+    if not argv:
+        print(__doc__)
+        return 1
+    changed = 0
+    for path in _iter_targets(argv):
+        if clean_file(path):
+            changed += 1
+    print(f"clean_llms_md: cleaned {changed} file(s)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

.github/scripts/build/fill_version.py

@@ -6,13 +6,17 @@
 version, so we fill it each build, like intros and changelogs.
 
 The value is the tracked version from .llm_cache/versions.json, looked up via the
-original-filename field. No record yet -> {major}.0.0 from the _vN.qmd name. No
+original-filename field. The doc's own `version:` frontmatter is never trusted as
+input - it is output we overwrite. No record yet -> seed one (max(major,1).0.0,
+so a _v0 or a name without _vN starts at 1.0.0) so the doc becomes tracked. No
 bump is computed here.
 """
 
 import argparse
+import json
 import re
 import sys
+from datetime import date
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # .github/scripts
@@ -68,7 +72,8 @@ def main():
     vf = Path(args.versions_file) if args.versions_file else repo_root / VERSIONS_FILE
     versions = load_json_or_empty(vf, label="versions")
 
-    filled = baseline = 0
+    filled = baseline = seeded = 0
+    today = date.today().isoformat()
     for qmd in sorted(Path(args.docs_dir).rglob("*.qmd")):
         if {"_site", ".quarto", "_meta"} & set(qmd.parts):
             continue
@@ -79,31 +84,49 @@ def main():
         _, end = bounds
 
         src = fm_value(lines, end, "original-filename")
-        major = major_from_name(src) if src else None
-        if major is None:
-            major = major_from_name(qmd.name)
-        if major is None:
-            print(f"[fill_version] {qmd}: no _vN in filename, skipping")
-            continue
-
+        mj = major_from_name(src) if src else None
+        if mj is None:
+            mj = major_from_name(qmd.name)
+        # First published version is 1.0.0: a _v0 or a name without _vN starts
+        # at 1.0.0, while _v4 etc. keep their filename major.
+        baseline_major = max(mj, 1) if mj is not None else 1
+
+        key = f"DOCS/{src}" if src else None
         tracked = None
-        if src:
-            entry = versions.get(f"DOCS/{src}") or versions.get(src)
+        if key:
+            entry = versions.get(key) or versions.get(src)
             tracked = (entry or {}).get("current_version")
 
-        if tracked and tracked.split(".")[0] == str(major):
+        if tracked and (mj is None or tracked.split(".")[0] == str(mj)):
             version = tracked
         else:
-            version = f"{major}.0.0"
+            version = f"{baseline_major}.0.0"
             baseline += 1
+            # No record yet: seed one so the doc becomes tracked, rather than
+            # re-deriving the fallback every build. Keyed by original-filename.
+            if key and key not in versions:
+                versions[key] = {
+                    "current_version": version,
+                    "last_bump": "initial",
+                    "last_bump_reason": "First release",
+                    "last_release_tag": "initial",
+                    "last_updated": today,
+                    "major_from_filename": baseline_major,
+                }
+                seeded += 1
 
         if set_version(lines, end, version):
             qmd.write_text("".join(lines), encoding="utf-8")
             filled += 1
 
+    if seeded:
+        vf.parent.mkdir(parents=True, exist_ok=True)
+        with vf.open("w", encoding="utf-8") as f:
+            json.dump(versions, f, indent=2, sort_keys=True)
+
     print(
         f"[fill_version] set version on {filled} files "
-        f"({baseline} fell back to {{major}}.0.0)"
+        f"({baseline} used max(major,1).0.0 baseline, {seeded} new cache entries)"
     )
 
 

.github/scripts/build/group_docs_by_category.py

@@ -2,6 +2,7 @@
 import shutil
 import json
 import secrets
+import sys
 from pathlib import Path
 import yaml
 
@@ -13,19 +14,15 @@
     ".quarto",
 }
 
-# Category to directory mapping
-# Multiple categories can map to the same directory
-CATEGORY_TO_DIRECTORY_MAP = {
-    "guidelines": "guidelines",
-    "products": "products",
-}
-
 
 # Change working directory to root of the repository
 script_dir = Path(__file__).parent
 root_dir = script_dir / "../../../"
 os.chdir(root_dir.resolve())
 
+sys.path.insert(0, str(script_dir.parent.resolve()))
+from helpers.categories import directory_for, non_browsable_names  # noqa: E402
+
 # Non-browsable doc mapping file
 NON_BROWSABLE_MAP_PATH = Path(".github/non_browsable_doc_map.json")
 
@@ -67,13 +64,7 @@ def add_secret_mapping(mappings, source, base, url):
 
 
 def get_directory_for_category(category):
-    if not category:
-        return "uncategorized"
-
-    if category in CATEGORY_TO_DIRECTORY_MAP:
-        return CATEGORY_TO_DIRECTORY_MAP[category]
-
-    return category
+    return directory_for(category)
 
 
 def read_qmd_frontmatter(file_path):
@@ -168,7 +159,7 @@ def group_qmd_files_by_category(source_dir="origin_DOCS", target_dir="DOCS"):
         rel_source = str(qmd_file.relative_to(source_path))
         project_name = qmd_file.parts[1] if len(qmd_file.parts) > 2 else ""
 
-        if category == "non-browsable":
+        if category in non_browsable_names():
             nb_dir = target_path / "non-browsable"
             nb_dir.mkdir(exist_ok=True)
             mapping = get_secret_mapping_for_source(secret_mappings, rel_source)