ADFA-4739: Kotlin docs DB pipeline + Build Kotlin Docs GitHub Action

.github/workflows/build-kotlin-docs.yaml

@@ -76,11 +76,39 @@ on:
           Ignored if skip_website_docs is true.
         required: false
         default: ''
+      kotlin_libs_version:
+        description: >-
+          Version of the published kotlin-stdlib/-reflect/-test artifacts to
+          document (Gradle -PdeployVersion). kotlin_big extracts the real
+          binaries at this version rather than requiring a local build of the
+          whole kotlin repo, which is what the checkout's own
+          defaultSnapshotVersion would otherwise demand. Keep this in step with
+          kotlin_ref. Set empty to fall back to that snapshot default, which
+          only resolves if you have built the kotlin repo yourself.
+        required: false
+        default: '2.4.10'
+      kotlin_libs_repo:
+        description: >-
+          Maven repository to resolve those artifacts from (Gradle
+          -PkotlinLibsRepo). kotlin_big already declares mavenCentral(), so a
+          released kotlin_libs_version needs nothing here; set it to point at a
+          private or snapshot repository instead.
+        required: false
+        default: ''
       skip_website_docs:
         description: 'Skip the kotlin-web-site steps and only refresh kotlin-stdlib/-reflect/-test JSON content.'
         required: false
         default: false
         type: boolean
+      skip_stdlib_docs:
+        description: >-
+          Skip the kotlin-stdlib/-reflect/-test steps (cloning JetBrains/kotlin,
+          the Dokka JSON build, and the sync into the database) and only refresh
+          the kotlin-web-site content. The mirror image of skip_website_docs -
+          setting both leaves nothing for the run to do and is rejected.
+        required: false
+        default: false
+        type: boolean
       dry_run:
         description: >-
           If true, build and verify everything but do NOT upload the result
@@ -96,6 +124,11 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 180
     env:
+      # The repo's uv convention, as one string so every step below reads the
+      # same: `$UV_RUN python3 <script>`. Mirrors run_e2e_pipeline_test.sh's
+      # own UV_RUN array. Expanded unquoted on purpose - it is a fixed literal
+      # with no paths or user input in it.
+      UV_RUN: uv run --with-requirements requirements.txt --with google-api-python-client --with google-auth-httplib2 --with google-auth-oauthlib --
       KOTLIN_WEB_SITE_REF: ${{ inputs.kotlin_web_site_ref }}
       KOTLIN_REF: ${{ inputs.kotlin_ref }}
       DB_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_FILE_ID }}
@@ -108,12 +141,34 @@ jobs:
       # Drive). Leave both empty ('') for normal operation.
       TEST_DB_FILE_ID: ''
       TEST_IMAGES_ZIP_FILE_ID: ''
+      SKIP_WEBSITE_DOCS: ${{ inputs.skip_website_docs }}
+      KOTLIN_LIBS_VERSION: ${{ inputs.kotlin_libs_version }}
+      KOTLIN_LIBS_REPO: ${{ inputs.kotlin_libs_repo }}
+      SKIP_STDLIB_DOCS: ${{ inputs.skip_stdlib_docs }}
+      # The ADFA-4737 blacklist, defined once and consumed by both the
+      # populate_db.py step and the verification step that checks its effect.
+      # Previously spelled out separately in each, which let the verification
+      # drift onto a different list than the one actually applied and still
+      # report PASS. run_e2e_pipeline_test.sh already had it right (a single
+      # BLACKLIST array expanded at both call sites); this matches that.
+      # One entry per line, read back with `mapfile -t`. Re-derive these from
+      # kotlin-web-site/docs/kr.tree if its nav structure has changed.
+      # "|-" (not "|") so there's no trailing blank line to become a 4th,
+      # empty array element.
+      BLACKLISTED_ELEMENT_TITLES: |-
+        Development\/Web development
+        Interoperability\/Swift/Objective-C and C interop
+        Interoperability\/JavaScript interop
     steps:
       - name: Checkout OfflineDocumentationTools
         uses: actions/checkout@v4
 
       - name: Resolve Google Drive file IDs
         run: |
+          if [ "$SKIP_WEBSITE_DOCS" = "true" ] && [ "$SKIP_STDLIB_DOCS" = "true" ]; then
+            echo "Error: skip_website_docs and skip_stdlib_docs are both true - that skips every step that changes the database, leaving nothing for this run to do" >&2
+            exit 1
+          fi
           DB_FILE_ID="${TEST_DB_FILE_ID:-$DB_FILE_ID_SECRET}"
           IMG_FILE_ID="${TEST_IMAGES_ZIP_FILE_ID:-${IMAGES_ZIP_FILE_ID_INPUT:-$IMAGES_ZIP_FILE_ID_SECRET}}"
           if [ -z "$DB_FILE_ID" ]; then
@@ -130,6 +185,9 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+
       - name: Set up JDK (for the kdoc-to-json / kotlin-stdlib-docs Gradle builds)
         uses: actions/setup-java@v4
         with:
@@ -148,15 +206,15 @@ jobs:
           # no Python binding exposes a custom dictionary (ADFA-5153).
           sudo apt-get install -y pngquant unzip zip sqlite3 brotli
 
-      - name: Install Python dependencies
-        run: |
-          pip install -r requirements.txt
-          # markdown-it-py/scour/cairosvg: ProcessKotlinWebsiteJSON's own
-          # requirements (see its README), not in the root requirements.txt.
-          # google-api-python-client & friends: Drive download/upload, same
-          # libraries check-tools/download_database.py already depends on.
-          pip install markdown-it-py scour cairosvg \
-            google-api-python-client google-auth-httplib2 google-auth-oauthlib
+      # No install step: every Python invocation below runs through $UV_RUN
+      # (see the job env), which resolves requirements.txt into a cached
+      # ephemeral environment per call. That is this repo's convention - uv,
+      # never pip or the `uv pip` shim - and is what ProcessKotlinWebsiteJSON's
+      # README and run_e2e_pipeline_test.sh already document and use.
+      #
+      # The former "pip install markdown-it-py" here was dead work: it is in
+      # requirements.txt (markdown-it-py>=2.0) and was already installed by the
+      # line above it. scour/cairosvg are in there too.
 
       - name: Authenticate to Google Cloud using Workload Identity Federation
         uses: google-github-actions/auth@v2
@@ -168,7 +226,7 @@ jobs:
 
       - name: Download current documentation.db from Google Drive
         run: |
-          python3 check-tools/download_database.py "$DB_FILE_ID" documentation.zip
+          $UV_RUN python3 check-tools/download_database.py "$DB_FILE_ID" documentation.zip
           unzip -o documentation.zip
           if [ ! -f documentation.db ]; then
             found="$(find . -maxdepth 2 -name documentation.db | head -n1)"
@@ -180,6 +238,7 @@ jobs:
           echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV"
 
       - name: 'Notify Slack: build started'
+        id: slack_started
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
@@ -208,12 +267,12 @@ jobs:
           # download_database.py is a generic Drive-file-by-ID downloader
           # despite its name - reused here rather than duplicating the
           # WIF/Drive-API download logic for a second file type.
-          python3 check-tools/download_database.py "$IMG_FILE_ID" webHelpImages.zip
+          $UV_RUN python3 check-tools/download_database.py "$IMG_FILE_ID" webHelpImages.zip
 
       - name: 'Step 1/5: find_missing_assets.py (source QA report)'
         if: ${{ !inputs.skip_website_docs }}
         run: |
-          python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py \
+          $UV_RUN python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py \
             kotlin-web-site/docs missing-assets-report.md
 
       - name: Upload missing-assets report
@@ -226,53 +285,58 @@ jobs:
       - name: 'Step 2/5: populate_db.py (convert docs, prune blacklist, insert into db)'
         if: ${{ !inputs.skip_website_docs }}
         run: |
-          # Same three blacklist entries as run_e2e_pipeline_test.sh
-          # (ADFA-4737) - re-derive these from kotlin-web-site/docs/kr.tree
-          # if its nav structure has changed since this was written.
-          python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \
+          # BLACKLISTED_ELEMENT_TITLES is defined once in this job's env: block
+          # (ADFA-4737); the verification step below reads the same variable.
+          mapfile -t BLACKLIST <<< "$BLACKLISTED_ELEMENT_TITLES"
+          $UV_RUN python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \
             kotlin-web-site/docs \
             ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json \
             webHelpImages.zip \
             documentation.db \
-            --blacklisted-element-titles \
-              'Development\/Web development' \
-              'Interoperability\/Swift/Objective-C and C interop' \
-              'Interoperability\/JavaScript interop'
+            --blacklisted-element-titles "${BLACKLIST[@]}"
 
       - name: 'Step 3/5: insert_optimized_media.py (re-optimize + reinsert images)'
         if: ${{ !inputs.skip_website_docs }}
         run: |
-          # --webp requires an "image/webp" ContentTypes row, which this
-          # database doesn't ship with by default (idempotent).
+          # --webp requires an "image/webp" ContentTypes row. The current
+          # production database already has one, so this is normally a no-op;
+          # it stays for older copies that predate it (idempotent either way).
           sqlite3 documentation.db \
             "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');"
           mkdir -p media
           unzip -q webHelpImages.zip -d media
-          python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py \
+          $UV_RUN python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py \
             media documentation.db \
             --jpeg-quality 85 --webp --webp-quality 90 --verbose
 
       - name: Clone kotlin (for kotlin-stdlib-docs)
+        if: ${{ !inputs.skip_stdlib_docs }}
         run: |
           ARGS=(--depth 1)
           [ -n "$KOTLIN_REF" ] && ARGS+=(--branch "$KOTLIN_REF")
           git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin.git kotlin-repo
 
       - name: 'Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON)'
+        if: ${{ !inputs.skip_stdlib_docs }}
         id: stdlib_docs
         run: |
-          OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh kotlin-repo stdlib-json-build)"
+          ARGS=()
+          [ -n "$KOTLIN_LIBS_VERSION" ] && ARGS+=(--kotlin-libs-version "$KOTLIN_LIBS_VERSION")
+          [ -n "$KOTLIN_LIBS_REPO" ] && ARGS+=(--kotlin-libs-repo "$KOTLIN_LIBS_REPO")
+          OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh \
+            "${ARGS[@]}" kotlin-repo stdlib-json-build)"
           echo "Generated JSON docs at $OUTPUT"
           echo "all_libs_dir=$OUTPUT" >> "$GITHUB_OUTPUT"
 
       - name: 'Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content)'
+        if: ${{ !inputs.skip_stdlib_docs }}
         run: |
-          python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \
+          $UV_RUN python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \
             "${{ steps.stdlib_docs.outputs.all_libs_dir }}" --db documentation.db
 
       - name: Summary
         run: |
-          python3 - documentation.db <<'PYEOF'
+          $UV_RUN python3 - documentation.db <<'PYEOF'
           import sqlite3
           import sys
 
@@ -294,10 +358,12 @@ jobs:
       - name: Blacklist pruning verification
         if: ${{ !inputs.skip_website_docs }}
         run: |
-          python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \
-            'Development\/Web development' \
-            'Interoperability\/Swift/Objective-C and C interop' \
-            'Interoperability\/JavaScript interop' <<'PYEOF'
+          # Same BLACKLISTED_ELEMENT_TITLES the populate_db.py step above
+          # applied - read from the job env rather than restated here, so this
+          # check can't silently verify a different list than the one used.
+          mapfile -t BLACKLIST <<< "$BLACKLISTED_ELEMENT_TITLES"
+          $UV_RUN python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \
+            "${BLACKLIST[@]}" <<'PYEOF'
           import sqlite3
           import sys
           import xml.etree.ElementTree as ET
@@ -351,7 +417,7 @@ jobs:
       - name: Upload updated database to Google Drive
         if: ${{ !inputs.dry_run }}
         run: |
-          python3 - <<'PYEOF'
+          $UV_RUN python3 - <<'PYEOF'
           import os
           from google.auth import default
           from googleapiclient.discovery import build
@@ -367,15 +433,38 @@ jobs:
           print(f"Uploaded new revision of {file_id}: {updated}")
           PYEOF
 
+      # The baton must be dropped even when the build fails, otherwise the channel
+      # shows it held forever by a dead run - the exact failure this convention
+      # exists to prevent. Deliberately NOT gated on dry_run: "build started" is
+      # ungated, and since dry_run defaults to true an asymmetric gate meant every
+      # ordinary run grabbed the baton and never dropped it. The message reports
+      # the outcome instead.
+      #
+      # Gated on the acquire having actually happened, though. "build started"
+      # runs after the database is in place, so a run that fails before it (a bad
+      # db_path, a failed download) never grabbed the baton - and a bare always()
+      # would post a release for a lock nobody holds, which is just the original
+      # asymmetry pointing the other way.
       - name: 'Notify Slack: build complete'
-        if: ${{ !inputs.dry_run }}
+        if: always() && steps.slack_started.outcome == 'success'
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          JOB_STATUS: ${{ job.status }}
+          DRY_RUN: ${{ inputs.dry_run }}
         run: |
+          # The baton is always dropped, so the text has to say what actually
+          # happened rather than always claiming an update.
+          if [ "$JOB_STATUS" != "success" ]; then
+            TEXT="Kotlin documentation build FAILED ($JOB_STATUS) - database unchanged. Dropping baton"
+          elif [ "$DRY_RUN" = "true" ]; then
+            TEXT="Kotlin documentation dry run complete - database unchanged. Dropping baton"
+          else
+            TEXT="Updated Kotlin documentation. Dropping baton"
+          fi
           if [ -z "$SLACK_WEBHOOK_URL" ]; then
             echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2
           else
             curl -sS -X POST -H 'Content-type: application/json' \
-              --data '{"text": "Updated Kotlin documentation. Dropping baton"}' \
+              --data "$(printf '{"text": "%s"}' "$TEXT")" \
               "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2
           fi

.github/workflows/python-tests.yaml

@@ -0,0 +1,79 @@
+name: Python tests
+
+# Runs the repo's pytest suites on every push and PR.
+#
+# Added because nothing here executed them: the Kotlin-docs pipeline ships
+# ~1,300 lines of regression tests covering permanent, silent data-loss paths
+# (a migration deleting an unrelated page, an image optimizer collapsing two
+# sources onto one output, a chunk chain reassembling truncated), and until now
+# they only ran when someone remembered to run them locally. Tests that never
+# run in CI rot, and these are exactly the ones whose failure is invisible
+# without them.
+#
+# The suites are separate because their dependencies are: docdb-studio and
+# check-tools each own a pyproject.toml + uv.lock, while ProcessKotlinWebsiteJSON
+# runs against the root requirements.txt.
+#
+# check-tools is NOT run here, deliberately. Its suite covers the health checker
+# behind the daily docdb-regression-test.yaml cron and genuinely ought to be
+# wired up - but 6 of its 13 tests already fail on main
+# (test_db_health_checker.py: the checker reports no issues where the tests
+# expect one), in code this PR doesn't touch. Adding the step as-is would make
+# every PR red for an unrelated reason. Fixing those belongs in its own change;
+# adding the step is a one-liner once they pass:
+#   - name: check-tools
+#     working-directory: check-tools
+#     run: uv run --frozen -- python -m pytest -q
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: python-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update -y
+          # pngquant: optimize_media.py shells out to it, and its tests exercise
+          #   the real binary rather than mocking it.
+          # brotli:   the CLI, not the Python package - DictionaryCompressor uses
+          #   it because no Python binding exposes a custom dictionary.
+          # zstd:     train_dictionary uses `zstd --train-fastcover` to build the
+          #   shared dictionary the migration tests need.
+          sudo apt-get install -y pngquant brotli zstd
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs'
+        run: |
+          uv run --with-requirements requirements.txt --with pytest -- \
+            python -m pytest \
+              ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON \
+              scripts/sync_kotlin_stdlib_docs \
+              -q
+
+      - name: docdb-studio
+        working-directory: docdb-studio
+        run: uv run --frozen -- python -m pytest -q
+

.gitignore

@@ -12,3 +12,11 @@ __pycache__/
 *$py.class
 *.db
 *.sqlite
+run_e2e_pipeline_test.local.sh
+grep_content_blobs.local.py
+
+# Timestamped safety backups written by populate_db.py /
+# insert_optimized_media.py / sync_kdoc_json_to_db.py before they modify a
+# database ("*.db" above does not match these - the timestamp comes last).
+*.db.backup-*
+*.db.bak.*