api-tests.yml

name: GitHub API Tests

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:
  # Nightly. Rolling-flip detection needs at least five scored runs before it
  # will classify anything, and a push-only trigger can go a fortnight without
  # producing one.
  schedule:
    - cron: "0 2 * * *"

# Grant GITHUB_TOKEN write permissions so it can deploy to gh-pages
permissions:
  contents: write
  pages: write
  id-token: write

jobs:
  api-tests:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install -r requirements.txt

      # The default GITHUB_TOKEN is an "integration" token and cannot access /user or create repos.
      # We must use a Personal Access Token (PAT) stored as a GitHub Secret.
      - name: Run API tests
        env:
          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
        run: >
          pytest tests/ -v --alluredir=allure-results
          --junitxml=junit-results.xml

      - name: Upload machine-readable results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: results-junit
          path: junit-results.xml
          retention-days: 30

      # The input for the `report` job below.
      - name: Upload Allure results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: allure-results
          path: allure-results/
          retention-days: 30

      # Feed this run into TestPulse, which is what puts this suite on the
      # public dashboard at testpulse-eight.vercel.app.
      #
      # always(), because a failing run is the data point that matters most -
      # skipping ingest on red would mean the flake history only ever sees green.
      #
      # Not on pull requests: branches that may never merge would pollute the
      # history, and the flake numbers would end up describing code nobody runs.
      - name: Record the run in TestPulse
        if: always() && github.event_name != 'pull_request'
        continue-on-error: true
        uses: Mohanad49/testpulse@main
        with:
          path: junit-results.xml
          format: junit
          suite: github-api
          database-url: ${{ secrets.TESTPULSE_DATABASE_URL }}
          dashboard-url: https://testpulse-eight.vercel.app
          comment: "false"

  # Builds the Allure report from the api-tests job's results and publishes it
  # to gh-pages.
  #
  # This used to be the last four steps of `api-tests`, using
  # simple-elf/allure-report-action. That is a Docker action, and GitHub builds a
  # Docker action's image while setting up the job, before the first step runs.
  # The image starts from public.ecr.aws, which rate-limits anonymous pulls, and
  # on 9 of the 20 nights up to 2026-10-30 the pull got a 429. The job died in
  # setup: no checkout, no install, no tests, nothing for TestPulse, and the
  # repeat runs were skipped with it. Then the deploy step, which ran on
  # always(), published the empty folder over the live report.
  #
  # Now the report cannot stop the tests. It has its own job, the Allure CLI
  # runs straight on the runner so there is no image to pull, it only deploys a
  # report it actually generated, and continue-on-error stops a publishing
  # problem from showing up as a failed test run.
  report:
    needs: api-tests
    if: ${{ !cancelled() }}
    # The Allure CLI needs Java, and 24.04 ships Java 17 as its default.
    runs-on: ubuntu-24.04
    continue-on-error: true
    steps:
      - name: Download Allure results
        uses: actions/download-artifact@v4
        with:
          name: allure-results
          path: allure-results

      - name: Load Allure history
        uses: actions/checkout@v4
        continue-on-error: true
        with:
          ref: gh-pages
          path: gh-pages

      # The same steps the action ran, with the same Allure release, so gh-pages
      # keeps its layout: a folder per run, the newest 20 kept, last-history
      # feeding the trend charts, and an index.html that redirects to the latest
      # run.
      - name: Generate Allure report
        env:
          ALLURE_VERSION: "2.44.0"
          ALLURE_SHA256: "7021a90828c00cd6ec992027cce48e8a94bd87fad43d0c2dcac4795cadc178d7"
          KEEP_REPORTS: "20"
        run: |
          set -euo pipefail

          curl --fail --silent --show-error --location --retry 5 --retry-all-errors \
            --output "$RUNNER_TEMP/allure.tgz" \
            "https://repo.maven.apache.org/maven2/io/qameta/allure/allure-commandline/$ALLURE_VERSION/allure-commandline-$ALLURE_VERSION.tgz"
          echo "$ALLURE_SHA256  $RUNNER_TEMP/allure.tgz" | sha256sum --check --strict
          tar -xzf "$RUNNER_TEMP/allure.tgz" -C "$RUNNER_TEMP"
          allure="$RUNNER_TEMP/allure-$ALLURE_VERSION/bin/allure"

          pages_url="https://$GITHUB_REPOSITORY_OWNER.github.io/${GITHUB_REPOSITORY#*/}"
          run="$GITHUB_RUN_NUMBER"

          mkdir -p gh-pages allure-history
          cp -r gh-pages/. allure-history
          rm -rf allure-history/.git allure-history/last-history

          # Make room for this run by keeping only the newest KEEP_REPORTS - 1.
          for dir in allure-history/*/; do
            name="$(basename "$dir")"
            if [[ "$name" =~ ^[0-9]+$ ]]; then echo "$name"; fi
          done | sort -rn | tail -n +"$KEEP_REPORTS" | while read -r old; do
            rm -rf "allure-history/$old"
          done

          if [ -d gh-pages/last-history ]; then
            mkdir -p allure-results/history
            cp -r gh-pages/last-history/. allure-results/history
          fi

          cat > allure-results/executor.json <<EOF
          {"name": "GitHub Actions", "type": "github", "reportName": "Allure Report with history",
           "url": "$pages_url", "reportUrl": "$pages_url/$run",
           "buildUrl": "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID",
           "buildName": "GitHub Actions Run #$GITHUB_RUN_ID", "buildOrder": "$run"}
          EOF

          "$allure" generate --clean allure-results -o allure-report

          cp -r allure-report/. "allure-history/$run"
          cp -r allure-report/history allure-history/last-history
          printf '%s\n' \
            '<!DOCTYPE html><meta charset="utf-8">' \
            "<meta http-equiv=\"refresh\" content=\"0; URL=$pages_url/$run/index.html\">" \
            '<meta http-equiv="Pragma" content="no-cache"><meta http-equiv="Expires" content="0">' \
            > allure-history/index.html

      - uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_branch: gh-pages
          publish_dir: allure-history

  # Repeat runs of one unchanged commit, on a schedule only.
  #
  # pytest cannot report retries the way Playwright can, so the same-commit
  # strategy has to get its evidence the other way: several runs of one SHA. If
  # two of these three disagree, the code was identical and the outcome was not.
  #
  # Separate from `api-tests` rather than a matrix on it, because the Allure
  # report on gh-pages is built from that one job's results, and parallel
  # copies would race on the push.
  #
  # `needs` and `max-parallel: 1` are the schedule-side half of the secondary
  # rate-limit fix, and they are the reason this job stopped being free.
  #
  # Added on 2026-07-29, this job ran its three attempts in parallel, alongside
  # `api-tests`, all four sharing one token. Every scheduled run from the next
  # night onward failed: four concurrent copies of a suite that creates
  # repositories, issues and comments is a burst of content creation, and
  # GitHub's secondary limit exists precisely to stop that. The suite was not
  # measuring flakiness in the API - it was measuring the interference between
  # its own jobs, which is the one thing a flake detector must not do.
  #
  # Serialised, the four runs take longer and produce the same evidence. Three
  # runs of one SHA disagreeing is only meaningful if nothing about the runs
  # differed, and "how many siblings were hammering the same token" is a
  # difference.
  repeat:
    name: Repeat run ${{ matrix.attempt }}
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    needs: api-tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        attempt: [1, 2, 3]
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - run: pip install -r requirements.txt

      # A failing run is the data being collected, not a broken pipeline.
      - name: Run tests
        continue-on-error: true
        env:
          # PAT_TOKEN, matching the main job. The automatic GITHUB_TOKEN cannot
          # create repositories or PATCH /user, which is what this suite does, so
          # using it here would have produced a 401 every night and looked
          # exactly like a flaky suite.
          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
        run: pytest tests/ -v --alluredir=allure-results --junitxml=junit-results.xml

      - name: Upload results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: results-attempt-${{ matrix.attempt }}
          path: |
            junit-results.xml
            allure-results/
          retention-days: 30

      # Each attempt is stored as its own run. TestPulse's natural key includes
      # the start time, so three runs of one unchanged commit do not collapse
      # into one - which is the entire reason this job exists.
      - name: Record the run in TestPulse
        if: always()
        continue-on-error: true
        uses: Mohanad49/testpulse@main
        with:
          path: junit-results.xml
          format: junit
          suite: github-api
          database-url: ${{ secrets.TESTPULSE_DATABASE_URL }}
          dashboard-url: https://testpulse-eight.vercel.app
          comment: "false"