From 7fe089b1df5b57e546e366a0347841aa1521ca13 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Wed, 7 Oct 2026 21:41:52 +0300 Subject: [PATCH 1/5] ci: run tests and analysis on self-hosted runners, builds stay on GitHub Tests and analysis pick their runner from the CI_RUNS_ON repository variable (default ubuntu-latest); pull requests from forks always use GitHub-hosted runners. Add scripts/ci/setup-runner.sh to prepare a machine and register runners, and docs/ci-runners.md. --- .github/workflows/ci.yml | 14 ++++-- docs/README.md | 1 + docs/ci-runners.md | 69 +++++++++++++++++++++++++++++ scripts/ci/setup-runner.sh | 90 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 docs/ci-runners.md create mode 100755 scripts/ci/setup-runner.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16e44036..241d85c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,10 @@ jobs: # in the `rest` section. test-sections: name: Tests (${{ matrix.section }}) - runs-on: ubuntu-latest + # Self-hosted when the repository variable CI_RUNS_ON is set (see + # docs/ci-runners.md); pull requests from forks always run on GitHub-hosted + # runners, never on our own machines. + runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && fromJSON('"ubuntu-latest"') || fromJSON(vars.CI_RUNS_ON || '"ubuntu-latest"') }} strategy: fail-fast: false matrix: @@ -39,6 +42,7 @@ jobs: cache: true - name: Install Linux dependencies (plugins) + if: runner.environment == 'github-hosted' run: | sudo apt-get update sudo apt-get install -y libsecret-1-dev @@ -54,7 +58,11 @@ jobs: echo "No tests in section ${{ matrix.section }}" exit 0 fi - flutter test --exclude-tags=golden "${paths[@]}" + # Several runners share one machine, so cap the test processes each + # job starts. + flutter test --exclude-tags=golden \ + --concurrency="${{ runner.environment == 'self-hosted' && '2' || '4' }}" \ + "${paths[@]}" # Single required check for the whole suite: passes only when every section did. test: @@ -79,7 +87,7 @@ jobs: analyze: name: Code Analysis - runs-on: ubuntu-latest + runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && fromJSON('"ubuntu-latest"') || fromJSON(vars.CI_RUNS_ON || '"ubuntu-latest"') }} steps: - uses: actions/checkout@v4 diff --git a/docs/README.md b/docs/README.md index 5b044155..5f39d1e3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,6 +22,7 @@ Index of Querya Desktop documentation, grouped by audience. - [Tags and releases](tags-and-releases.md) — tag/release policy. - [Release checklist](release-checklist.md) — step-by-step release flow. +- [Self-hosted CI runners](ci-runners.md) — where tests and analysis run, the fork rule, the `CI_RUNS_ON` switch. - [macOS signing](macos-signing.md) — signing and notarization track. ## Planning diff --git a/docs/ci-runners.md b/docs/ci-runners.md new file mode 100644 index 00000000..e4c76aa2 --- /dev/null +++ b/docs/ci-runners.md @@ -0,0 +1,69 @@ +# Self-hosted CI runners + +Tests and static analysis (`Run Tests` sections and `Code Analysis` in +`.github/workflows/ci.yml`) can run on our own machines. Builds stay on GitHub: +`build-linux-release` in `ci.yml` and everything in `release.yml` (Windows, +macOS with signing, Linux packages) use GitHub-hosted runners. + +## How a job picks its runner + +Each of the two jobs has + +```yaml +runs-on: ${{ && 'ubuntu-latest' || fromJSON(vars.CI_RUNS_ON || '"ubuntu-latest"') }} +``` + +- **Pull requests from forks always run on GitHub-hosted runners**, never on + ours, whatever the variable says. +- Otherwise the repository variable `CI_RUNS_ON` decides. Unset (or deleted) means + `ubuntu-latest`. Set it to move the jobs to our runners: + +```bash +gh variable set CI_RUNS_ON --repo QueryaHub/Querya-Desktop \ + --body '["self-hosted","linux","querya-ci"]' +# back to GitHub-hosted: +gh variable delete CI_RUNS_ON --repo QueryaHub/Querya-Desktop +``` + +That switch is the first thing to flip if our machine is down or being serviced. + +On a self-hosted runner the apt step is skipped (the native packages are +installed by `scripts/ci/setup-runner.sh`) and `flutter test` runs with +`--concurrency=2`, because several runners share one machine. + +## Repository settings + +The repository is public, so under Settings -> Actions -> General keep +**Fork pull request workflows from outside collaborators** on *Require approval +for all external contributors*. Do not add `pull_request_target` workflows that +check out and run PR code on these runners. + +## Setting up the machine + +The runners live in an Ubuntu 24.04 container or VM (here: LXC `querya-ci` on +Proxmox: 8 cores, 16 GiB RAM, 80 GiB disk, outbound access only). Inside it, as +root: + +```bash +TOKEN=$(gh api -X POST repos/QueryaHub/Querya-Desktop/actions/runners/registration-token --jq .token) +RUNNER_URL=https://github.com/QueryaHub/Querya-Desktop \ +RUNNER_TOKEN="$TOKEN" \ +RUNNER_COUNT=4 \ +./scripts/ci/setup-runner.sh +``` + +The script installs the packages the test jobs need, creates an unprivileged +`runner` user without sudo, and registers `RUNNER_COUNT` runners +(`querya-ci-1..N`, label `querya-ci`) as systemd services. Re-running it skips +runners that already exist. The Flutter SDK is installed per job by +`subosito/flutter-action`, exactly as on GitHub-hosted runners, so the version is +pinned in one place (`ci.yml`). + +## Operating notes + +- Each runner runs one job at a time; four runners run four test sections in + parallel. Queue is visible in the Actions tab; add runners by raising + `RUNNER_COUNT` and re-running the script. +- Keep the machine updated (`apt upgrade`) and clean old workspaces under + `/home/runner/actions-runner-*/_work` if the disk fills. +- The runners hold no secrets of ours; do not add any to the container. diff --git a/scripts/ci/setup-runner.sh b/scripts/ci/setup-runner.sh new file mode 100755 index 00000000..89049169 --- /dev/null +++ b/scripts/ci/setup-runner.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Prepares a Debian/Ubuntu machine (VM or LXC) as a host for self-hosted GitHub +# Actions runners that run the Querya test and analysis jobs (see +# docs/ci-runners.md). Safe to re-run: it skips what already exists. +# +# Run as root: +# RUNNER_URL=https://github.com/QueryaHub/Querya-Desktop \ +# RUNNER_TOKEN= \ +# RUNNER_COUNT=4 \ +# ./scripts/ci/setup-runner.sh +# +# The registration token comes from +# gh api -X POST repos/QueryaHub/Querya-Desktop/actions/runners/registration-token --jq .token +# and expires after an hour. +# +# Optional: RUNNER_NAME_PREFIX (default querya-ci), RUNNER_LABELS (default +# querya-ci), RUNNER_USER (default runner), RUNNER_VERSION (default: latest). +set -euo pipefail + +: "${RUNNER_URL:?set RUNNER_URL to the repository or organization URL}" +RUNNER_COUNT="${RUNNER_COUNT:-4}" +RUNNER_NAME_PREFIX="${RUNNER_NAME_PREFIX:-querya-ci}" +RUNNER_LABELS="${RUNNER_LABELS:-querya-ci}" +RUNNER_USER="${RUNNER_USER:-runner}" + +if [[ "$(id -u)" -ne 0 ]]; then + echo "run as root" >&2 + exit 1 +fi + +echo "==> Packages" +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +# The native dependencies the hosted Ubuntu test job installs for the plugins, +# plus what the Flutter SDK itself needs (git, curl, unzip, xz, zip, GLU). +apt-get install -y -qq \ + ca-certificates curl git jq unzip xz-utils zip libglu1-mesa \ + libsecret-1-dev + +echo "==> User ${RUNNER_USER} (no sudo)" +if ! id "$RUNNER_USER" >/dev/null 2>&1; then + useradd --create-home --shell /bin/bash "$RUNNER_USER" +fi + +echo "==> Runner package" +if [[ -z "${RUNNER_VERSION:-}" ]]; then + RUNNER_VERSION="$(curl -fsSL https://api.github.com/repos/actions/runner/releases/latest \ + | jq -r .tag_name | sed 's/^v//')" +fi +ARCH="$(uname -m)" +case "$ARCH" in + x86_64) RUNNER_ARCH=x64 ;; + aarch64) RUNNER_ARCH=arm64 ;; + *) echo "unsupported architecture: $ARCH" >&2; exit 1 ;; +esac +TARBALL="/var/cache/actions-runner-${RUNNER_VERSION}-${RUNNER_ARCH}.tar.gz" +if [[ ! -s "$TARBALL" ]]; then + curl -fsSL -o "$TARBALL" \ + "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" +fi + +for i in $(seq 1 "$RUNNER_COUNT"); do + NAME="${RUNNER_NAME_PREFIX}-${i}" + DIR="/home/${RUNNER_USER}/actions-runner-${i}" + echo "==> ${NAME} (${DIR})" + + if [[ -f "$DIR/.runner" ]]; then + echo " already configured, skipping" + continue + fi + : "${RUNNER_TOKEN:?set RUNNER_TOKEN (registration token) to add runners}" + + mkdir -p "$DIR" + tar -xzf "$TARBALL" -C "$DIR" + chown -R "$RUNNER_USER:$RUNNER_USER" "$DIR" + + # Keep the job environment predictable. + printf 'LANG=C.UTF-8\nLC_ALL=C.UTF-8\n' > "$DIR/.env" + chown "$RUNNER_USER:$RUNNER_USER" "$DIR/.env" + + (cd "$DIR" && runuser -u "$RUNNER_USER" -- ./config.sh --unattended --replace \ + --url "$RUNNER_URL" --token "$RUNNER_TOKEN" \ + --name "$NAME" --labels "$RUNNER_LABELS" --work _work) + + # systemd service running as the unprivileged user. + (cd "$DIR" && ./svc.sh install "$RUNNER_USER" && ./svc.sh start) +done + +echo "==> Done. Services:" +systemctl list-units --type=service --no-legend 'actions.runner.*' || true From c31987fdb0bed29ce5b1f2307b79fb624dad63a4 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Wed, 7 Oct 2026 22:06:49 +0300 Subject: [PATCH 2/5] ci: trigger a run on the self-hosted runners From ab52ec93086dbd5b21a76a1584625211b0da9064 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Wed, 7 Oct 2026 22:31:19 +0300 Subject: [PATCH 3/5] ci: install libsqlite3-dev on self-hosted runner hosts --- scripts/ci/setup-runner.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/ci/setup-runner.sh b/scripts/ci/setup-runner.sh index 89049169..3567d5e1 100755 --- a/scripts/ci/setup-runner.sh +++ b/scripts/ci/setup-runner.sh @@ -33,9 +33,11 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -qq # The native dependencies the hosted Ubuntu test job installs for the plugins, # plus what the Flutter SDK itself needs (git, curl, unzip, xz, zip, GLU). +# libsqlite3-dev provides the unversioned libsqlite3.so that sqflite_common_ffi +# loads (the hosted Ubuntu image has it; a minimal container does not). apt-get install -y -qq \ ca-certificates curl git jq unzip xz-utils zip libglu1-mesa \ - libsecret-1-dev + libsecret-1-dev libsqlite3-dev echo "==> User ${RUNNER_USER} (no sudo)" if ! id "$RUNNER_USER" >/dev/null 2>&1; then From 42ba08c2d1df4676de353b98f0272f4a895a2966 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Wed, 7 Oct 2026 23:04:25 +0300 Subject: [PATCH 4/5] ci: cap job time and retry apt so a stuck mirror cannot hang a run --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 241d85c4..6a12577c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ jobs: # in the `rest` section. test-sections: name: Tests (${{ matrix.section }}) + timeout-minutes: 30 # Self-hosted when the repository variable CI_RUNS_ON is set (see # docs/ci-runners.md); pull requests from forks always run on GitHub-hosted # runners, never on our own machines. @@ -43,9 +44,12 @@ jobs: - name: Install Linux dependencies (plugins) if: runner.environment == 'github-hosted' + # A stuck package mirror used to hold a job for tens of minutes. + timeout-minutes: 8 run: | - sudo apt-get update - sudo apt-get install -y libsecret-1-dev + APT="-o Acquire::Retries=3 -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20" + sudo apt-get $APT update + sudo apt-get $APT install -y libsecret-1-dev - name: Get dependencies run: flutter pub get @@ -87,6 +91,7 @@ jobs: analyze: name: Code Analysis + timeout-minutes: 20 runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && fromJSON('"ubuntu-latest"') || fromJSON(vars.CI_RUNS_ON || '"ubuntu-latest"') }} steps: - uses: actions/checkout@v4 From 427caea7c9dd0a4ca4e9899252e35b94055bed31 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Thu, 8 Oct 2026 05:27:33 +0300 Subject: [PATCH 5/5] ci: re-run on GitHub-hosted runners with the new timeouts