diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index fd5e346f..20cf0d3f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,12 +1,32 @@ -FROM ghcr.io/cirruslabs/flutter:3.41.6 +# Querya Desktop dev container: Flutter (pinned to the CI version) + the Linux +# desktop toolchain + the Docker CLI for the database stack. +FROM ubuntu:24.04 -# Linux desktop toolchain + libs the app links against. +# Keep in step with `flutter-version` in .github/workflows/ci.yml +# (test/guards/devcontainer_flutter_version_test.dart fails when they differ). +ARG FLUTTER_VERSION=3.47.1 + +ENV DEBIAN_FRONTEND=noninteractive \ + FLUTTER_HOME=/opt/flutter \ + PUB_CACHE=/root/.pub-cache +ENV PATH=/opt/flutter/bin:/opt/flutter/bin/cache/dart-sdk/bin:/root/.pub-cache/bin:$PATH + +# Linux desktop toolchain + libs the app links against; bubblewrap for the +# extension sandbox tests; socat to forward localhost ports to the databases. RUN apt-get update \ && apt-get install -y --no-install-recommends \ - clang cmake ninja-build pkg-config libgtk-3-dev libsecret-1-dev \ - libsqlite3-dev libglu1-mesa xvfb git curl ca-certificates \ + build-essential clang cmake ninja-build pkg-config libgtk-3-dev \ + libsecret-1-dev libsqlite3-dev liblzma-dev libglu1-mesa xvfb \ + git curl unzip xz-utils zip ca-certificates bubblewrap socat \ && rm -rf /var/lib/apt/lists/* +# Flutter at the pinned tag (the cirruslabs image registry has no 3.47.x tag). +RUN git config --system --add safe.directory /opt/flutter \ + && git clone --depth 1 --branch "${FLUTTER_VERSION}" \ + https://github.com/flutter/flutter.git /opt/flutter \ + && flutter config --no-analytics --enable-linux-desktop \ + && flutter precache --linux + # Docker CLI so the dev stack (docker/docker-compose.yml) can be started from inside the container. COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker COPY --from=docker:27-cli /usr/local/libexec/docker/cli-plugins/docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ff4ef266..cca3c70d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,14 +8,16 @@ "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {} }, "postCreateCommand": "flutter config --enable-linux-desktop && flutter pub get", - "postStartCommand": "docker compose -f docker/docker-compose.yml up -d || echo 'Docker stack not started (run it manually: cd docker && docker compose up -d)'", - "forwardPorts": [5432, 3306, 6379, 27017, 8123, 9000], + "postStartCommand": "bash scripts/dev/devcontainer_start.sh", + "forwardPorts": [5432, 3306, 6379, 27017, 8123, 9000, 3307], "portsAttributes": { "5432": { "label": "PostgreSQL" }, "3306": { "label": "MySQL" }, "6379": { "label": "Redis" }, "27017": { "label": "MongoDB" }, - "8123": { "label": "ClickHouse HTTP" } + "8123": { "label": "ClickHouse HTTP" }, + "9000": { "label": "ClickHouse native" }, + "3307": { "label": "MySQL 8.0 (profile live-tests)" } }, "customizations": { "vscode": { diff --git a/.github/workflows/devcontainer-image.yml b/.github/workflows/devcontainer-image.yml index c19e5de4..ff03605b 100644 --- a/.github/workflows/devcontainer-image.yml +++ b/.github/workflows/devcontainer-image.yml @@ -40,7 +40,7 @@ jobs: - name: Flutter version from the Dockerfile id: flutter run: | - v=$(sed -n 's#^FROM ghcr.io/cirruslabs/flutter:\(.*\)$#\1#p' .devcontainer/Dockerfile | head -n1) + v=$(sed -n 's#^ARG FLUTTER_VERSION=\(.*\)$#\1#p' .devcontainer/Dockerfile | head -n1) test -n "$v" echo "version=$v" >> "$GITHUB_OUTPUT" @@ -70,7 +70,9 @@ jobs: - name: Smoke test run: | set -euo pipefail - docker run --rm querya-devcontainer:test flutter --version + docker run --rm querya-devcontainer:test flutter --version | tee /dev/stderr | grep -q "Flutter ${{ steps.flutter.outputs.version }}" + docker run --rm querya-devcontainer:test bwrap --version + docker run --rm querya-devcontainer:test socat -V docker run --rm querya-devcontainer:test docker --version docker run --rm querya-devcontainer:test docker compose version docker run --rm querya-devcontainer:test bash -c 'clang --version && cmake --version && ninja --version && pkg-config --exists gtk+-3.0 libsecret-1' diff --git a/docker/.env.example b/docker/.env.example index 8b14d3a2..1a672726 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -3,6 +3,8 @@ POSTGRES_PORT=5432 MYSQL_PORT=3306 +# MySQL 8.0 + mysql_native_password for the live tests (compose profile live-tests) +MYSQL_LIVE_PORT=3307 REDIS_PORT=6379 MONGO_PORT=27017 CLICKHOUSE_HTTP_PORT=8123 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 13d1d6f7..c8bdb4c1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -16,6 +16,8 @@ # auth source: admin collections: users, products, orders # ClickHouse HTTP 8123 / native 9000 db querya user querya password querya # tables: customers, products, orders, order_lines, events +# MySQL 8.0 port 3307 (profile live-tests: docker compose --profile live-tests up -d) +# mysql_native_password, no TLS: for test/core/database/stream_take_drain_test.dart # SQLite local file ./sqlite/data/querya.db # tables: users, products, orders # ───────────────────────────────────────────────────────────────────────── @@ -79,6 +81,44 @@ services: retries: 20 start_period: 30s + # MySQL for the live database tests (profile `live-tests`, not started by a + # plain `docker compose up`): 8.0 with mysql_native_password, because the + # client in those tests connects without TLS and 8.4's default + # caching_sha2_password needs a secure connection. Same server CI starts. + # docker compose --profile live-tests up -d --wait mysql-live + # MYSQL_PORT=3307 flutter test test/core/database/stream_take_drain_test.dart + mysql-live: + profiles: ["live-tests"] + image: mysql:8.0 + container_name: querya-mysql-live + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-querya} + MYSQL_DATABASE: ${MYSQL_DATABASE:-querya} + MYSQL_USER: ${MYSQL_USER:-querya} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:-querya} + ports: + - "${MYSQL_LIVE_PORT:-3307}:3306" + command: + - --default-authentication-plugin=mysql_native_password + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_unicode_ci + healthcheck: + test: + [ + "CMD", + "mysqladmin", + "ping", + "-h", + "127.0.0.1", + "-u${MYSQL_USER:-querya}", + "-p${MYSQL_PASSWORD:-querya}", + ] + interval: 5s + timeout: 5s + retries: 20 + start_period: 30s + redis: image: redis:7-alpine container_name: querya-redis diff --git a/docs/dev-containers.md b/docs/dev-containers.md index 423f8b6d..574434f7 100644 --- a/docs/dev-containers.md +++ b/docs/dev-containers.md @@ -10,7 +10,8 @@ list of seeded data see the header of [`docker/docker-compose.yml`](../docker/do | File | What it does | |------|--------------| | [`.devcontainer/devcontainer.json`](../.devcontainer/devcontainer.json) | The dev container definition: build, features, post-create / post-start commands, forwarded ports, VS Code extensions, host requirements (4 CPUs, 8 GB). | -| [`.devcontainer/Dockerfile`](../.devcontainer/Dockerfile) | `ghcr.io/cirruslabs/flutter` plus the Linux desktop toolchain (clang, cmake, ninja, GTK 3, libsecret, SQLite headers, Xvfb) and the Docker CLI with the Compose plugin. `DISPLAY` is `:99`. | +| [`.devcontainer/Dockerfile`](../.devcontainer/Dockerfile) | Ubuntu 24.04 with Flutter pinned to the version CI uses (`ARG FLUTTER_VERSION`, checked against `ci.yml` by a guard test), the Linux desktop toolchain (clang, cmake, ninja, GTK 3, libsecret, SQLite headers, Xvfb), `bubblewrap` for the extension sandbox tests, `socat`, and the Docker CLI with the Compose plugin. `DISPLAY` is `:99`. | +| [`scripts/dev/devcontainer_start.sh`](../scripts/dev/devcontainer_start.sh) | Runs on every start: brings the stack up, waits for the databases to be healthy, joins the stack's network and forwards `localhost:` to the services. | | [`docker/docker-compose.yml`](../docker/docker-compose.yml) | The database stack (project `querya-dev`): PostgreSQL 16, MySQL 8.4, Redis 7, MongoDB 7, ClickHouse 24.8 and a seeded SQLite file. | The image is also published to GitHub Packages as @@ -21,9 +22,11 @@ cache (`build.cacheFrom`), so the first build pulls the toolchain layers instead them; if the package is unreachable the build simply falls back to building from the Dockerfile. When the container is created it runs `flutter config --enable-linux-desktop` -and `flutter pub get`; every time it starts it tries -`docker compose -f docker/docker-compose.yml up -d` and prints a hint if Docker -is not reachable. +and `flutter pub get`; every time it starts it runs +[`scripts/dev/devcontainer_start.sh`](../scripts/dev/devcontainer_start.sh): it brings the +stack up, waits (up to `QUERYA_STACK_TIMEOUT`, 180 s) for PostgreSQL, MySQL, Redis, MongoDB +and ClickHouse to be healthy, prints which are not, and does not fail the container start +when Docker is unreachable. ## Open the dev container @@ -83,23 +86,25 @@ the ports above. ### From inside the dev container -`localhost` inside the container is the container itself, **not** the host, so -`localhost:5432` finds nothing even though the port is "forwarded" (the forward -goes from the container to your machine, and the databases do not listen in the -container). Pick one: +The databases run on the host's daemon next to the dev container, so nothing +listens on the container's own loopback. The start script therefore does two +things for you: -1. **Join the stack's network** (works on Linux Docker Engine and Docker Desktop): +- joins the container to the stack's network (`querya-dev_default`), so the + service names `postgres`, `mysql`, `redis`, `mongo`, `clickhouse` (and + `mysql-live`) resolve, with the container-side ports 5432, 3306, 6379, 27017, + 8123 / 9000; +- forwards `127.0.0.1:` to those services with `socat`, so `localhost` and + the usual ports work too (`psql -h localhost -U querya querya`, + `redis-cli -h localhost ping`, the connection form with `localhost`). A port + something already listens on is left alone. - ```bash - docker network connect querya-dev_default "$(hostname)" - ``` +The forwards live as long as the container does; after `docker compose down` / +`up` re-run `bash scripts/dev/devcontainer_start.sh`. If you start the +container with host networking, `localhost` already reaches the published ports +and nothing is forwarded. - Then use the service names as hosts: `postgres`, `mysql`, `redis`, `mongo`, - `clickhouse` (with the container-side ports 5432, 3306, 6379, 27017, 8123). -2. **Docker Desktop only:** `host.docker.internal` with the host ports. -3. **Share the network namespace** of a database with the dev container, so - that `127.0.0.1` works for tests that hard-code it (see the live MySQL test - below): start that one container with `--network container:$(hostname)`. +On Docker Desktop `host.docker.internal` with the host ports works as well. ### Per database @@ -182,10 +187,11 @@ scripts/ci/test_sections.sh check # every test is in a se ([`test/e2e/README.md`](../test/e2e/README.md)). - **Goldens** are tagged `golden` and excluded in CI; run them only when you change the UI they cover. -- **Extension sandbox tests** need `bubblewrap` (`sudo apt-get install bubblewrap`) - and unprivileged user namespaces; without them they skip. Inside a container - they may also skip because nested namespaces are restricted: CI runs them on a - VM. +- **Extension sandbox tests** need `bubblewrap` (installed in the image) and + unprivileged user namespaces; without them they skip. Inside a container they + may still skip because nested namespaces are restricted (run it with + `--security-opt seccomp=unconfined` or add `"runArgs": ["--privileged"]` to + your own `devcontainer.json` if you need them locally): CI runs them on a VM. - **Analysis:** `flutter analyze`; CI runs it on every PR. ### The live MySQL test @@ -193,20 +199,20 @@ scripts/ci/test_sections.sh check # every test is in a se `test/core/database/stream_take_drain_test.dart` talks to a real MySQL on `127.0.0.1` and **skips** when there is none or when it cannot log in. It reads `MYSQL_PORT` (default 3306), `MYSQL_USER`, `MYSQL_PASSWORD` and `MYSQL_DATABASE` -(defaults `querya`) and connects **without TLS**, so the stack's MySQL 8.4 does -not work for it. Use a server like the one CI starts: +(defaults `querya`) and connects **without TLS**, so the stack's MySQL 8.4 +(`caching_sha2_password`) does not work for it. The `live-tests` compose profile +starts the server CI uses, MySQL 8.0 with `mysql_native_password`, on its own +port: ```bash -docker run -d --name mysql-live -p 3307:3306 \ - -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=querya \ - -e MYSQL_USER=querya -e MYSQL_PASSWORD=querya \ - mysql:8.0 --default-authentication-plugin=mysql_native_password - +cd docker +docker compose --profile live-tests up -d --wait mysql-live # host port 3307, MYSQL_LIVE_PORT to change +cd .. MYSQL_PORT=3307 flutter test test/core/database/stream_take_drain_test.dart ``` -Inside the dev container add `--network container:$(hostname)` (and drop -`-p`) so that `127.0.0.1:3306` is that server, and leave `MYSQL_PORT` unset. +Inside the dev container the start script forwards `localhost:3307` to it when +the service is running (re-run the script after starting the profile). ### Testing an extension @@ -223,26 +229,18 @@ a usage error. ## Known gaps -- **Flutter version.** CI pins Flutter **3.47.1**; the dev container image is - `ghcr.io/cirruslabs/flutter:3.41.6` and the image registry has no 3.47.x tag - yet. To match CI inside the container: - - ```bash - FLUTTER_ROOT="$(dirname "$(dirname "$(readlink -f "$(which flutter)")")")" - git -C "$FLUTTER_ROOT" fetch --tags - git -C "$FLUTTER_ROOT" checkout 3.47.1 - flutter precache --linux && flutter pub get - ``` - - This changes the image's SDK for that container only. +- **Headless L2** (the app under Xvfb in a script and a CI job) and the + extension-author container are not done yet; both are tracked in + [#1487](https://github.com/QueryaHub/Querya-Desktop/issues/1487). ## Troubleshooting | Symptom | Fix | |---------|-----| -| `Docker stack not started` on start | Docker is not reachable from the container: check that the host daemon runs and the `docker-outside-of-docker` feature is installed, then `cd docker && docker compose up -d`. | +| `Docker is not reachable from the container` on start | Check that the host daemon runs and the `docker-outside-of-docker` feature is installed, then `bash scripts/dev/devcontainer_start.sh`. | +| `not healthy after 180s: ...` on start | A service is still starting or failed: `docker compose -f docker/docker-compose.yml ps` and `logs `; raise `QUERYA_STACK_TIMEOUT` for a slow machine. | | `port is already allocated` | A local PostgreSQL / MySQL / Redis uses the port. Set another one in `docker/.env` (`POSTGRES_PORT`, `MYSQL_PORT`, ...). | -| Connection refused to `localhost` from inside the container | See *From inside the dev container* above: use the service names after `docker network connect`. | +| Connection refused to `localhost` from inside the container | The forwards are started by the start script: run `bash scripts/dev/devcontainer_start.sh` again (after a `docker compose down` / `up`), or use the service names (`postgres`, `mysql`, ...). | | MySQL: connection refused or *requires a secure connection* | Tick **Use SSL/TLS** in the form (MySQL 8.4 uses `caching_sha2_password`). | | MongoDB: authentication failed | The authentication database is `admin`, not `querya`. | | Seeds missing or outdated | `docker compose down -v && docker compose up -d`. | diff --git a/scripts/dev/devcontainer_start.sh b/scripts/dev/devcontainer_start.sh new file mode 100755 index 00000000..3de37616 --- /dev/null +++ b/scripts/dev/devcontainer_start.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Runs every time the dev container starts (devcontainer.json: postStartCommand). +# +# 1. starts the database stack on the host's Docker daemon and waits until the +# long-running services are healthy, with a clear status instead of hiding a +# failure; +# 2. joins this container to the stack's network, so the service names resolve; +# 3. makes `localhost:` work inside the container by forwarding the usual +# ports to the services (the databases run next to the dev container, not in +# it, so nothing listens on its loopback otherwise). +# +# It never fails the container start: a dev container without the databases is +# still useful for builds and tests. +set -u + +compose_file="docker/docker-compose.yml" +project="querya-dev" +timeout="${QUERYA_STACK_TIMEOUT:-180}" + +say() { printf '[querya-dev] %s\n' "$*"; } + +if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then + say "Docker is not reachable from the container: the database stack is not started." + say "Check that the host daemon runs, then: cd docker && docker compose up -d" + exit 0 +fi + +say "starting the database stack ..." +if ! docker compose -f "$compose_file" up -d; then + say "docker compose up failed (see the output above)." + exit 0 +fi + +# Long-running services with a healthcheck; the seed jobs exit on their own. +services=(querya-postgres querya-mysql querya-redis querya-mongo querya-clickhouse) +deadline=$((SECONDS + timeout)) +pending=("${services[@]}") +while [ "${#pending[@]}" -gt 0 ] && [ "$SECONDS" -lt "$deadline" ]; do + next=() + for name in "${pending[@]}"; do + state="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$name" 2>/dev/null || echo missing)" + case "$state" in + healthy | running) ;; + *) next+=("$name") ;; + esac + done + pending=("${next[@]}") + [ "${#pending[@]}" -gt 0 ] && sleep 3 +done + +if [ "${#pending[@]}" -gt 0 ]; then + say "not healthy after ${timeout}s: ${pending[*]} (docker compose -f $compose_file ps)" +else + say "all databases are healthy." +fi + +# Join the stack's network (a no-op when it is already joined, and not possible +# with host networking, where localhost already reaches the published ports). +docker network connect "${project}_default" "$(hostname)" >/dev/null 2>&1 || true + +# localhost: -> service:. Skipped for a port something already +# listens on, and for a service name that does not resolve. +forward() { + local port="$1" host="$2" + if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + return 0 + fi + if ! getent hosts "$host" >/dev/null 2>&1; then + return 0 + fi + command -v socat >/dev/null 2>&1 || return 0 + nohup socat "TCP-LISTEN:${port},bind=127.0.0.1,fork,reuseaddr" "TCP:${host}:${port}" \ + >/dev/null 2>&1 & +} + +forward 5432 postgres +forward 3306 mysql +forward 6379 redis +forward 27017 mongo +forward 8123 clickhouse +forward 9000 clickhouse +# Profile `live-tests`: MySQL 8.0 with mysql_native_password, its own host port. +forward "${MYSQL_LIVE_PORT:-3307}" mysql-live + +say "postgres :5432 mysql :3306 redis :6379 mongo :27017 clickhouse :8123/:9000 on localhost" +docker compose -f "$compose_file" ps --format 'table {{.Name}}\t{{.Status}}' 2>/dev/null || true +exit 0 diff --git a/test/guards/devcontainer_flutter_version_test.dart b/test/guards/devcontainer_flutter_version_test.dart new file mode 100644 index 00000000..c80bf503 --- /dev/null +++ b/test/guards/devcontainer_flutter_version_test.dart @@ -0,0 +1,28 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +/// #1487: the dev container has to build with the Flutter version CI uses, +/// otherwise "works in the container" and "passes in CI" drift apart. +void main() { + test('the dev container Dockerfile pins the Flutter version of ci.yml', () { + final dockerfile = File('.devcontainer/Dockerfile').readAsStringSync(); + final arg = RegExp(r'^ARG FLUTTER_VERSION=(\S+)$', multiLine: true) + .firstMatch(dockerfile); + expect(arg, isNotNull, reason: 'ARG FLUTTER_VERSION= not found'); + + final ci = File('.github/workflows/ci.yml').readAsStringSync(); + final versions = RegExp(r"flutter-version:\s*'([^']+)'") + .allMatches(ci) + .map((m) => m.group(1)!) + .toSet(); + + expect(versions, isNotEmpty, reason: 'no flutter-version in ci.yml'); + expect( + versions, + {arg!.group(1)}, + reason: 'ci.yml uses $versions, .devcontainer/Dockerfile pins ' + '${arg.group(1)}: change them together', + ); + }); +}