fix(devcontainer): reachable databases, Flutter 3.47.1, bubblewrap, live-tests profile (#1487, part 1)

.devcontainer/Dockerfile

@@ -1,12 +1,32 @@
-FROM ghcr.io/cirruslabs/flutter:3.41.6
+# Querya Desktop dev container: Flutter (pinned to the CI version) + the Linux
+# desktop toolchain + the Docker CLI for the database stack.
+FROM ubuntu:24.04
 
-# Linux desktop toolchain + libs the app links against.
+# Keep in step with `flutter-version` in .github/workflows/ci.yml
+# (test/guards/devcontainer_flutter_version_test.dart fails when they differ).
+ARG FLUTTER_VERSION=3.47.1
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    FLUTTER_HOME=/opt/flutter \
+    PUB_CACHE=/root/.pub-cache
+ENV PATH=/opt/flutter/bin:/opt/flutter/bin/cache/dart-sdk/bin:/root/.pub-cache/bin:$PATH
+
+# Linux desktop toolchain + libs the app links against; bubblewrap for the
+# extension sandbox tests; socat to forward localhost ports to the databases.
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
-      clang cmake ninja-build pkg-config libgtk-3-dev libsecret-1-dev \
-      libsqlite3-dev libglu1-mesa xvfb git curl ca-certificates \
+      build-essential clang cmake ninja-build pkg-config libgtk-3-dev \
+      libsecret-1-dev libsqlite3-dev liblzma-dev libglu1-mesa xvfb \
+      git curl unzip xz-utils zip ca-certificates bubblewrap socat \
  && rm -rf /var/lib/apt/lists/*
 
+# Flutter at the pinned tag (the cirruslabs image registry has no 3.47.x tag).
+RUN git config --system --add safe.directory /opt/flutter \
+ && git clone --depth 1 --branch "${FLUTTER_VERSION}" \
+      https://github.com/flutter/flutter.git /opt/flutter \
+ && flutter config --no-analytics --enable-linux-desktop \
+ && flutter precache --linux
+
 # Docker CLI so the dev stack (docker/docker-compose.yml) can be started from inside the container.
 COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker
 COPY --from=docker:27-cli /usr/local/libexec/docker/cli-plugins/docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose

.devcontainer/devcontainer.json

@@ -8,14 +8,16 @@
     "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
   },
   "postCreateCommand": "flutter config --enable-linux-desktop && flutter pub get",
-  "postStartCommand": "docker compose -f docker/docker-compose.yml up -d || echo 'Docker stack not started (run it manually: cd docker && docker compose up -d)'",
-  "forwardPorts": [5432, 3306, 6379, 27017, 8123, 9000],
+  "postStartCommand": "bash scripts/dev/devcontainer_start.sh",
+  "forwardPorts": [5432, 3306, 6379, 27017, 8123, 9000, 3307],
   "portsAttributes": {
     "5432": { "label": "PostgreSQL" },
     "3306": { "label": "MySQL" },
     "6379": { "label": "Redis" },
     "27017": { "label": "MongoDB" },
-    "8123": { "label": "ClickHouse HTTP" }
+    "8123": { "label": "ClickHouse HTTP" },
+    "9000": { "label": "ClickHouse native" },
+    "3307": { "label": "MySQL 8.0 (profile live-tests)" }
   },
   "customizations": {
     "vscode": {

.github/workflows/devcontainer-image.yml

@@ -40,7 +40,7 @@ jobs:
       - name: Flutter version from the Dockerfile
         id: flutter
         run: |
-          v=$(sed -n 's#^FROM ghcr.io/cirruslabs/flutter:\(.*\)$#\1#p' .devcontainer/Dockerfile | head -n1)
+          v=$(sed -n 's#^ARG FLUTTER_VERSION=\(.*\)$#\1#p' .devcontainer/Dockerfile | head -n1)
           test -n "$v"
           echo "version=$v" >> "$GITHUB_OUTPUT"
 
@@ -70,7 +70,9 @@ jobs:
       - name: Smoke test
         run: |
           set -euo pipefail
-          docker run --rm querya-devcontainer:test flutter --version
+          docker run --rm querya-devcontainer:test flutter --version | tee /dev/stderr | grep -q "Flutter ${{ steps.flutter.outputs.version }}"
+          docker run --rm querya-devcontainer:test bwrap --version
+          docker run --rm querya-devcontainer:test socat -V
           docker run --rm querya-devcontainer:test docker --version
           docker run --rm querya-devcontainer:test docker compose version
           docker run --rm querya-devcontainer:test bash -c 'clang --version && cmake --version && ninja --version && pkg-config --exists gtk+-3.0 libsecret-1'

docker/.env.example

@@ -3,6 +3,8 @@
 
 POSTGRES_PORT=5432
 MYSQL_PORT=3306
+# MySQL 8.0 + mysql_native_password for the live tests (compose profile live-tests)
+MYSQL_LIVE_PORT=3307
 REDIS_PORT=6379
 MONGO_PORT=27017
 CLICKHOUSE_HTTP_PORT=8123

docker/docker-compose.yml

@@ -16,6 +16,8 @@
 #              auth source: admin        collections: users, products, orders
 # ClickHouse  HTTP 8123 / native 9000    db querya  user querya  password querya
 #              tables: customers, products, orders, order_lines, events
+# MySQL 8.0  port 3307 (profile live-tests: docker compose --profile live-tests up -d)
+#              mysql_native_password, no TLS: for test/core/database/stream_take_drain_test.dart
 # SQLite      local file ./sqlite/data/querya.db
 #              tables: users, products, orders
 # ─────────────────────────────────────────────────────────────────────────
@@ -79,6 +81,44 @@ services:
       retries: 20
       start_period: 30s
 
+  # MySQL for the live database tests (profile `live-tests`, not started by a
+  # plain `docker compose up`): 8.0 with mysql_native_password, because the
+  # client in those tests connects without TLS and 8.4's default
+  # caching_sha2_password needs a secure connection. Same server CI starts.
+  #   docker compose --profile live-tests up -d --wait mysql-live
+  #   MYSQL_PORT=3307 flutter test test/core/database/stream_take_drain_test.dart
+  mysql-live:
+    profiles: ["live-tests"]
+    image: mysql:8.0
+    container_name: querya-mysql-live
+    restart: unless-stopped
+    environment:
+      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-querya}
+      MYSQL_DATABASE: ${MYSQL_DATABASE:-querya}
+      MYSQL_USER: ${MYSQL_USER:-querya}
+      MYSQL_PASSWORD: ${MYSQL_PASSWORD:-querya}
+    ports:
+      - "${MYSQL_LIVE_PORT:-3307}:3306"
+    command:
+      - --default-authentication-plugin=mysql_native_password
+      - --character-set-server=utf8mb4
+      - --collation-server=utf8mb4_unicode_ci
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "mysqladmin",
+          "ping",
+          "-h",
+          "127.0.0.1",
+          "-u${MYSQL_USER:-querya}",
+          "-p${MYSQL_PASSWORD:-querya}",
+        ]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 30s
+
   redis:
     image: redis:7-alpine
     container_name: querya-redis

docs/dev-containers.md

@@ -10,7 +10,8 @@ list of seeded data see the header of [`docker/docker-compose.yml`](../docker/do
 | File | What it does |
 |------|--------------|
 | [`.devcontainer/devcontainer.json`](../.devcontainer/devcontainer.json) | The dev container definition: build, features, post-create / post-start commands, forwarded ports, VS Code extensions, host requirements (4 CPUs, 8 GB). |
-| [`.devcontainer/Dockerfile`](../.devcontainer/Dockerfile) | `ghcr.io/cirruslabs/flutter` plus the Linux desktop toolchain (clang, cmake, ninja, GTK 3, libsecret, SQLite headers, Xvfb) and the Docker CLI with the Compose plugin. `DISPLAY` is `:99`. |
+| [`.devcontainer/Dockerfile`](../.devcontainer/Dockerfile) | Ubuntu 24.04 with Flutter pinned to the version CI uses (`ARG FLUTTER_VERSION`, checked against `ci.yml` by a guard test), the Linux desktop toolchain (clang, cmake, ninja, GTK 3, libsecret, SQLite headers, Xvfb), `bubblewrap` for the extension sandbox tests, `socat`, and the Docker CLI with the Compose plugin. `DISPLAY` is `:99`. |
+| [`scripts/dev/devcontainer_start.sh`](../scripts/dev/devcontainer_start.sh) | Runs on every start: brings the stack up, waits for the databases to be healthy, joins the stack's network and forwards `localhost:<port>` to the services. |
 | [`docker/docker-compose.yml`](../docker/docker-compose.yml) | The database stack (project `querya-dev`): PostgreSQL 16, MySQL 8.4, Redis 7, MongoDB 7, ClickHouse 24.8 and a seeded SQLite file. |
 
 The image is also published to GitHub Packages as
@@ -21,9 +22,11 @@ cache (`build.cacheFrom`), so the first build pulls the toolchain layers instead
 them; if the package is unreachable the build simply falls back to building from the Dockerfile.
 
 When the container is created it runs `flutter config --enable-linux-desktop`
-and `flutter pub get`; every time it starts it tries
-`docker compose -f docker/docker-compose.yml up -d` and prints a hint if Docker
-is not reachable.
+and `flutter pub get`; every time it starts it runs
+[`scripts/dev/devcontainer_start.sh`](../scripts/dev/devcontainer_start.sh): it brings the
+stack up, waits (up to `QUERYA_STACK_TIMEOUT`, 180 s) for PostgreSQL, MySQL, Redis, MongoDB
+and ClickHouse to be healthy, prints which are not, and does not fail the container start
+when Docker is unreachable.
 
 ## Open the dev container
 
@@ -83,23 +86,25 @@ the ports above.
 
 ### From inside the dev container
 
-`localhost` inside the container is the container itself, **not** the host, so
-`localhost:5432` finds nothing even though the port is "forwarded" (the forward
-goes from the container to your machine, and the databases do not listen in the
-container). Pick one:
+The databases run on the host's daemon next to the dev container, so nothing
+listens on the container's own loopback. The start script therefore does two
+things for you:
 
-1. **Join the stack's network** (works on Linux Docker Engine and Docker Desktop):
+- joins the container to the stack's network (`querya-dev_default`), so the
+  service names `postgres`, `mysql`, `redis`, `mongo`, `clickhouse` (and
+  `mysql-live`) resolve, with the container-side ports 5432, 3306, 6379, 27017,
+  8123 / 9000;
+- forwards `127.0.0.1:<port>` to those services with `socat`, so `localhost` and
+  the usual ports work too (`psql -h localhost -U querya querya`,
+  `redis-cli -h localhost ping`, the connection form with `localhost`). A port
+  something already listens on is left alone.
 
-   ```bash
-   docker network connect querya-dev_default "$(hostname)"
-   ```
+The forwards live as long as the container does; after `docker compose down` /
+`up` re-run `bash scripts/dev/devcontainer_start.sh`. If you start the
+container with host networking, `localhost` already reaches the published ports
+and nothing is forwarded.
 
-   Then use the service names as hosts: `postgres`, `mysql`, `redis`, `mongo`,
-   `clickhouse` (with the container-side ports 5432, 3306, 6379, 27017, 8123).
-2. **Docker Desktop only:** `host.docker.internal` with the host ports.
-3. **Share the network namespace** of a database with the dev container, so
-   that `127.0.0.1` works for tests that hard-code it (see the live MySQL test
-   below): start that one container with `--network container:$(hostname)`.
+On Docker Desktop `host.docker.internal` with the host ports works as well.
 
 ### Per database
 
@@ -182,31 +187,32 @@ scripts/ci/test_sections.sh check                        # every test is in a se
   ([`test/e2e/README.md`](../test/e2e/README.md)).
 - **Goldens** are tagged `golden` and excluded in CI; run them only when you
   change the UI they cover.
-- **Extension sandbox tests** need `bubblewrap` (`sudo apt-get install bubblewrap`)
-  and unprivileged user namespaces; without them they skip. Inside a container
-  they may also skip because nested namespaces are restricted: CI runs them on a
-  VM.
+- **Extension sandbox tests** need `bubblewrap` (installed in the image) and
+  unprivileged user namespaces; without them they skip. Inside a container they
+  may still skip because nested namespaces are restricted (run it with
+  `--security-opt seccomp=unconfined` or add `"runArgs": ["--privileged"]` to
+  your own `devcontainer.json` if you need them locally): CI runs them on a VM.
 - **Analysis:** `flutter analyze`; CI runs it on every PR.
 
 ### The live MySQL test
 
 `test/core/database/stream_take_drain_test.dart` talks to a real MySQL on
 `127.0.0.1` and **skips** when there is none or when it cannot log in. It reads
 `MYSQL_PORT` (default 3306), `MYSQL_USER`, `MYSQL_PASSWORD` and `MYSQL_DATABASE`
-(defaults `querya`) and connects **without TLS**, so the stack's MySQL 8.4 does
-not work for it. Use a server like the one CI starts:
+(defaults `querya`) and connects **without TLS**, so the stack's MySQL 8.4
+(`caching_sha2_password`) does not work for it. The `live-tests` compose profile
+starts the server CI uses, MySQL 8.0 with `mysql_native_password`, on its own
+port:
 
 ```bash
-docker run -d --name mysql-live -p 3307:3306 \
-  -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=querya \
-  -e MYSQL_USER=querya -e MYSQL_PASSWORD=querya \
-  mysql:8.0 --default-authentication-plugin=mysql_native_password
-
+cd docker
+docker compose --profile live-tests up -d --wait mysql-live     # host port 3307, MYSQL_LIVE_PORT to change
+cd ..
 MYSQL_PORT=3307 flutter test test/core/database/stream_take_drain_test.dart
 ```
 
-Inside the dev container add `--network container:$(hostname)` (and drop
-`-p`) so that `127.0.0.1:3306` is that server, and leave `MYSQL_PORT` unset.
+Inside the dev container the start script forwards `localhost:3307` to it when
+the service is running (re-run the script after starting the profile).
 
 ### Testing an extension
 
@@ -223,26 +229,18 @@ a usage error.
 
 ## Known gaps
 
-- **Flutter version.** CI pins Flutter **3.47.1**; the dev container image is
-  `ghcr.io/cirruslabs/flutter:3.41.6` and the image registry has no 3.47.x tag
-  yet. To match CI inside the container:
-
-  ```bash
-  FLUTTER_ROOT="$(dirname "$(dirname "$(readlink -f "$(which flutter)")")")"
-  git -C "$FLUTTER_ROOT" fetch --tags
-  git -C "$FLUTTER_ROOT" checkout 3.47.1
-  flutter precache --linux && flutter pub get
-  ```
-
-  This changes the image's SDK for that container only.
+- **Headless L2** (the app under Xvfb in a script and a CI job) and the
+  extension-author container are not done yet; both are tracked in
+  [#1487](https://github.com/QueryaHub/Querya-Desktop/issues/1487).
 
 ## Troubleshooting
 
 | Symptom | Fix |
 |---------|-----|
-| `Docker stack not started` on start | Docker is not reachable from the container: check that the host daemon runs and the `docker-outside-of-docker` feature is installed, then `cd docker && docker compose up -d`. |
+| `Docker is not reachable from the container` on start | Check that the host daemon runs and the `docker-outside-of-docker` feature is installed, then `bash scripts/dev/devcontainer_start.sh`. |
+| `not healthy after 180s: ...` on start | A service is still starting or failed: `docker compose -f docker/docker-compose.yml ps` and `logs <service>`; raise `QUERYA_STACK_TIMEOUT` for a slow machine. |
 | `port is already allocated` | A local PostgreSQL / MySQL / Redis uses the port. Set another one in `docker/.env` (`POSTGRES_PORT`, `MYSQL_PORT`, ...). |
-| Connection refused to `localhost` from inside the container | See *From inside the dev container* above: use the service names after `docker network connect`. |
+| Connection refused to `localhost` from inside the container | The forwards are started by the start script: run `bash scripts/dev/devcontainer_start.sh` again (after a `docker compose down` / `up`), or use the service names (`postgres`, `mysql`, ...). |
 | MySQL: connection refused or *requires a secure connection* | Tick **Use SSL/TLS** in the form (MySQL 8.4 uses `caching_sha2_password`). |
 | MongoDB: authentication failed | The authentication database is `admin`, not `querya`. |
 | Seeds missing or outdated | `docker compose down -v && docker compose up -d`. |

scripts/dev/devcontainer_start.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Runs every time the dev container starts (devcontainer.json: postStartCommand).
+#
+# 1. starts the database stack on the host's Docker daemon and waits until the
+#    long-running services are healthy, with a clear status instead of hiding a
+#    failure;
+# 2. joins this container to the stack's network, so the service names resolve;
+# 3. makes `localhost:<port>` work inside the container by forwarding the usual
+#    ports to the services (the databases run next to the dev container, not in
+#    it, so nothing listens on its loopback otherwise).
+#
+# It never fails the container start: a dev container without the databases is
+# still useful for builds and tests.
+set -u
+
+compose_file="docker/docker-compose.yml"
+project="querya-dev"
+timeout="${QUERYA_STACK_TIMEOUT:-180}"
+
+say() { printf '[querya-dev] %s\n' "$*"; }
+
+if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then
+  say "Docker is not reachable from the container: the database stack is not started."
+  say "Check that the host daemon runs, then: cd docker && docker compose up -d"
+  exit 0
+fi
+
+say "starting the database stack ..."
+if ! docker compose -f "$compose_file" up -d; then
+  say "docker compose up failed (see the output above)."
+  exit 0
+fi
+
+# Long-running services with a healthcheck; the seed jobs exit on their own.
+services=(querya-postgres querya-mysql querya-redis querya-mongo querya-clickhouse)
+deadline=$((SECONDS + timeout))
+pending=("${services[@]}")
+while [ "${#pending[@]}" -gt 0 ] && [ "$SECONDS" -lt "$deadline" ]; do
+  next=()
+  for name in "${pending[@]}"; do
+    state="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$name" 2>/dev/null || echo missing)"
+    case "$state" in
+      healthy | running) ;;
+      *) next+=("$name") ;;
+    esac
+  done
+  pending=("${next[@]}")
+  [ "${#pending[@]}" -gt 0 ] && sleep 3
+done
+
+if [ "${#pending[@]}" -gt 0 ]; then
+  say "not healthy after ${timeout}s: ${pending[*]} (docker compose -f $compose_file ps)"
+else
+  say "all databases are healthy."
+fi
+
+# Join the stack's network (a no-op when it is already joined, and not possible
+# with host networking, where localhost already reaches the published ports).
+docker network connect "${project}_default" "$(hostname)" >/dev/null 2>&1 || true
+
+# localhost:<port> -> service:<port>. Skipped for a port something already
+# listens on, and for a service name that does not resolve.
+forward() {
+  local port="$1" host="$2"
+  if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then
+    return 0
+  fi
+  if ! getent hosts "$host" >/dev/null 2>&1; then
+    return 0
+  fi
+  command -v socat >/dev/null 2>&1 || return 0
+  nohup socat "TCP-LISTEN:${port},bind=127.0.0.1,fork,reuseaddr" "TCP:${host}:${port}" \
+    >/dev/null 2>&1 &
+}
+
+forward 5432 postgres
+forward 3306 mysql
+forward 6379 redis
+forward 27017 mongo
+forward 8123 clickhouse
+forward 9000 clickhouse
+# Profile `live-tests`: MySQL 8.0 with mysql_native_password, its own host port.
+forward "${MYSQL_LIVE_PORT:-3307}" mysql-live
+
+say "postgres :5432  mysql :3306  redis :6379  mongo :27017  clickhouse :8123/:9000 on localhost"
+docker compose -f "$compose_file" ps --format 'table {{.Name}}\t{{.Status}}' 2>/dev/null || true
+exit 0