diff --git a/CLAUDE.md b/CLAUDE.md index a30c899d..5bf4d0f4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,7 +147,7 @@ dotfiles/ │ ├── mise/ # Global mise config for the no-Nix, non-admin account │ # (bootstrap.sh links it to ~/.config/mise; no module) │ └── venari/ # the rented box: twins of its /srv compose files (Soft Serve, -│ # Ghost) and /etc drop-ins; nothing deploys them +│ # Ghost, Convex) and /etc drop-ins; nothing deploys them ├── secrets/ # agenix/ragenix age-encrypted secrets ├── docker/ # container notes (per-arch CLAUDE.md) + entrypoint ├── docs/solutions/ # documented solutions to past problems - bugs, practices, diff --git a/tools/just/personal.just b/tools/just/personal.just index 99bb673b..1fa3535a 100644 --- a/tools/just/personal.just +++ b/tools/just/personal.just @@ -36,6 +36,59 @@ ghost-tunnel: ghost-tunnel-close: @pkill -f 'ssh -fN venari-ghost' && echo "ghost tunnel closed" || echo "no ghost tunnel open" +# Convex on venari (tools/venari/README.md, "Convex"): backend, HTTP actions and +# dashboard listen on the box's loopback. These recipes use the `venari` and +# `venari-root` aliases from ~/.ssh/config (README, "Access"; the box's address +# is never in this repo). The forwards are inline, so unlike Ghost they need no +# third alias. Laptop ports are the box's plus 10000 and must match +# CONVEX_CLOUD_ORIGIN / CONVEX_SITE_ORIGIN in convex/docker-compose.yml. +# The tunnel is an ssh control master on a socket, so `ssh -O check` / `-O exit` +# find and close it on macOS and Linux alike - no pgrep/pkill, whose flags differ +# (procps pgrep has no -q) and whose -f pattern also matches the calling shell. + +# open the Convex tunnel: 13210 -> API, 13211 -> HTTP actions, 16791 -> dashboard +[group('personal')] +convex-tunnel: + #!/usr/bin/env bash + set -euo pipefail + sock="$HOME/.ssh/venari-convex.sock" + if ssh -o ControlPath="$sock" -O check venari 2>/dev/null; then + echo "convex tunnel already open" + exit 0 + fi + rm -f "$sock" # a master that died uncleanly leaves its socket behind + ssh -fN -o ControlMaster=yes -o ControlPath="$sock" -o ExitOnForwardFailure=yes \ + -L 127.0.0.1:13210:127.0.0.1:3210 -L 127.0.0.1:13211:127.0.0.1:3211 -L 127.0.0.1:16791:127.0.0.1:6791 venari + echo "convex tunnel open: API 13210, HTTP actions 13211, dashboard 16791" + +# close the Convex tunnel +[group('personal')] +convex-tunnel-close: + #!/usr/bin/env bash + set -euo pipefail + sock="$HOME/.ssh/venari-convex.sock" + if ssh -o ControlPath="$sock" -O exit venari 2>/dev/null; then + echo "convex tunnel closed" + else + rm -f "$sock" + echo "no convex tunnel open" + fi + +# print a new Convex admin key (each call differs; all stay valid until data/ is lost) +[group('personal')] +convex-admin-key: + #!/usr/bin/env bash + set -euo pipefail + # Without pipefail a failed ssh or exec still exits 0 through `tail`, and the + # empty line or error text gets pasted into .env.local as the key. So the + # script also refuses anything not shaped like a key: |. + key=$(ssh venari-root 'cd /srv/convex && docker compose exec -T backend ./generate_admin_key.sh' | tail -n 1) + if [[ ! "$key" =~ ^[a-z0-9_-]+\|[0-9a-f]+$ ]]; then + echo "convex-admin-key: got no admin key from venari (output: '${key}')" >&2 + exit 1 + fi + printf '%s\n' "$key" + # The box has two vCPUs and no Go, so the server binary is cross-compiled here # and only the static binary travels; the image on the box is Alpine plus that # file (tools/venari/ghost/server/Dockerfile). diff --git a/tools/venari/README.md b/tools/venari/README.md index 989bd786..ced513fc 100644 --- a/tools/venari/README.md +++ b/tools/venari/README.md @@ -22,6 +22,7 @@ the box is the running copy. | `ssh/soft-tunnel.authorized_keys` | `/var/lib/soft-tunnel/.ssh/authorized_keys` (root-owned, 644) | | `ghost/docker-compose.yml` | `/srv/ghost/docker-compose.yml`, with `.env` (from `.env.example`), `certs/`, `data/` and `server/` beside it | | `ghost/server/Dockerfile` | `/srv/ghost/server/Dockerfile`, next to the `ghost-server` binary it packages | +| `convex/docker-compose.yml` | `/srv/convex/docker-compose.yml`, with `.env` (from `.env.example`) and `data/` beside it; its database is in Ghost's cluster | The dev-box side is `tools/mise/venari/` — the same arrangement felixia has, for the same reason: no Nix here either. @@ -191,7 +192,153 @@ Then encrypt the box's `GHOST_API_KEY` into `secrets/personal/ghost-api-key.age` Not supported, and answered with 501: billing, spaces beyond the one, members, invites, shares, API-key management (the key is configuration). `ghost logs` returns an empty page. Backups: the cluster rides the provider's box backups, -like Soft Serve's `data/`; Ghost databases are disposable by design. +like Soft Serve's `data/`. Ghost databases are disposable by design, but the +cluster also holds Convex's database, which is not (see "Convex"). + +## Convex: a self-hosted app backend + +[Convex](https://www.convex.dev) is the backend create-epoch-app is built on: +a document database, server functions (queries, mutations, actions), and live +queries that push new results to subscribed clients. `/srv/convex/` runs the +open-source backend and dashboard from +[convex/convex-backend](https://github.com/convex/convex-backend) +(`self-hosted/`), pinned to one revision for both images. + +It is not a replacement for Ghost, and Ghost is not one for it. Ghost hands out +disposable Postgres databases, so it is the *test* database. Convex is where an +app's data lives. They do share one thing: the cluster. + +- **Postgres in Ghost's cluster**, so the box runs one Postgres, not two. + Upstream's default is SQLite. Convex gets its own database + (`convex_self_hosted`), owned by its own role (`convex`), with `CONNECT` + revoked from `PUBLIC`. `tsdbadmin`, the shared role Ghost gives to agents, + cannot open it ("User does not have CONNECT privilege"), and + `ghost password` does not touch it. `ghost-server` only acts on databases + listed in its own `ghost.databases` table. Sizes, forks, pauses, deletes and + connection kills all join on that table. So `ghost list` never shows + Convex's database, and nothing Ghost does can drop it. The storage limit is + reported, not enforced (`internal/server/config.go` in the fork). +- **One network, owned by Ghost.** `ghost-postgres` and `convex-backend` both + join `venari-postgres`. Ghost's compose file creates it; Convex's declares + it external. So Ghost runs with or without Convex, and Convex needs Ghost + up first, which it needs for the database anyway. `docker compose down` in + `/srv/ghost` while Convex is attached keeps the network ("Resource is still + in use", exit 0); Convex keeps its route, and the next `up` reattaches + Postgres (checked with Docker Compose 2 on 2026-10-16). +- **Plaintext on that network.** The backend verifies Postgres TLS, and + Ghost's certificate is self-signed, so a handshake fails with + `CaUsedAsEndEntity`. The fix needs both `DO_NOT_REQUIRE_SSL=1` and + `?sslmode=disable`; either one alone still fails. The hop is container to + container and never leaves the host. The alternative, not tested, is to + re-issue Ghost's certificate from a CA the backend trusts. +- **A connection budget.** The cluster allows 25 connections, 3 reserved for + superusers (TimescaleDB derives the limit from its memory). Convex's pool is + capped at 8 (`POSTGRES_MAX_CONNECTIONS`; its default of 128 could take every + slot), and the `convex` role has `CONNECTION LIMIT 10`, so Postgres enforces + the cap whatever the knob says. That leaves about 12 for agents' Ghost + databases. Under a 500-mutation burst Convex used 7. +- **State is split across two places.** The documents are in + `convex_self_hosted` in the cluster. Files, function modules, search indexes + (`data/storage/`) and `data/credentials/` (the instance name, and the secret + that admin keys are signed with) are on disk. A backup needs both from the + same moment. Two kinds of copy give that: a whole-disk snapshot of the box, + which takes both at once (crash-consistent, like a power cut, *if* the + provider snapshots the disk atomically, which nobody has checked), and + `npx convex export`, which is consistent by construction and is the only + copy that leaves the box. Copying the two halves separately (a `pg_dump` + of the database, then `data/`) does not. + +Measured on 2026-09-22 in a sandbox, not on venari. The setup there followed +this README: Ghost with its self-signed certificate, and Convex on the same +cluster, both at the pinned images. + +- **A workload.** A project with a schema, a mutation, a query and a + `"use node"` action deployed with `npx convex deploy` (convex 1.46.0). A + `ConvexClient` subscription saw 271 pushed updates, from 1 row to 501, while + 500 concurrent mutations ran. +- **Ghost unaffected.** `ghost create`, `fork`, `delete` and `list` behaved + as before. +- **Outages.** With `/srv/ghost` brought `down` and back up, Convex never + restarted and kept its data. On a cold start with Convex first, it + crash-looped (7 restarts) until the cluster was healthy, then recovered by + itself. `restart: unless-stopped` is what gets it through a reboot, because + `depends_on` cannot reach across compose projects. +- **Peak memory.** Convex backend 132 MiB and dashboard 88 MiB, against limits + of 512m and 384m. `ghost-postgres` peaked at 213 MiB of its 768m while + serving both. Across all containers, the box's limits total about 2 GB of + its 4 GB. +- **Disk.** The images take about 1.5 GB (798 MB + 668 MB). + +None of these are load figures for this box, which has 2 vCPUs. + +All three ports listen on the box's loopback (3210 API, 3211 HTTP actions, +6791 dashboard). The tunnel's laptop ends are the same numbers plus 10000, +like Ghost's 15432, because 3210/3211 are where a local `npx convex dev` +deployment starts. With equal numbers, a project whose `.env.local` says +`127.0.0.1:3210` would deploy to whichever backend held the port. From the +laptop: + +```sh +just convex-tunnel # 13210 -> API, 13211 -> HTTP actions, 16791 -> dashboard +just convex-admin-key # prints a new admin key (via venari-root) +open http://127.0.0.1:16791 # dashboard; paste the admin key +just convex-tunnel-close +``` + +In a Convex project, put the URL and key in `.env.local`, which is gitignored +by the project and never in this repo. `npx convex dev` and `deploy` then +target venari: + +```sh +CONVEX_SELF_HOSTED_URL='http://127.0.0.1:13210' +CONVEX_SELF_HOSTED_ADMIN_KEY='' +``` + +Every call to `generate_admin_key.sh` returns a different key. All of them +stay valid, because they are signed with the instance secret in `data/`. A +key issued before a restart still worked after it. So there is no single key +to keep in agenix. Revoking a key means rotating the instance secret, which +invalidates every key. + +`CONVEX_CLOUD_ORIGIN` and `NEXT_PUBLIC_DEPLOYMENT_URL` say +`http://127.0.0.1:13210`, which is the laptop's end of the tunnel. That means +this deployment serves one person through ssh. A frontend that other people +use needs a public HTTPS origin for the API. That means a port other than 22 +in the cloud firewall and a TLS reverse proxy. It is a separate decision, +not a config change. + +Both outbound calls the images make by default are off. `DISABLE_BEACON` +turns off the anonymous usage ping. `NEXT_PUBLIC_LOAD_MONACO_INTERNALLY` +serves the dashboard's editor from the image instead of from a CDN. + +Setup, as root on the box, after copying `convex/` here to `/srv/convex/`. +Ghost must be running from this repo's `ghost/docker-compose.yml`, which +creates `venari-postgres`. A box still on the older Ghost file needs one +`docker compose up` there first; it re-creates Postgres on the new network. + +```sh +(cd /srv/ghost && docker compose up -d --wait) # creates venari-postgres if missing + +cd /srv/convex && umask 077 && mkdir -p data +printf 'CONVEX_POSTGRES_PASSWORD=%s\n' "$(openssl rand -hex 24)" > .env +. ./.env +(cd /srv/ghost && docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -U postgres -v pw="$CONVEX_POSTGRES_PASSWORD") <<'SQL' +CREATE ROLE convex LOGIN CONNECTION LIMIT 10 PASSWORD :'pw'; +CREATE DATABASE convex_self_hosted OWNER convex; +REVOKE CONNECT, TEMPORARY ON DATABASE convex_self_hosted FROM PUBLIC; +SQL +docker compose up -d --wait +docker compose logs backend | grep -m1 'Connected to Postgres' +``` + +Upgrading, from upstream `self-hosted/advanced/upgrading.md`: + +1. Run `npx convex export` from a project against this deployment, so there is + a copy to restore. +2. Bump **both** image tags to the same revision. Upstream does not guarantee + that a mismatched backend and dashboard work together. +3. Run `docker compose up -d --wait`, then watch the logs for the + `MigrationComplete` line. ## Updates and reboots @@ -209,7 +356,15 @@ original: a `*.bak-` left in `/etc/apt/apt.conf.d/` makes apt print ## Not covered yet -- Backups. `data/` (repos plus db) should be backed up as a unit, and the +- Backups. Soft Serve's `data/` (repos plus db) should be backed up as a unit, and the planned home for that is restic to R2. `ledger-age` is itself a backup (ciphertext, with the plaintext still on felixia), so losing it is recoverable. The Moment repos are not. + Ghost's cluster is no longer all disposable: `convex_self_hosted` in it, + plus Convex's `data/`, is application data. On the box it rides the + provider's backups like everything else (see "Ghost"), which is one + copy, on the same provider, whose atomicity is unchecked. Until restic + lands, `npx convex export` from the laptop is the only off-box backup and + the only one consistent by construction. Restic for Convex must not + repeat the separate-copies mistake: export, or dump and copy with the + backend stopped. diff --git a/tools/venari/convex/.env.example b/tools/venari/convex/.env.example new file mode 100644 index 00000000..fbae1ea3 --- /dev/null +++ b/tools/venari/convex/.env.example @@ -0,0 +1,4 @@ +# Copy to .env on the box (mode 600). Generated there: +# CONVEX_POSTGRES_PASSWORD=$(openssl rand -hex 24) +# The same value goes into the role in Ghost's cluster (README, "Convex"). +CONVEX_POSTGRES_PASSWORD= diff --git a/tools/venari/convex/.gitignore b/tools/venari/convex/.gitignore new file mode 100644 index 00000000..24f9f8d0 --- /dev/null +++ b/tools/venari/convex/.gitignore @@ -0,0 +1,2 @@ +.env +data/ diff --git a/tools/venari/convex/docker-compose.yml b/tools/venari/convex/docker-compose.yml new file mode 100644 index 00000000..b82ac04f --- /dev/null +++ b/tools/venari/convex/docker-compose.yml @@ -0,0 +1,77 @@ +# Convex on venari: the self-hosted backend and dashboard, from +# convex/convex-backend's self-hosted/docker/docker-compose.yml. +# +# On the box: /srv/convex/docker-compose.yml, next to an untracked .env (see +# .env.example) and data/ (never tracked). The database is in Ghost's cluster, +# so Ghost must be up first. +# +# The reasons for every choice below - the shared cluster and its isolation, +# the network, plaintext, the connection budget, the port offset, the pinned +# revision, admin keys and backups - are in tools/venari/README.md, "Convex", +# and only there. The comments here say what must stay true when you edit. +name: convex +services: + backend: + # 0cf49cb = convex/convex-backend main on 2026-09-21; the tag :latest + # pointed at on 2026-09-22. + # ghcr.io/convex/, not get-convex/: the old path went away with the org + # rename (denied, even for :latest), though upstream's compose still names it. + image: ghcr.io/convex/convex-backend:0cf49cbf8c4b7e22e631ceda9b5111f6cccbcd49 + container_name: convex-backend + restart: unless-stopped + stop_grace_period: 10s + stop_signal: SIGINT + volumes: + - ./data:/convex/data + networks: + - default + - venari-postgres + ports: + - "127.0.0.1:3210:3210" # client API, websocket sync + - "127.0.0.1:3211:3211" # HTTP actions + environment: + # The laptop's end of the tunnel, not the container's address. Must match + # the forwards in `just convex-tunnel` (box port + 10000). + CONVEX_CLOUD_ORIGIN: http://127.0.0.1:13210 + CONVEX_SITE_ORIGIN: http://127.0.0.1:13211 + # No database name: the backend appends one derived from INSTANCE_NAME + # (default convex-self-hosted -> convex_self_hosted). Plaintext on + # purpose; it takes BOTH sslmode=disable and DO_NOT_REQUIRE_SSL. + POSTGRES_URL: "postgresql://convex:${CONVEX_POSTGRES_PASSWORD:?set it in .env}@ghost-postgres:5432?sslmode=disable" + DO_NOT_REQUIRE_SSL: "1" + # Keep below the convex role's CONNECTION LIMIT (10, README setup). The + # cluster has 25 connections in total, shared with every Ghost database. + POSTGRES_MAX_CONNECTIONS: "8" + # The anonymous usage beacon (a random ID plus the backend version). + DISABLE_BEACON: "true" + DISABLE_METRICS_ENDPOINT: "true" + # Upstream's compose default: keep old document versions for 2 days. + DOCUMENT_RETENTION_DELAY: "172800" + RUST_LOG: info + healthcheck: + test: curl -f http://localhost:3210/version + interval: 5s + start_period: 10s + mem_limit: 512m + + dashboard: + image: ghcr.io/convex/convex-dashboard:0cf49cbf8c4b7e22e631ceda9b5111f6cccbcd49 + container_name: convex-dashboard + restart: unless-stopped + stop_grace_period: 10s + stop_signal: SIGINT + ports: + - "127.0.0.1:6791:6791" + environment: + # Read in the browser, so it is the laptop's tunnel end too. + NEXT_PUBLIC_DEPLOYMENT_URL: http://127.0.0.1:13210 + # Otherwise the editor panes load monaco from a CDN. + NEXT_PUBLIC_LOAD_MONACO_INTERNALLY: "true" + depends_on: + backend: + condition: service_healthy + mem_limit: 384m + +networks: + venari-postgres: + external: true diff --git a/tools/venari/ghost/docker-compose.yml b/tools/venari/ghost/docker-compose.yml index 1284f260..3bdb006f 100644 --- a/tools/venari/ghost/docker-compose.yml +++ b/tools/venari/ghost/docker-compose.yml @@ -32,6 +32,14 @@ services: - ./certs:/certs:ro ports: - "127.0.0.1:5432:5432" + # Also on venari-postgres, the network Convex reaches this cluster through + # (tools/venari/convex/docker-compose.yml). This project creates it, so + # Ghost needs no setup step for Convex's sake. `docker compose down` here + # while Convex is attached leaves it in place ("Resource is still in use", + # exit 0) and Convex keeps running; the next `up` reattaches postgres. + networks: + - default + - venari-postgres shm_size: 128m mem_limit: 768m healthcheck: @@ -62,3 +70,7 @@ services: ports: - "127.0.0.1:8787:8787" mem_limit: 128m + +networks: + venari-postgres: + name: venari-postgres # fixed, so Convex can name it; no project prefix