diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ac6b5af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +# Only Dockerfile, initdb/ and scripts/ reach the image. +.git +.github +test +Makefile +*.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..b0cf7d4 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,46 @@ +name: Build + +# The only place the image is actually built before a release: build it and +# run the smoke test, without pushing. Path-filtered to conserve Actions quota. +on: + pull_request: + paths: + - Dockerfile + - .dockerignore + - initdb/** + - scripts/** + - test/** + - .github/workflows/** + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v4 + # Builder state is removed at job end; no remote cache is exported. + with: + cache-binary: false + keep-state: false + cleanup: true + + - name: Build image + uses: docker/build-push-action@v7 + env: + # Build records are persistent Actions artifacts. + DOCKER_BUILD_RECORD_UPLOAD: 'false' + with: + context: . + platforms: linux/amd64 + load: true + tags: rawback-db:ci + + - name: Smoke test + run: test/smoke.sh rawback-db:ci diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..677fb5b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,106 @@ +name: Release + +on: + push: + branches: + - main + +permissions: + contents: write + pull-requests: write + packages: write + +jobs: + release-please: + runs-on: blacksmith-2vcpu-ubuntu-2404 + outputs: + release_created: ${{ steps.release.outputs.release_created }} + tag_name: ${{ steps.release.outputs.tag_name }} + steps: + - name: Release Please + id: release + uses: googleapis/release-please-action@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + + docker: + permissions: + contents: read + packages: write + runs-on: blacksmith-2vcpu-ubuntu-2404 + needs: release-please + if: ${{ needs.release-please.outputs.release_created }} + steps: + - name: Checkout code + uses: actions/checkout@v7 + + # The Dockerfile ARGs are the single source of truth for the + # - tag. + - name: Read image versions + id: versions + run: | + pg_major="$(sed -n 's/^ARG PG_MAJOR=//p' Dockerfile)" + postgis="$(sed -n 's/^ARG POSTGIS_VERSION=//p' Dockerfile)" + if [ -z "$pg_major" ] || [ -z "$postgis" ]; then + echo "::error::Could not read PG_MAJOR / POSTGIS_VERSION from the Dockerfile" + exit 1 + fi + echo "pg_major=$pg_major" >> "$GITHUB_OUTPUT" + echo "postgis=$postgis" >> "$GITHUB_OUTPUT" + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v4 + # Builder state is removed at job end; no remote cache is exported. + with: + cache-binary: false + keep-state: false + cleanup: true + + - name: Build image + uses: docker/build-push-action@v7 + env: + # Build records are persistent Actions artifacts. + DOCKER_BUILD_RECORD_UPLOAD: 'false' + with: + context: . + platforms: linux/amd64 + load: true + tags: rawback-db:release + + # Nothing is pushed unless the exact layers about to be published pass. + - name: Smoke test + run: test/smoke.sh rawback-db:release + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: | + ghcr.io/rawback-app/db + tags: | + type=semver,pattern={{version}},value=${{ needs.release-please.outputs.tag_name }} + type=raw,value=${{ steps.versions.outputs.pg_major }}-${{ steps.versions.outputs.postgis }} + type=raw,value=${{ steps.versions.outputs.pg_major }} + type=raw,value=latest + + # Same builder, same context: every layer comes from the cache of the + # build the smoke test ran against, so this pushes what was tested. + - name: Build and push Docker image to GHCR + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_RECORD_UPLOAD: 'false' + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 1301e01..c426ede 100644 --- a/.gitignore +++ b/.gitignore @@ -1,149 +1,8 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files +# Local data directories and environment files +data/ .env .env.* !.env.example -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist -.output - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp directory -.temp - -# Sveltekit cache directory -.svelte-kit/ - -# vitepress build output -**/.vitepress/dist - -# vitepress cache directory -**/.vitepress/cache - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# Firebase cache directory -.firebase/ - -# TernJS port file -.tern-port - -# Stores Visual Studio Code versions used for testing Visual Studio Code extensions -.vscode-test - -# pnpm -.pnpm-store - -# yarn v3 -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/sdks -!.yarn/versions - -# Vite files -vite.config.js.timestamp-* -vite.config.ts.timestamp-* -.vite/ - -# Astro generated types -.astro/ - -# Turbo Repo -.turbo +*.log +.DS_Store diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..e18ee07 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.0.0" +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6b49760 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,97 @@ +# db — agent guide + +The house PostgreSQL image for Rawback, published as `ghcr.io/rawback-app/db`. It is PostgreSQL 18 +plus PostGIS 3.6 plus PGroonga 4.0 on Debian trixie, for `linux/amd64`. It is a drop-in replacement +for `docker.io/postgis/postgis:18-3.6` (same base image, same `PGDATA`, same `template_postgis`), with +PGroonga added. + +There is no application code here: a Dockerfile, two shell scripts that ship in the image, a smoke +test, and the CI that builds and publishes it. README.md is the user-facing half (tags, quick start, +migration, upgrading). This file is the operational half. + +Consumers are `../server` (it needs `postgis` and `pg_trgm`; see "Search & geo" in its AGENTS.md) and, +later, every other Rawback service that needs Postgres. + +## Commands + +```bash +make build # docker build -t rawback-db:dev . +make test # build + test/smoke.sh rawback-db:dev +make run / make stop # local server on 127.0.0.1:5432, volume rawback-db-data +make psql # psql inside the running container +DOCKER=podman make test # every target takes DOCKER= +test/smoke.sh # smoke-test any image, e.g. one pulled from GHCR +``` + +## Layout + +| Path | What | +| --- | --- | +| `Dockerfile` | `FROM postgres:${PG_MAJOR}-trixie`, plus PostGIS from apt.postgresql.org and PGroonga from packages.groonga.org | +| `initdb/10-rawback-extensions.sh` | Copied to `/docker-entrypoint-initdb.d/`. On first start it creates `template_postgis`, then `postgis`, `pg_trgm` and `pgroonga` in it and in `$POSTGRES_DB`. | +| `scripts/update-extensions.sh` | Copied to `/usr/local/bin/`. Run after an image upgrade: `postgis_extensions_upgrade()` and `ALTER EXTENSION ... UPDATE` in every database that accepts connections. | +| `test/smoke.sh`, `test/smoke.sql` | Start the image, assert that every extension works, run the health check command and the update script, and print the versions (also to `$GITHUB_STEP_SUMMARY`). | +| `.github/workflows/build.yml` | On PR (path-filtered) and `workflow_dispatch`: build and smoke test, no push | +| `.github/workflows/release.yml` | release-please; on a release, build, smoke test, then push to GHCR | +| `release-please-config.json`, `.release-please-manifest.json`, `version.txt` | `release-type: simple`, `always-bump-patch` | + +## Invariants + +- **`PG_MAJOR` is a data-format decision, not a version bump.** A PG 19 server cannot open a PG 18 + data directory. Changing the `ARG` needs a migration plan (`pg_upgrade` or dump and restore) for + every deployment. The `18` and `18-3.6` tags must never point at another major. +- **The base must stay trixie or newer.** PGroonga dropped Debian 12 in 4.0.8. The Groonga apt + source is chosen by the codename in `/etc/os-release`, so a base change follows automatically, but + only if Groonga publishes for that codename. +- **Minor lines are pinned; patch releases float.** `POSTGIS_VERSION` and `PGROONGA_VERSION` are + asserted against `dpkg-query` after install. A new minor in apt fails the build on purpose: existing + databases need `update-extensions.sh` after a minor bump, and `POSTGIS_VERSION` is part of the + image tag. Bump the `ARG`, and don't loosen the check. +- **The release workflow reads `ARG PG_MAJOR=` and `ARG POSTGIS_VERSION=` with `sed`** to build the + `-` and `` tags. Keep those lines in the form `ARG NAME=value`, one per line. +- **The init script runs only on an empty data directory.** Anything that existing deployments also + need belongs in `update-extensions.sh` and the README's upgrade steps, not only in `initdb/`. The + script is executable, so the entrypoint runs it as a subprocess instead of sourcing it. It + therefore uses its own `psql`, not the entrypoint's `docker_process_sql`. +- **Only the extensions Rawback uses are created** (`postgis`, `pg_trgm`, `pgroonga`). Unlike + `postgis/postgis`, topology and the tiger geocoder are installed but not created. Adding an + extension to the init script means adding it to `smoke.sql`, to `update-extensions.sh`, and to the + README's "First start". +- **The health check is TCP on purpose** (`pg_isready -h 127.0.0.1`). During first start the + entrypoint runs a socket-only server, and the smoke test relies on the same distinction to wait for + the real one. +- **Release pushes what was tested.** The release job builds with `load: true`, smoke-tests that + image, then runs build-push again on the same builder, so every layer comes from that build's + cache. Don't add a step between them that changes the build context. + +## Validation + +```bash +make test +``` + +The build needs apt.postgresql.org, deb.debian.org and packages.groonga.org. A sandbox that cannot +reach them cannot build the image, and then **CI is the only build verification**. The `Build` +workflow runs on any PR that touches `Dockerfile`, `.dockerignore`, `initdb/`, `scripts/`, `test/` or +the workflows. Read its job summary for the versions it actually resolved. + +Shell scripts should pass `shellcheck` and `bash -n`. + +## Commit & PR + +Conventional Commits with a scope. `feat` and `fix` cut a release (`always-bump-patch`, so it is +always a patch bump). `chore`, `test` and `ci` never cut one, because their changelog sections are +hidden. + +``` +feat(image): add pg_cron +fix(deps): bump PGroonga to 4.1 +docs(readme): document the quadlet health check +ci(release): tag - +``` + +A rebuild that only picks up floating patch versions (a PostgreSQL minor, a PostGIS or PGroonga patch) +still needs a `fix(deps): ...` commit to produce a release. + +PRs should list which component versions change. When `PG_MAJOR` or a minor line changes, they +should also give the upgrade steps that consumers have to run. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1d7337f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +# syntax=docker/dockerfile:1 + +# PostgreSQL major. Changing it is a data migration (pg_upgrade or dump and +# restore), never an image swap: see "Upgrading" in README.md. The release +# workflow reads this line for the image tags. +ARG PG_MAJOR=18 + +# trixie is not a free choice: PGroonga dropped Debian 12 (bookworm) in 4.0.8. +FROM postgres:${PG_MAJOR}-trixie + +# The PostGIS and PGroonga minor lines this image ships. Patch releases float +# to the newest in their apt repositories at build time; a new minor fails the +# build until the line is bumped here, because existing databases then need an +# extension update (scripts/update-extensions.sh). The release workflow reads +# POSTGIS_VERSION for the image tags. +ARG POSTGIS_MAJOR=3 +ARG POSTGIS_VERSION=3.6 +ARG PGROONGA_VERSION=4.0 + +LABEL org.opencontainers.image.title="rawback-db" \ + org.opencontainers.image.description="PostgreSQL ${PG_MAJOR} with PostGIS ${POSTGIS_VERSION} and PGroonga ${PGROONGA_VERSION}" \ + org.opencontainers.image.source="https://github.com/rawback-app/db" \ + org.opencontainers.image.licenses="MIT" + +# PG_MAJOR comes from the base image's ENV. PostGIS is on apt.postgresql.org, +# which the base image already configures; PGroonga comes from Groonga's own +# repository, whose postgresql-*-pgdg-pgroonga packages are built against it. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl; \ + codename="$(. /etc/os-release && echo "$VERSION_CODENAME")"; \ + curl -fsSL --retry 3 -o /tmp/groonga-apt-source.deb \ + "https://packages.groonga.org/debian/groonga-apt-source-latest-${codename}.deb"; \ + apt-get install -y --no-install-recommends /tmp/groonga-apt-source.deb; \ + rm /tmp/groonga-apt-source.deb; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + "postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR}" \ + "postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR}-scripts" \ + "postgresql-${PG_MAJOR}-pgdg-pgroonga"; \ + postgis="$(dpkg-query -W -f '${Version}' "postgresql-${PG_MAJOR}-postgis-${POSTGIS_MAJOR}")"; \ + case "$postgis" in \ + "${POSTGIS_VERSION}".*) ;; \ + *) echo "expected PostGIS ${POSTGIS_VERSION}.x, apt resolved ${postgis}" >&2; exit 1 ;; \ + esac; \ + pgroonga="$(dpkg-query -W -f '${Version}' "postgresql-${PG_MAJOR}-pgdg-pgroonga")"; \ + case "$pgroonga" in \ + "${PGROONGA_VERSION}".*) ;; \ + *) echo "expected PGroonga ${PGROONGA_VERSION}.x, apt resolved ${pgroonga}" >&2; exit 1 ;; \ + esac; \ + apt-get purge -y --auto-remove curl; \ + rm -rf /var/lib/apt/lists/* + +COPY --chmod=755 initdb/ /docker-entrypoint-initdb.d/ +COPY --chmod=755 scripts/update-extensions.sh /usr/local/bin/update-extensions.sh + +# TCP on purpose: during first-start initialisation the entrypoint runs a +# socket-only server, so this stays unhealthy until the real one is up. +HEALTHCHECK --interval=10s --timeout=5s --start-period=60s --retries=5 \ + CMD ["pg_isready", "-q", "-h", "127.0.0.1"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7e4695a --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +# Local build and smoke test. DOCKER=podman works for every target. +DOCKER ?= docker +IMAGE ?= rawback-db:dev +NAME ?= rawback-db +PORT ?= 5432 +PASSWORD ?= postgres + +.PHONY: build test run stop psql + +build: + $(DOCKER) build -t $(IMAGE) . + +test: build + DOCKER=$(DOCKER) test/smoke.sh $(IMAGE) + +run: build + $(DOCKER) run -d --name $(NAME) --shm-size=256m \ + -e POSTGRES_PASSWORD=$(PASSWORD) \ + -p 127.0.0.1:$(PORT):5432 \ + -v $(NAME)-data:/var/lib/postgresql \ + $(IMAGE) + +stop: + $(DOCKER) rm -f $(NAME) + +psql: + $(DOCKER) exec -it $(NAME) psql -U postgres diff --git a/README.md b/README.md index 5dd2402..cbdd521 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,214 @@ -# db -Postgres + PostGIS + PGroonga +# Rawback DB + +The PostgreSQL image that every Rawback service runs on. It is PostgreSQL 18 with PostGIS 3.6 and +PGroonga 4.0 on Debian trixie, published as `ghcr.io/rawback-app/db`. + +It is a drop-in replacement for `docker.io/postgis/postgis:18-3.6`. It shares that image's base image, +data directory layout and `template_postgis`, and adds [PGroonga](https://pgroonga.github.io/) for +full-text search in Japanese, Chinese, Korean and every other language. + +## What's inside + +| Component | Version | Source | +| --- | --- | --- | +| PostgreSQL | 18, newest 18.x at build time | the official [`postgres:18-trixie`](https://hub.docker.com/_/postgres) image | +| PostGIS | 3.6.x (raster, topology and SFCGAL libraries included) | apt.postgresql.org, `postgresql-18-postgis-3` | +| PGroonga | 4.0.x | packages.groonga.org, `postgresql-18-pgdg-pgroonga` | +| pg_trgm | bundled with PostgreSQL | contrib | + +The Dockerfile pins the minor versions and lets patch releases float, so every build takes the newest +18.x, 3.6.x and 4.0.x. If apt resolves a different PostGIS or PGroonga minor, the build fails rather +than ship it. Each release job's summary lists the exact versions that went into it. + +The image is built for `linux/amd64` only. Apple Silicon runs it under emulation. + +## Tags + +| Tag | Example | Changes on | Use it for | +| --- | --- | --- | --- | +| `-` | `18-3.6` | every release on that pair | Apps and CI. It pins the same things `postgis/postgis:18-3.6` does. | +| `` | `18` | every release on PostgreSQL 18 | Anything that should only follow the data directory's major version. | +| `` | `0.0.1` | never | Pinning production. | +| `latest` | | every release | Throwaway local use only. It will cross a PostgreSQL major one day. | + +Neither `18` nor `18-3.6` ever moves to another PostgreSQL major, so neither can produce a data +directory that the next container cannot open. + +## Quick start + +```bash +docker run -d --name rawback-db --shm-size=256m \ + -e POSTGRES_PASSWORD=change-me \ + -e POSTGRES_DB=rawback \ + -p 127.0.0.1:5432:5432 \ + -v rawback-db:/var/lib/postgresql \ + ghcr.io/rawback-app/db:18-3.6 + +psql "postgres://postgres:change-me@127.0.0.1:5432/rawback?sslmode=disable" +``` + +The volume goes on `/var/lib/postgresql`, which is where PostgreSQL 18 images expect it. `PGDATA` is +`/var/lib/postgresql/18/docker`. `--shm-size` gives parallel queries more than Docker's default 64 MB +of shared memory. + +As a Podman quadlet (`/etc/containers/systemd/rawback-db.container`): + +```ini +[Unit] +Description=Rawback PostgreSQL + +[Container] +ContainerName=rawback-db +Image=ghcr.io/rawback-app/db:18-3.6 +Environment=POSTGRES_DB=rawback +# podman secret create rawback-db-password - +Secret=rawback-db-password,type=env,target=POSTGRES_PASSWORD +Volume=rawback-db:/var/lib/postgresql +PublishPort=127.0.0.1:5432:5432 +PodmanArgs=--shm-size=256m + +[Service] +Restart=always + +[Install] +WantedBy=multi-user.target +``` + +The image ships a `HEALTHCHECK` (`pg_isready -h 127.0.0.1`). It turns healthy only once the real +server is listening on TCP, not while the first-start initialisation is still running. + +Every variable and behaviour of the official `postgres` image applies unchanged, including +`POSTGRES_USER`, `POSTGRES_INITDB_ARGS` and extra scripts in `/docker-entrypoint-initdb.d`. Name your +own scripts so that they sort after `10-`, and the extensions will already exist when they run. + +## First start + +On an empty volume, `initdb/10-rawback-extensions.sh` does two things: + +- It creates `template_postgis`, a template database, as `postgis/postgis` does. +- It creates `postgis`, `pg_trgm` and `pgroonga` in `template_postgis` and in `POSTGRES_DB`. + +Application roles that are not superusers can therefore use these extensions without the privilege +to create them. `CREATE DATABASE app TEMPLATE template_postgis` starts a new database with the same +set. + +There are two differences from `postgis/postgis`. This image does not create `postgis_topology`, +`fuzzystrmatch` or `postgis_tiger_geocoder`; their libraries are installed, so `CREATE EXTENSION` +them if you need them. It does create `pgroonga`. + +The script never runs against an existing volume, because the postgres entrypoint skips +`/docker-entrypoint-initdb.d` once a data directory exists. On an existing volume, create the +extensions yourself. + +## Using PGroonga + +```sql +CREATE INDEX images_title_pgroonga ON images USING pgroonga (title); + +SELECT id, title FROM images WHERE title &@~ '夜景 OR night'; +``` + +- **`&@~` takes Groonga query syntax.** Whitespace means AND, `OR` means OR, `-` excludes + (`tokyo - tower`), and quotes mark a phrase. `&@` matches a single keyword. +- **The defaults are `TokenBigram` and `NormalizerAuto`.** CJK text is indexed as bigrams, runs of + Latin letters as words, and matching ignores case and character width. No dictionary is needed. +- **MeCab is not bundled.** If you need morphological tokenisation for Japanese, extend the image with + `groonga-tokenizer-mecab`. +- See the [PGroonga reference](https://pgroonga.github.io/reference/) for operators, scoring and + index options. + +## Migrating from `postgis/postgis:18-3.6` + +Both images are built `FROM postgres:18-trixie`. The PostgreSQL major, the glibc and ICU collation +versions and the data directory layout are all the same, so you swap the image on the existing +volume. There is no dump and restore. + +1. Back up: run `pg_dumpall`, or snapshot the volume. +2. Stop the old container. Start this image with the same volume, mounted at the same path, and the + same environment. +3. Run `docker exec -u postgres update-extensions.sh`. This brings PostGIS's SQL objects + up to the library if the patch versions differ. +4. In each database that wants it, run `CREATE EXTENSION pgroonga;` as a superuser. The first-start + script does not run on an existing volume. + +## Upgrading + +- **A new release of this image** brings a PostgreSQL minor, PostGIS and PGroonga patches, or a + rebuilt base. Pull it, recreate the container on the same volume, then run + `docker exec -u postgres update-extensions.sh`. The script is idempotent and updates + `postgis` (and the rest of its family), `pg_trgm` and `pgroonga` in every database that has them. +- **A PostGIS minor bump** (for example 3.7) gets a new `18-3.7` tag. Follow the same steps once you + move to that tag. +- **A PostgreSQL major** (19) is not an image swap. The data directory has to be migrated with + `pg_upgrade`, or by dump and restore into a fresh volume. This repo will publish `19-*` tags + alongside the `18-*` ones. + +## Development + +| Command | What it does | +| --- | --- | +| `make build` | Build `rawback-db:dev` | +| `make test` | Build, then run `test/smoke.sh` against a throwaway container | +| `make run` / `make stop` | Run a local server on `127.0.0.1:5432` with a `rawback-db-data` volume, or remove it | +| `make psql` | Open `psql` in the running container | + +Every target accepts `DOCKER=podman`. + +`test/smoke.sh ` starts the image on an empty volume and runs `test/smoke.sql`. That file +checks that the extensions and `template_postgis` exist, that `ST_DWithin` works on geography, that +the `<%` trigram operator works, and that PGroonga searches Japanese, Chinese, Korean and English +through its index. The script then runs the health check command and `update-extensions.sh`, and +prints the bundled versions. + +### Bumping versions + +Change the `ARG` lines at the top of the `Dockerfile`: + +| ARG | Meaning | +| --- | --- | +| `PG_MAJOR` | PostgreSQL major and the base image tag. Changing it needs a data migration (see Upgrading) and changes the `` tags. | +| `POSTGIS_VERSION` | PostGIS minor line, which is part of the `-` tag | +| `PGROONGA_VERSION` | PGroonga minor line | + +The release workflow reads `PG_MAJOR` and `POSTGIS_VERSION` from those lines to build the tags. + +## Releases + +[release-please](https://github.com/googleapis/release-please) runs on every push to `main`, and +commits follow Conventional Commits. Merging its release PR does three things. It builds the image, +runs the smoke test against that exact build, and only then pushes ``, `-`, +`` and `latest` to `ghcr.io/rawback-app/db`. + +A PostgreSQL minor or a patch release of PostGIS or PGroonga needs no code change, but it only +reaches the published image with a new release. Merge any `fix(deps): ...` commit to get one. + +Pull requests that touch the image are built and smoke-tested by `.github/workflows/build.yml`, +without pushing. + +### Pulling from GHCR + +If the package is private: + +- **Hosts** need `podman login ghcr.io` (or `docker login`) with a token that has `read:packages`. +- **Another repository's workflow** needs read access granted under the package's settings (*Manage + Actions access*). It also needs registry credentials on its service container: + + ```yaml + services: + postgres: + image: ghcr.io/rawback-app/db:18-3.6 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + ``` + +## Related repos + +- [rawback-app/server](https://github.com/rawback-app/server) is the Go backend and the first + consumer. It needs `postgis` and `pg_trgm`. +- [rawback-app/web](https://github.com/rawback-app/web) is the Next.js web client. +- [rawback-app/ios](https://github.com/rawback-app/ios) is the iOS, iPadOS and Mac Catalyst client. + +## License + +MIT diff --git a/initdb/10-rawback-extensions.sh b/initdb/10-rawback-extensions.sh new file mode 100755 index 0000000..513ab31 --- /dev/null +++ b/initdb/10-rawback-extensions.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# First start only: the postgres entrypoint runs /docker-entrypoint-initdb.d +# when the data directory is empty and never against an existing one. +# +# Creates template_postgis, as the postgis/postgis image does, and the +# extensions this image exists for in it and in $POSTGRES_DB. Databases created +# with `TEMPLATE template_postgis` then start with the same set, and roles that +# are not superusers can use them without being able to create them. +set -Eeuo pipefail + +# The server is socket-only at this point; never follow a PGHOST from the +# container's environment (the entrypoint's own docker_process_sql does the same). +unset PGHOST PGHOSTADDR +psql=(psql -v ON_ERROR_STOP=1 --no-psqlrc --no-password --username "$POSTGRES_USER") + +"${psql[@]}" --dbname postgres <<'SQL' +CREATE DATABASE template_postgis IS_TEMPLATE true; +SQL + +for db in template_postgis "$POSTGRES_DB"; do + echo "rawback-db: creating postgis, pg_trgm and pgroonga in $db" + "${psql[@]}" --dbname "$db" <<'SQL' +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE EXTENSION IF NOT EXISTS pgroonga; +SQL +done diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..db18dd7 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "include-component-in-tag": false, + "packages": { + ".": { + "release-type": "simple", + "versioning": "always-bump-patch", + "changelog-sections": [ + { "type": "feat", "section": "Features" }, + { "type": "fix", "section": "Bug Fixes" }, + { "type": "perf", "section": "Performance Improvements" }, + { "type": "refactor", "section": "Code Refactoring" }, + { "type": "docs", "section": "Documentation" }, + { "type": "build", "section": "Build System" }, + { "type": "chore", "section": "Miscellaneous Chores", "hidden": true }, + { "type": "test", "section": "Tests", "hidden": true }, + { "type": "ci", "section": "Continuous Integration", "hidden": true } + ] + } + } +} diff --git a/scripts/update-extensions.sh b/scripts/update-extensions.sh new file mode 100755 index 0000000..07546ef --- /dev/null +++ b/scripts/update-extensions.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Brings the SQL side of the bundled extensions up to the libraries in this +# image, in every database that accepts connections. Run it after starting a +# newer image on an existing data directory: +# +# docker exec -u postgres update-extensions.sh +# +# Idempotent: an extension that is already current is left as it is, and a +# database without one of them is skipped. +set -Eeuo pipefail + +psql=(psql -v ON_ERROR_STOP=1 --no-psqlrc --no-password --username "${POSTGRES_USER:-postgres}") + +databases="$("${psql[@]}" --dbname template1 -AtX \ + -c 'SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname')" + +while IFS= read -r db; do + [ -n "$db" ] || continue + echo "rawback-db: updating extensions in $db" + "${psql[@]}" --dbname "$db" <<'SQL' +DO $$ +BEGIN + IF EXISTS (SELECT FROM pg_extension WHERE extname = 'postgis') THEN + -- Also updates postgis_raster, postgis_topology and the rest of the family. + PERFORM postgis_extensions_upgrade(); + END IF; + IF EXISTS (SELECT FROM pg_extension WHERE extname = 'pg_trgm') THEN + ALTER EXTENSION pg_trgm UPDATE; + END IF; + IF EXISTS (SELECT FROM pg_extension WHERE extname = 'pgroonga') THEN + ALTER EXTENSION pgroonga UPDATE; + END IF; +END +$$; +SQL +done <<<"$databases" diff --git a/test/smoke.sh b/test/smoke.sh new file mode 100755 index 0000000..4912955 --- /dev/null +++ b/test/smoke.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Starts the image on an empty volume, waits for the real server, then runs +# smoke.sql, the image's health check command and update-extensions.sh against +# it. Prints the bundled versions, and adds them to the job summary in CI. +# +# test/smoke.sh # DOCKER=podman test/smoke.sh +set -Eeuo pipefail + +image="${1:?usage: test/smoke.sh }" +docker="${DOCKER:-docker}" +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +name="rawback-db-smoke-$$" +password=smoke + +cleanup() { + local status=$? + if [ "$status" -ne 0 ]; then + echo "smoke test failed; container logs:" >&2 + "$docker" logs "$name" >&2 || true + fi + "$docker" rm -f "$name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +"$docker" run -d --name "$name" \ + -e POSTGRES_PASSWORD="$password" \ + -e POSTGRES_DB=rawback_smoke \ + "$image" >/dev/null + +# Over TCP on purpose: the entrypoint runs the init scripts against a +# socket-only server and restarts it, so only the final server answers here. +psql=("$docker" exec -i -e PGPASSWORD="$password" "$name" + psql -h 127.0.0.1 -U postgres -d rawback_smoke -X -v ON_ERROR_STOP=1) + +for _ in $(seq 1 90); do + if "${psql[@]}" -qAt -c 'SELECT 1' >/dev/null 2>&1; then + ready=1 + break + fi + if [ "$("$docker" inspect -f '{{.State.Running}}' "$name")" != true ]; then + echo "container exited during startup" >&2 + exit 1 + fi + sleep 1 +done +if [ -z "${ready:-}" ]; then + echo "server did not accept TCP connections within 90s" >&2 + exit 1 +fi + +"${psql[@]}" -q <"$here/smoke.sql" +"$docker" exec "$name" pg_isready -q -h 127.0.0.1 +"$docker" exec -u postgres "$name" update-extensions.sh + +versions="$("${psql[@]}" -qAt -F ' | ' <<'SQL' +SELECT 'PostgreSQL', current_setting('server_version'); +SELECT 'PostGIS', postgis_lib_version(); +SELECT 'PGroonga', extversion FROM pg_extension WHERE extname = 'pgroonga'; +SELECT 'pg_trgm', extversion FROM pg_extension WHERE extname = 'pg_trgm'; +SQL +)" + +echo "smoke test passed for $image" +echo "$versions" + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### \`$image\`" + echo + echo "| Component | Version |" + echo "| --- | --- |" + while IFS= read -r line; do + echo "| $line |" + done <<<"$versions" + } >>"$GITHUB_STEP_SUMMARY" +fi diff --git a/test/smoke.sql b/test/smoke.sql new file mode 100644 index 0000000..6b8a525 --- /dev/null +++ b/test/smoke.sql @@ -0,0 +1,86 @@ +-- Smoke test, run by test/smoke.sh against a fresh container with +-- ON_ERROR_STOP: each extension is installed where the init script puts it +-- and does the job Rawback uses it for. A failed ASSERT aborts the run. + +-- 1. First start created the extensions in POSTGRES_DB and made +-- template_postgis a template. +DO $$ +DECLARE + ext text; +BEGIN + FOREACH ext IN ARRAY ARRAY['postgis', 'pg_trgm', 'pgroonga'] LOOP + ASSERT EXISTS (SELECT FROM pg_extension WHERE extname = ext), + format('extension %s is missing from %s', ext, current_database()); + END LOOP; + ASSERT EXISTS (SELECT FROM pg_database WHERE datname = 'template_postgis' AND datistemplate), + 'template_postgis is missing or not a template'; +END +$$; + +-- 2. PostGIS: geography distance, the way images.location is queried. +DO $$ +DECLARE + tokyo_tower geography := ST_SetSRID(ST_MakePoint(139.7454, 35.6586), 4326)::geography; + zojoji geography := ST_SetSRID(ST_MakePoint(139.7480, 35.6575), 4326)::geography; + kyoto geography := ST_SetSRID(ST_MakePoint(135.7681, 34.9858), 4326)::geography; +BEGIN + ASSERT ST_DWithin(tokyo_tower, zojoji, 1000), 'ST_DWithin: points ~300 m apart are not within 1 km'; + ASSERT NOT ST_DWithin(tokyo_tower, kyoto, 1000), 'ST_DWithin: Tokyo and Kyoto are within 1 km'; +END +$$; + +-- 3. pg_trgm: the word-similarity operator library search uses, at the +-- default pg_trgm.word_similarity_threshold. +DO $$ +BEGIN + ASSERT 'sunsett' <% 'sunset over lisbon', 'pg_trgm: <% missed a one-letter typo'; + ASSERT NOT ('kyoto' <% 'sunset over lisbon'), 'pg_trgm: <% matched an unrelated word'; +END +$$; + +-- 4. PGroonga: full-text search over mixed-script text through the index. +CREATE TABLE smoke_captions ( + id int PRIMARY KEY, + caption text NOT NULL +); +INSERT INTO smoke_captions VALUES + (1, '東京タワーの夜景'), + (2, '京都の紅葉と金閣寺'), + (3, '上海外滩的夜色'), + (4, '서울 남산타워 야경'), + (5, 'Sunset over Lisbon'); +CREATE INDEX smoke_captions_pgroonga ON smoke_captions USING pgroonga (caption); +SET enable_seqscan = off; + +DO $$ +BEGIN + ASSERT (SELECT array_agg(id ORDER BY id) FROM smoke_captions WHERE caption &@~ '夜景') = ARRAY[1], + 'pgroonga: Japanese query'; + ASSERT (SELECT array_agg(id ORDER BY id) FROM smoke_captions WHERE caption &@~ '夜色') = ARRAY[3], + 'pgroonga: Chinese query'; + ASSERT (SELECT array_agg(id ORDER BY id) FROM smoke_captions WHERE caption &@~ '야경') = ARRAY[4], + 'pgroonga: Korean query'; + ASSERT (SELECT array_agg(id ORDER BY id) FROM smoke_captions WHERE caption &@~ 'lisbon') = ARRAY[5], + 'pgroonga: case-insensitive English query'; + ASSERT (SELECT array_agg(id ORDER BY id) FROM smoke_captions WHERE caption &@~ '京都 OR 上海') = ARRAY[2, 3], + 'pgroonga: OR query'; +END +$$; + +RESET enable_seqscan; +DROP TABLE smoke_captions; + +-- 5. A database created from template_postgis starts with the extensions. +CREATE DATABASE smoke_from_template TEMPLATE template_postgis; +\connect smoke_from_template + +DO $$ +DECLARE + ext text; +BEGIN + FOREACH ext IN ARRAY ARRAY['postgis', 'pg_trgm', 'pgroonga'] LOOP + ASSERT EXISTS (SELECT FROM pg_extension WHERE extname = ext), + format('extension %s is missing from a database created from template_postgis', ext); + END LOOP; +END +$$; diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..77d6f4c --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.0.0