feat(database)!: replace postgres with sqlite

.env.example

@@ -1,15 +1,18 @@
 ENV="staging"
 BOT_TOKEN="your-bot-token"
 GUILD_ID= #"123456789012345678" - uncomment to register commands to a single guild for faster dev/test updates
-DATABASE_URL= #"postgres://pen_bot:pen_bot_password@postgres:5432/pen_bot?sslmode=disable"
-DB_NAME="pen_bot"
-DB_HOST="localhost"
-DB_PORT="5432"
-DB_USER="pen_bot"
-DB_PASSWORD="pen_bot_password"
+
+## DATABASE
+# Where the SQLite database file lives. The parent directory is created if it
+# does not exist. The container image sets this to /data/pen-bot.db.
+DB_PATH="/data/pen-bot.db"
+# Distinguishes instances that share one directory, so that two of them can
+# run side by side without seeing each other's data. Each instance gets its
+# own file. Ignored when DB_PATH is set.
 DB_BOT_INSTANCE_ID="pen-bot-1"
-DB_MAX_OPEN_CONNS="16"
+# Size of the connection pool. SQLite serialises writes, so a small pool is
+# enough.
+DB_MAX_OPEN_CONNS="4"
 DB_MAX_IDLE_CONNS="4"
 DB_CONN_MAX_LIFETIME="30m"
 DB_CACHE_CLEANUP_INTERVAL="15m"
-POSTGRES_PASSWORD="pen_bot_password"

.gitignore

@@ -36,6 +36,9 @@ Editor/IDE
 build/
 /custom-gcl
 
+# Local SQLite database, written by a bot run outside the container
+/data/
+
 # Sphinx HTML build output, per the command in docs/conf.py and docs.yml
 /public/
 

AGENTS.md

@@ -37,9 +37,10 @@ uv run sphinx-build -W -b html docs public
   NilAway from `.custom-gcl.yml`; `nix develop` builds it on shell entry. Re-run
   `golangci-lint custom` after editing `.custom-gcl.yml` or the pinned version.
 - **Focused runs:** `go test ./internal/db/...`, `go test -run TestExpiry
-  ./internal/db/`. Tests are pure unit tests — no Postgres, Discord, or network
-  needed. Only `internal/db` has tests; everything else reports "no test files".
-  Unexported-helper tests live in `package db`, public-API tests in
+  ./internal/db/`. Tests are pure unit tests — no Discord or network needed, and
+  the database tests need no server either: they open a SQLite file in a
+  temporary directory. Only `internal/db` has tests; everything else reports "no
+  test files". Unexported-helper tests live in `package db`, public-API tests in
   `package db_test`.
 - `pre-commit run --all-files` is a superset of `ci.yml` (adds staticcheck,
   gosec, govulncheck). There is no hook for the docs build, so run that by hand.
@@ -88,17 +89,52 @@ One root module, `github.com/Neon-Genesis-Linux/pen-bot`, one binary:
 
 ## Database
 
+- The database is SQLite, through `modernc.org/sqlite`. The driver is pure Go, so
+  the build stays `CGO_ENABLED=0` and the distroless image is unchanged. It
+  depends on `modernc.org/libc`, and upstream asks that the two stay in step:
+  Go's module resolution will not select a `libc` older than the one
+  `modernc.org/sqlite` requires, but a newer one can be selected, and an
+  incompatible one fails to compile rather than misbehaving.
+- Timestamps are **Unix milliseconds in an `INTEGER` column**, declared
+  `DATETIME`. Do not store them as text: the driver writes text while a column
+  default writes an integer, and SQLite then orders the column by kind rather
+  than by instant, so rows come back in the wrong order. The DSN parameters
+  `_time_integer_format=unix_milli` and `_inttotime=1` are what make the
+  driver read and write that representation.
+- The driver converts a timestamp on the way out **only when the column's
+  declared type is `DATE`, `DATETIME` or `TIMESTAMP`**. An aggregate or an
+  arithmetic expression over such a column has no declared type, so it comes
+  back as an integer and scanning it into a `time.Time` fails. Select the
+  column itself, or convert the integer in SQL.
+- The driver reads an integer back as milliseconds only when it is larger than
+  1e12, which is September 2001. A timestamp before that is silently read as
+  seconds. Every timestamp here is written by the running bot, so this cannot
+  happen; a column holding a historic timestamp would need care.
 - Migrations are `//go:embed`ed from `internal/db/migrations/` and applied by the
   running bot at startup (`db.ApplyMigrations`). There is no migration CLI. Add
   `NNN_name.sql`; the filename is recorded in `schema_migrations` and the file
   runs in one transaction.
-- The runner splits each file on `;`, so no semicolon may appear inside a string
-  literal, a dollar-quoted body, or a `DO $$` block. Keep migrations to plain
-  DDL.
+- A migration file is handed to the driver as one statement, not split on
+  semicolons, so a semicolon inside a string literal is safe. Keep migrations
+  to plain DDL anyway.
+- Primary keys are `INTEGER PRIMARY KEY AUTOINCREMENT`. The `AUTOINCREMENT` is
+  required: without it SQLite reuses an id after a delete, which breaks the
+  keyset pagination that walks a log by `(created_at, id)`.
+- An index that serves a query has to carry the order the query asks for. The
+  moderation log index ends with `id` because the pagination cursor compares
+  the pair `(created_at, id)`: without it SQLite sorts every page again with a
+  temporary B-tree over the whole table. `internal/db/migrations_test.go` asserts
+  the query plan, so a change that reintroduces the sort fails the build.
 - `db.GlobalDB()` is filled asynchronously, after `Start` has already returned to
-  its signal wait (`internal/core/start.go`), and stays nil forever if Postgres is
-  unreachable — the bot runs happily without a database. Callers must tolerate
-  nil rather than assume a live handle.
+  its signal wait (`internal/core/start.go`), and stays nil forever if the
+  database cannot be opened — the bot runs happily without a database. Callers
+  must tolerate nil rather than assume a live handle.
+- SQLite has one writer. The DSN sets `journal_mode(WAL)`, a `busy_timeout` and
+  `_txlock=immediate`, and the pool is small. Do not raise the pool size to
+  raise write throughput: writes are serialised regardless.
+- `_txlock=immediate` is load-bearing, not a default. A deferred transaction
+  that reads and then writes cannot always take the write lock afterwards, and
+  fails with `SQLITE_BUSY` in a way the busy timeout does not retry.
 - The cache in `internal/db/cache.go` is a process-local map with TTL, not Redis.
 
 ## Configuration
@@ -110,9 +146,9 @@ One root module, `github.com/Neon-Genesis-Linux/pen-bot`, one binary:
 - `ENV=production` selects INFO, anything else DEBUG (`internal/logger`).
 - `GUILD_ID` is optional: set it to sync commands to one guild (instant) instead
   of globally (slow to propagate).
-- For the database, `DATABASE_URL` wins outright; otherwise `DB_HOST`, `DB_PORT`,
-  `DB_USER`, `DB_PASSWORD`, `DB_NAME` are composed, and `DB_BOT_INSTANCE_ID`
-  renames the database to `pen_bot_<id>`. `POSTGRES_PASSWORD` is compose-only.
+- `DB_PATH` names the database file. `DB_BOT_INSTANCE_ID` names the file when
+  `DB_PATH` is unset, so that instances sharing a directory each get their own.
+  The container image sets `DB_PATH=/data/pen-bot.db` and mounts a volume there.
 - The compose file is `compose.yaml` (Compose v2); use `docker compose up`, or
   `docker compose watch` to honour the `develop.watch` rebuild. Keep
   `.dockerignore` — it is what keeps `.env` and the 52MB `custom-gcl` out of the

DOCUMENTATION.md

@@ -210,8 +210,8 @@ func main() {
 :::
 
 :::{note}
-The bot retries its database connection in the background, so it starts and
-stays up even when Postgres is not yet reachable.
+The bot opens its database in the background, so it starts and stays up even
+when the database file cannot be reached.
 :::
 ````
 

Dockerfile

@@ -13,10 +13,20 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
     go build -trimpath -ldflags="-s -w" -o /output/pen-bot ./cmd/pen-fun
 
+# The database lives on a volume mounted here. The directory is created in
+# this stage so that it can be given to the unprivileged user the runtime
+# image runs as: Docker seeds a fresh named volume with the ownership of the
+# directory it covers, so a volume created from this image is writable by that
+# user without the operator having to fix its permissions by hand.
+RUN mkdir -p /data && chown 65532:65532 /data
+
 FROM gcr.io/distroless/static-debian13:nonroot
 
 WORKDIR /app
 
 COPY --from=build /output/pen-bot /app/pen-bot
+COPY --from=build --chown=65532:65532 /data /data
+
+ENV DB_PATH=/data/pen-bot.db
 
 CMD ["/app/pen-bot"]

README.md

@@ -73,6 +73,11 @@ go build -o pen-fun ./cmd/pen-fun
 
 The `compose.yaml` file handles the build and deployment process with hot-reload for development.
 
+The bot stores its data in a SQLite file, so there is no database server to run
+and no credentials to configure. The compose file mounts a volume that holds
+the file. See [the database documentation](docs/OPERATIONS/DATABASE.md) for
+where the file lives, how to back it up and how to inspect it.
+
 ## Contributing
 
 Contributions are welcome! Please follow the established code style and include tests for new features.

compose.yaml

@@ -4,27 +4,12 @@ services:
     pull_policy: build
     env_file:
       - .env
-    depends_on:
-      postgres:
-        condition: service_healthy
+    volumes:
+      - bot_data:/data
     develop:
       watch:
         - action: rebuild
           path: .
 
-  postgres:
-    image: postgres:16-alpine
-    environment:
-      POSTGRES_DB: pen_bot
-      POSTGRES_USER: pen_bot
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-pen_bot_password}
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER}"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
 volumes:
-  postgres_data:
+  bot_data:

docs/INDEX.md

@@ -11,10 +11,11 @@ documented by the comment above its declaration, and `go doc` renders those.
 
 ## Contents
 
-Pages are added to the toctree below as they are written. It stays commented out
-until those pages exist: an enabled toctree naming a missing document warns
-rather than failing, so the build would still go green and ship a broken
-navigation tree. Uncomment it once the pages land.
+Pages are added to the toctree below as they are written. Only documents that
+exist are listed: an enabled toctree naming a missing document warns rather
+than failing, so the build would still go green and ship a broken navigation
+tree. A page that is not listed is also a warning, so add the entry in the same
+change that adds the page.
 
 Sphinx treats that warning as an error only under `-W`, which
 `.github/workflows/docs.yml` passes on every push and pull request. Run it the
@@ -27,15 +28,12 @@ uv run sphinx-build -W -b html docs public
 
 The output directory is `public/` and is not tracked.
 
-<!--
 :::{toctree}
 :maxdepth: 2
 :caption: Contents
 
-SETUP
-ARCHITECTURE
+OPERATIONS/DATABASE
 :::
--->
 
 ## Planned documents
 

docs/OPERATIONS/DATABASE.md

@@ -0,0 +1,126 @@
+# Database
+
+The bot stores its data in a SQLite database, in a single file on disk. There is
+no database server to run, no credentials to manage and no separate container
+in the deployment.
+
+## Where the file lives
+
+`DB_PATH` names the file. The directory holding it is created if it does not
+exist.
+
+```bash
+DB_PATH=/data/pen-bot.db
+```
+
+When `DB_PATH` is unset, the file name comes from `DB_BOT_INSTANCE_ID`, which
+exists so that several instances can share one directory without seeing each
+other's data:
+
+```bash
+DB_BOT_INSTANCE_ID=pen-bot-1   # writes data/pen-bot-pen-bot-1.db
+```
+
+Characters that are not letters, digits, `-` or `_` are dropped from the
+instance id, so the name cannot escape the directory it is in.
+
+## The container
+
+The image sets `DB_PATH=/data/pen-bot.db`, and the compose file mounts a
+volume there:
+
+```yaml
+services:
+  app:
+    volumes:
+      - bot_data:/data
+```
+
+The image creates `/data` owned by the unprivileged user the container runs as.
+Docker seeds a fresh named volume with the ownership of the directory it
+covers, so the volume is writable without any manual permission fixing.
+
+## The schema
+
+Three tables hold the bot's data, plus one that records which migrations have
+run.
+
+`guilds`
+: One row per guild, holding the guild's settings as a JSON document.
+
+`users`
+: One row per user per guild, holding the user's profile as a JSON document.
+
+`moderation_logs`
+: One row per moderation action, with the moderator, the target, the action
+  and the reason.
+
+`schema_migrations`
+: One row per applied migration, recording the file name and when it ran.
+
+Timestamps are stored as Unix milliseconds in an `INTEGER` column, declared
+`DATETIME`. Read them by selecting them into a `time.Time`; the driver converts
+them.
+
+The driver converts a timestamp only when the column it came from is declared
+`DATE`, `DATETIME` or `TIMESTAMP`. An aggregate over such a column, such as
+`MAX(created_at)`, has no declared type and comes back as an integer, so
+scanning it into a `time.Time` fails. Select the column itself instead.
+
+## Migrations
+
+Migrations are files in `internal/db/migrations/`, named `NNN_name.sql`. The
+running bot applies them at startup, before it serves any command, so a
+deployment needs no separate migration step. Each file runs in one transaction,
+and its name is recorded in `schema_migrations` so it is not applied twice.
+
+A migration must be plain DDL. The file is handed to the database as one
+statement, so a semicolon inside a string literal is safe.
+
+The name of the file is what records it as applied. Editing a file that a
+database has already run changes nothing for that database: it keeps the
+schema it has. To change the schema of a database that is already in use, add a
+new migration file rather than editing an old one.
+
+## Backing up
+
+The database is one file, but while the bot is running it also has a write-ahead
+log beside it. Copying only the `.db` file would miss the most recent writes.
+
+Take a consistent copy with `VACUUM INTO`, which writes a complete snapshot
+without stopping the bot:
+
+```bash
+sqlite3 /data/pen-bot.db "VACUUM INTO '/data/backup-$(date +%F).db'"
+```
+
+Restore by stopping the bot and putting the snapshot back in place:
+
+```bash
+docker compose stop app
+cp backup-2026-11-10.db /data/pen-bot.db
+rm -f /data/pen-bot.db-wal /data/pen-bot.db-shm
+docker compose start app
+```
+
+The bot applies any pending migrations on the next start.
+
+## Inspecting the data
+
+Any SQLite client can open the file, and the `sqlite3` command line tool is
+enough for most questions:
+
+```bash
+sqlite3 /data/pen-bot.db ".tables"
+sqlite3 /data/pen-bot.db "SELECT * FROM moderation_logs ORDER BY created_at DESC LIMIT 10;"
+```
+
+Stop the bot first if you intend to write, so that the two do not contend for
+the same lock.
+
+## Concurrency
+
+SQLite allows one writer at a time. The bot runs with the write-ahead log
+enabled, a busy timeout, and a small connection pool, which is enough for the
+traffic one guild produces. Writes are serialised no matter how large the pool
+is, so raising the pool size will not make them faster.

go.mod

@@ -8,16 +8,24 @@ require (
 	github.com/disgoorg/omit v1.0.0
 	github.com/disgoorg/snowflake/v2 v2.0.3
 	github.com/go-resty/resty/v2 v2.17.2
-	github.com/lib/pq v1.12.3
+	modernc.org/sqlite v1.62.0
 )
 
 require (
 	github.com/disgoorg/godave v0.3.0 // indirect
 	github.com/disgoorg/json/v2 v2.0.0 // indirect
+	github.com/dustin/go-humanize v1.1.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/klauspost/compress v1.20.4 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/ncruces/go-strftime v1.1.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/sasha-s/go-csync v0.0.0-20240107134140-fcbab37b09ad // indirect
 	golang.org/x/crypto v0.58.0 // indirect
 	golang.org/x/net v0.62.0 // indirect
 	golang.org/x/sys v0.50.0 // indirect
+	modernc.org/libc v1.80.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.12.1 // indirect
 )