diff --git a/README.md b/README.md index 2dbce9d..8c54ecf 100644 --- a/README.md +++ b/README.md @@ -343,6 +343,32 @@ from the manifest is removed from the live object. replication slot cannot fill the volume. - Rebuild a single broken replica with `kubectl cnpg destroy high-command-postgres -n high-command`. CNPG re-clones it with `pg_basebackup` and drops the old slot. +- Helm patches custom resources with a two-way merge (previous manifest to new manifest), so a + field changed live and not in git is never corrected unless the manifest changes it. + +### Backups + +WAL is archived continuously and a base backup runs daily at 09:30 UTC through the Barman Cloud +plugin, installed by [gitops-core `cnpg-barman-cloud`](https://github.com/DataKnifeAI/gitops-core/tree/main/cnpg-barman-cloud). +The config is in `k8s/high-command/objectstore.yaml` (ObjectStore + ScheduledBackup) and the +`plugins` section of the Cluster. + +- **Where**: `s3://rke2-backups/cnpg/prd-apps/high-command-postgres/` on rustfs + (`https://rustfs.dataknife.net:30292`), gzip, 14 day retention. +- **Credentials**: `cnpg-backup-rustfs` secret in `high-command` (keys `ACCESS_KEY_ID`, + `ACCESS_SECRET_KEY`), created by hand, not in git. + +```bash +kubectl cnpg status high-command-postgres -n high-command +kubectl -n high-command get backups.postgresql.cnpg.io +kubectl cnpg backup high-command-postgres -n high-command --method=plugin --plugin-name=barman-cloud.cloudnative-pg.io +``` + +Restore (PITR possible): create a new Cluster with `bootstrap.recovery.source` set to an +`externalClusters` entry that uses plugin `barman-cloud.cloudnative-pg.io` with +`barmanObjectName: high-command-postgres-rustfs` and `serverName: high-command-postgres`. Then +point the Pooler (`spec.cluster.name`) or the apps at the new cluster. A full example is in the +gitops-core `cnpg-barman-cloud` README. ## Documentation diff --git a/k8s/high-command/objectstore.yaml b/k8s/high-command/objectstore.yaml new file mode 100644 index 0000000..a0e379a --- /dev/null +++ b/k8s/high-command/objectstore.yaml @@ -0,0 +1,48 @@ +# Barman Cloud plugin object store for high-command-postgres (plugin installed by gitops-core cnpg-barman-cloud) +# Backups land in s3://rke2-backups/cnpg/prd-apps/high-command-postgres/ +# Requires: cnpg-backup-rustfs secret (ACCESS_KEY_ID, ACCESS_SECRET_KEY) in this namespace +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: high-command-postgres-rustfs + namespace: high-command +spec: + retentionPolicy: "14d" + configuration: + destinationPath: s3://rke2-backups/cnpg/prd-apps + endpointURL: https://rustfs.dataknife.net:30292 + s3Credentials: + accessKeyId: + name: cnpg-backup-rustfs + key: ACCESS_KEY_ID + secretAccessKey: + name: cnpg-backup-rustfs + key: ACCESS_SECRET_KEY + wal: + compression: gzip + maxParallel: 2 + data: + compression: gzip + jobs: 2 + instanceSidecarConfiguration: + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + memory: 1Gi +--- +# Daily base backup, 09:30 UTC (02:30 PDT). Staggered with other CNPG clusters. +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: high-command-postgres-daily + namespace: high-command +spec: + schedule: "0 30 9 * * *" + backupOwnerReference: self + cluster: + name: high-command-postgres + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io diff --git a/k8s/high-command/postgres-cluster.yaml b/k8s/high-command/postgres-cluster.yaml index a238187..b7a72c3 100644 --- a/k8s/high-command/postgres-cluster.yaml +++ b/k8s/high-command/postgres-cluster.yaml @@ -84,6 +84,12 @@ spec: work_mem: 4MB syncReplicaElectionConstraint: enabled: false + # WAL archiving + base backups to rustfs (ObjectStore in objectstore.yaml) + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: high-command-postgres-rustfs primaryUpdateMethod: restart primaryUpdateStrategy: unsupervised probes: