Support PostgreSQL connection secrets

.github/workflows/helm-e2e.yaml

@@ -36,6 +36,10 @@ jobs:
         go-version-file: go.mod
     - name: Setup Helm
       uses: azure/setup-helm@v4
+    - name: Test Helm chart
+      run: |
+        helm plugin install https://github.com/helm-unittest/helm-unittest.git --version 1.0.3 --verify=false
+        make helm-test
     - name: Cache micro-VM assets
       uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:

Makefile

@@ -139,6 +139,10 @@ clean:
 helm-template:
 	@./hack/render-manifests.sh
 
+.PHONY: helm-test
+helm-test:
+	@helm unittest charts/substrate
+
 # Verify that manifests/ate-install/ matches the chart output. Used in CI.
 .PHONY: verify-helm-template
 verify-helm-template:

charts/substrate/README.md

@@ -41,7 +41,11 @@ See `values.yaml` for the full set; the important keys:
 | Key | Default | Notes |
 |-----|---------|-------|
 | `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance |
-| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL |
+| `postgres.connectionString` | `""` (in-cluster) | Runtime/DML connection for external PostgreSQL |
+| `postgres.connectionStringSecretRef` | disabled | Read the runtime/DML connection from a Secret; its name defaults to `<release>-postgres-connection` when enabled |
+| `postgres.ddlConnectionString` | `""` (runtime connection) | Optional schema-owner connection for migrations and maintenance |
+| `postgres.ddlConnectionStringSecretRef` | disabled | Read the optional schema-owner connection string from a Secret |
+| `postgres.pool.maxConnLifetime` | `""` (pgx default) | Maximum physical connection lifetime; bounds Secret credential turnover |
 | `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema |
 | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size |
 | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots |
@@ -57,3 +61,16 @@ See `values.yaml` for the full set; the important keys:
 | `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` |
 | `otel.logs.enabled` | `true` | Set to `false` to export no logs. Gates both OTLP log sources: ateapi's actor lifecycle events and the router access log |
 | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` |
+
+## PostgreSQL credential rotation
+
+Secret-backed connection strings are mounted as projected files. Kubernetes
+updates these files when the Secret changes, and Substrate reads the current
+value when it opens a new physical connection. Inline connection strings are
+static until the pod restarts.
+
+`postgres.pool.maxConnLifetime` bounds how long established connections may
+continue using an old credential; rotation is not immediate. Keep old and new
+credentials valid long enough for Kubernetes projection and connection
+turnover. The host, port, database, user, and fallback targets must remain the
+same during rotation; changing any of them requires a restart.

charts/substrate/templates/_helpers.tpl

@@ -78,6 +78,18 @@ Plaintext HTTP URL that clients use to reach atenet-router.
 {{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}}
 {{- end -}}
 
+{{/* PostgreSQL connection Secret, when configured. */}}
+{{- define "substrate.postgres.connectionStringSecretEnabled" -}}
+{{- $ref := .Values.postgres.connectionStringSecretRef | default dict -}}
+{{- if or (get $ref "enabled") (get $ref "name") -}}true{{- end -}}
+{{- end -}}
+
+{{/* PostgreSQL DDL connection Secret, when configured. */}}
+{{- define "substrate.postgres.ddlConnectionStringSecretEnabled" -}}
+{{- $ref := .Values.postgres.ddlConnectionStringSecretRef | default dict -}}
+{{- if or (get $ref "enabled") (get $ref "name") -}}true{{- end -}}
+{{- end -}}
+
 {{/*
 OTLP endpoint a signal exports to, or empty when the signal is disabled or no
 endpoint resolves. The per-signal endpoint wins over the generic one, matching

charts/substrate/templates/ate-api-server-envvars.yaml

@@ -14,14 +14,30 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }}
-{{- fail "postgres.connectionString is required when postgres.enabled=false" }}
+{{- $secretEnabled := include "substrate.postgres.connectionStringSecretEnabled" . -}}
+{{- $ddlSecretEnabled := include "substrate.postgres.ddlConnectionStringSecretEnabled" . -}}
+{{- if and .Values.postgres.connectionString $secretEnabled }}
+{{- fail "postgres.connectionString and postgres.connectionStringSecretRef are mutually exclusive" }}
+{{- end }}
+{{- if and .Values.postgres.ddlConnectionString $ddlSecretEnabled }}
+{{- fail "postgres.ddlConnectionString and postgres.ddlConnectionStringSecretRef are mutually exclusive" }}
+{{- end }}
+{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) (not $secretEnabled) }}
+{{- fail "postgres.connectionString or postgres.connectionStringSecretRef is required when postgres.enabled=false" }}
+{{- end }}
+{{- if and (or .Values.postgres.ddlConnectionString $ddlSecretEnabled) (empty .Values.postgres.connectionString) (not $secretEnabled) }}
+{{- fail "postgres.connectionString or postgres.connectionStringSecretRef is required when a DDL connection is configured" }}
 {{- end }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ .Values.ateApiServerEnvVarsConfigMap }}
   namespace: {{ .Release.Namespace }}
 data:
+  {{- if not $secretEnabled }}
   ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }}
+  {{- end }}
+  {{- if and .Values.postgres.ddlConnectionString (not $ddlSecretEnabled) }}
+  ATE_API_POSTGRES_DDL_CONNECTION_STRING: {{ .Values.postgres.ddlConnectionString | quote }}
+  {{- end }}
   ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }}

charts/substrate/templates/ate-api-server.yaml

@@ -14,6 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- $connectionStringSecretRef := .Values.postgres.connectionStringSecretRef -}}
+{{- $connectionStringSecretEnabled := include "substrate.postgres.connectionStringSecretEnabled" . -}}
+{{- $ddlConnectionStringSecretRef := .Values.postgres.ddlConnectionStringSecretRef -}}
+{{- $ddlConnectionStringSecretEnabled := include "substrate.postgres.ddlConnectionStringSecretEnabled" . -}}
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -85,8 +90,20 @@ spec:
         - "--grpc-listen-addr=0.0.0.0:443"
         - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem"
         - "--authentication-config=/etc/ateapi/authentication/authentication.yaml"
+{{- if $connectionStringSecretEnabled }}
+        - "--postgres-connection-string=@file:/etc/ateapi/postgres/runtime/connection-string"
+{{- else }}
         - "--postgres-connection-string=@env"
+{{- end }}
+{{- if $ddlConnectionStringSecretEnabled }}
+        - "--postgres-ddl-connection-string=@file:/etc/ateapi/postgres/ddl/connection-string"
+{{- else if .Values.postgres.ddlConnectionString }}
+        - "--postgres-ddl-connection-string=@env"
+{{- end }}
         - "--postgres-schema=@env"
+{{- with .Values.postgres.pool.maxConnLifetime }}
+        - {{ printf "--postgres-max-conn-lifetime=%s" . | quote }}
+{{- end }}
         - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json"
         - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json"
         - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443"
@@ -139,6 +156,16 @@ spec:
         - { name: actor-id-ca-pool,    mountPath: /run/actor-id-ca-pool, readOnly: true }
         - { name: podidentity,         mountPath: /run/podidentity.podcert.ate.dev, readOnly: true }
         - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true }
+{{- if $connectionStringSecretEnabled }}
+        - name: postgres-runtime-connection
+          mountPath: /etc/ateapi/postgres/runtime
+          readOnly: true
+{{- end }}
+{{- if $ddlConnectionStringSecretEnabled }}
+        - name: postgres-ddl-connection
+          mountPath: /etc/ateapi/postgres/ddl
+          readOnly: true
+{{- end }}
         ports:
         - containerPort: 443
         - name: prometheus
@@ -200,6 +227,26 @@ spec:
                 matchLabels:
                   podcert.ate.dev/canarying: live
               path: trust-bundle.pem
+{{- if $connectionStringSecretEnabled }}
+      - name: postgres-runtime-connection
+        projected:
+          sources:
+          - secret:
+              name: {{ get $connectionStringSecretRef "name" | default (include "substrate.fullname" (list "postgres-connection" .)) | quote }}
+              items:
+              - key: {{ get $connectionStringSecretRef "key" | default "connectionString" | quote }}
+                path: connection-string
+{{- end }}
+{{- if $ddlConnectionStringSecretEnabled }}
+      - name: postgres-ddl-connection
+        projected:
+          sources:
+          - secret:
+              name: {{ get $ddlConnectionStringSecretRef "name" | default (include "substrate.fullname" (list "postgres-connection" .)) | quote }}
+              items:
+              - key: {{ get $ddlConnectionStringSecretRef "key" | default "ddlConnectionString" | quote }}
+                path: connection-string
+{{- end }}
 ---
 apiVersion: policy/v1
 kind: PodDisruptionBudget