diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 6a7438a89..5b3922ba7 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -36,6 +36,10 @@ jobs: go-version-file: go.mod - name: Setup Helm uses: azure/setup-helm@v4 + - name: Test Helm chart + run: | + helm plugin install https://github.com/helm-unittest/helm-unittest.git --version 1.0.3 --verify=false + make helm-test - name: Cache micro-VM assets uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: diff --git a/Makefile b/Makefile index b04750a7d..01dbb9d36 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,10 @@ clean: helm-template: @./hack/render-manifests.sh +.PHONY: helm-test +helm-test: + @helm unittest charts/substrate + # Verify that manifests/ate-install/ matches the chart output. Used in CI. .PHONY: verify-helm-template verify-helm-template: diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 7fd7b09ed..e513bb613 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -41,7 +41,11 @@ See `values.yaml` for the full set; the important keys: | Key | Default | Notes | |-----|---------|-------| | `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance | -| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.connectionString` | `""` (in-cluster) | Runtime/DML connection for external PostgreSQL | +| `postgres.connectionStringSecretRef` | disabled | Read the runtime/DML connection from a Secret; its name defaults to `-postgres-connection` when enabled | +| `postgres.ddlConnectionString` | `""` (runtime connection) | Optional schema-owner connection for migrations and maintenance | +| `postgres.ddlConnectionStringSecretRef` | disabled | Read the optional schema-owner connection string from a Secret | +| `postgres.pool.maxConnLifetime` | `""` (pgx default) | Maximum physical connection lifetime; bounds Secret credential turnover | | `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema | | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | @@ -57,3 +61,16 @@ See `values.yaml` for the full set; the important keys: | `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` | | `otel.logs.enabled` | `true` | Set to `false` to export no logs. Gates both OTLP log sources: ateapi's actor lifecycle events and the router access log | | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | + +## PostgreSQL credential rotation + +Secret-backed connection strings are mounted as projected files. Kubernetes +updates these files when the Secret changes, and Substrate reads the current +value when it opens a new physical connection. Inline connection strings are +static until the pod restarts. + +`postgres.pool.maxConnLifetime` bounds how long established connections may +continue using an old credential; rotation is not immediate. Keep old and new +credentials valid long enough for Kubernetes projection and connection +turnover. The host, port, database, user, and fallback targets must remain the +same during rotation; changing any of them requires a restart. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 45184413f..303c9d660 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -78,6 +78,18 @@ Plaintext HTTP URL that clients use to reach atenet-router. {{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} {{- end -}} +{{/* PostgreSQL connection Secret, when configured. */}} +{{- define "substrate.postgres.connectionStringSecretEnabled" -}} +{{- $ref := .Values.postgres.connectionStringSecretRef | default dict -}} +{{- if or (get $ref "enabled") (get $ref "name") -}}true{{- end -}} +{{- end -}} + +{{/* PostgreSQL DDL connection Secret, when configured. */}} +{{- define "substrate.postgres.ddlConnectionStringSecretEnabled" -}} +{{- $ref := .Values.postgres.ddlConnectionStringSecretRef | default dict -}} +{{- if or (get $ref "enabled") (get $ref "name") -}}true{{- end -}} +{{- end -}} + {{/* OTLP endpoint a signal exports to, or empty when the signal is disabled or no endpoint resolves. The per-signal endpoint wins over the generic one, matching diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index ca76ae3ef..3a13e254b 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -14,8 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }} -{{- fail "postgres.connectionString is required when postgres.enabled=false" }} +{{- $secretEnabled := include "substrate.postgres.connectionStringSecretEnabled" . -}} +{{- $ddlSecretEnabled := include "substrate.postgres.ddlConnectionStringSecretEnabled" . -}} +{{- if and .Values.postgres.connectionString $secretEnabled }} +{{- fail "postgres.connectionString and postgres.connectionStringSecretRef are mutually exclusive" }} +{{- end }} +{{- if and .Values.postgres.ddlConnectionString $ddlSecretEnabled }} +{{- fail "postgres.ddlConnectionString and postgres.ddlConnectionStringSecretRef are mutually exclusive" }} +{{- end }} +{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) (not $secretEnabled) }} +{{- fail "postgres.connectionString or postgres.connectionStringSecretRef is required when postgres.enabled=false" }} +{{- end }} +{{- if and (or .Values.postgres.ddlConnectionString $ddlSecretEnabled) (empty .Values.postgres.connectionString) (not $secretEnabled) }} +{{- fail "postgres.connectionString or postgres.connectionStringSecretRef is required when a DDL connection is configured" }} {{- end }} apiVersion: v1 kind: ConfigMap @@ -23,5 +34,10 @@ metadata: name: {{ .Values.ateApiServerEnvVarsConfigMap }} namespace: {{ .Release.Namespace }} data: + {{- if not $secretEnabled }} ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} + {{- end }} + {{- if and .Values.postgres.ddlConnectionString (not $ddlSecretEnabled) }} + ATE_API_POSTGRES_DDL_CONNECTION_STRING: {{ .Values.postgres.ddlConnectionString | quote }} + {{- end }} ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index b93dbf420..3aaa50cd4 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- $connectionStringSecretRef := .Values.postgres.connectionStringSecretRef -}} +{{- $connectionStringSecretEnabled := include "substrate.postgres.connectionStringSecretEnabled" . -}} +{{- $ddlConnectionStringSecretRef := .Values.postgres.ddlConnectionStringSecretRef -}} +{{- $ddlConnectionStringSecretEnabled := include "substrate.postgres.ddlConnectionStringSecretEnabled" . -}} + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -85,8 +90,20 @@ spec: - "--grpc-listen-addr=0.0.0.0:443" - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" +{{- if $connectionStringSecretEnabled }} + - "--postgres-connection-string=@file:/etc/ateapi/postgres/runtime/connection-string" +{{- else }} - "--postgres-connection-string=@env" +{{- end }} +{{- if $ddlConnectionStringSecretEnabled }} + - "--postgres-ddl-connection-string=@file:/etc/ateapi/postgres/ddl/connection-string" +{{- else if .Values.postgres.ddlConnectionString }} + - "--postgres-ddl-connection-string=@env" +{{- end }} - "--postgres-schema=@env" +{{- with .Values.postgres.pool.maxConnLifetime }} + - {{ printf "--postgres-max-conn-lifetime=%s" . | quote }} +{{- end }} - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" @@ -139,6 +156,16 @@ spec: - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } +{{- if $connectionStringSecretEnabled }} + - name: postgres-runtime-connection + mountPath: /etc/ateapi/postgres/runtime + readOnly: true +{{- end }} +{{- if $ddlConnectionStringSecretEnabled }} + - name: postgres-ddl-connection + mountPath: /etc/ateapi/postgres/ddl + readOnly: true +{{- end }} ports: - containerPort: 443 - name: prometheus @@ -200,6 +227,26 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem +{{- if $connectionStringSecretEnabled }} + - name: postgres-runtime-connection + projected: + sources: + - secret: + name: {{ get $connectionStringSecretRef "name" | default (include "substrate.fullname" (list "postgres-connection" .)) | quote }} + items: + - key: {{ get $connectionStringSecretRef "key" | default "connectionString" | quote }} + path: connection-string +{{- end }} +{{- if $ddlConnectionStringSecretEnabled }} + - name: postgres-ddl-connection + projected: + sources: + - secret: + name: {{ get $ddlConnectionStringSecretRef "name" | default (include "substrate.fullname" (list "postgres-connection" .)) | quote }} + items: + - key: {{ get $ddlConnectionStringSecretRef "key" | default "ddlConnectionString" | quote }} + path: connection-string +{{- end }} --- apiVersion: policy/v1 kind: PodDisruptionBudget diff --git a/charts/substrate/tests/postgres_test.yaml b/charts/substrate/tests/postgres_test.yaml new file mode 100644 index 000000000..5cdd647e3 --- /dev/null +++ b/charts/substrate/tests/postgres_test.yaml @@ -0,0 +1,374 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +suite: PostgreSQL configuration +templates: +- ate-api-server-envvars.yaml +- ate-api-server.yaml +- postgres.yaml +tests: +- it: configures the bundled PostgreSQL database by default + template: ate-api-server-envvars.yaml + asserts: + - isNotNull: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + - equal: + path: data.ATE_API_POSTGRES_SCHEMA + value: public + +- it: uses the static runtime source and no DDL override by default + template: ate-api-server.yaml + documentIndex: 3 + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-connection-string=@env + - notContains: + path: spec.template.spec.containers[0].args + content: --postgres-ddl-connection-string=@env + - notContains: + path: spec.template.spec.volumes + content: + name: postgres-runtime-connection + - notContains: + path: spec.template.spec.volumes + content: + name: postgres-ddl-connection + +- it: configures an external PostgreSQL connection string and schema + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + connectionString: postgresql://user:pass@database:5432/agents + schema: substrate + asserts: + - equal: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + value: postgresql://user:pass@database:5432/agents + - equal: + path: data.ATE_API_POSTGRES_SCHEMA + value: substrate + +- it: configures separate runtime and DDL connection strings + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + connectionString: postgresql://runtime@database:5432/agents + ddlConnectionString: postgresql://owner@database:5432/agents + asserts: + - equal: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + value: postgresql://runtime@database:5432/agents + - equal: + path: data.ATE_API_POSTGRES_DDL_CONNECTION_STRING + value: postgresql://owner@database:5432/agents + +- it: omits bundled PostgreSQL for an external connection + template: postgres.yaml + set: + postgres: + enabled: false + connectionString: postgresql://user:pass@database:5432/agents + asserts: + - hasDocuments: + count: 0 + +- it: mounts an explicitly named runtime connection Secret + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionStringSecretRef: + name: shared-database + key: url + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-connection-string=@file:/etc/ateapi/postgres/runtime/connection-string + - notContains: + path: spec.template.spec.containers[0].args + content: --postgres-ddl-connection-string=@env + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_CONNECTION_STRING + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: postgres-runtime-connection + mountPath: /etc/ateapi/postgres/runtime + readOnly: true + - notExists: + path: spec.template.spec.containers[0].volumeMounts[?(@.name == "postgres-runtime-connection")].subPath + - contains: + path: spec.template.spec.volumes + content: + name: postgres-runtime-connection + projected: + sources: + - secret: + name: shared-database + items: + - key: url + path: connection-string + +- it: mounts separate runtime and DDL keys from one Secret + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionStringSecretRef: + name: substrate-database + key: runtimeUrl + ddlConnectionStringSecretRef: + name: substrate-database + key: ddlUrl + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-connection-string=@file:/etc/ateapi/postgres/runtime/connection-string + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-ddl-connection-string=@file:/etc/ateapi/postgres/ddl/connection-string + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_CONNECTION_STRING + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ATE_API_POSTGRES_DDL_CONNECTION_STRING + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: postgres-runtime-connection + mountPath: /etc/ateapi/postgres/runtime + readOnly: true + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: postgres-ddl-connection + mountPath: /etc/ateapi/postgres/ddl + readOnly: true + - notExists: + path: spec.template.spec.containers[0].volumeMounts[?(@.name == "postgres-runtime-connection")].subPath + - notExists: + path: spec.template.spec.containers[0].volumeMounts[?(@.name == "postgres-ddl-connection")].subPath + - contains: + path: spec.template.spec.volumes + content: + name: postgres-runtime-connection + projected: + sources: + - secret: + name: substrate-database + items: + - key: runtimeUrl + path: connection-string + - contains: + path: spec.template.spec.volumes + content: + name: postgres-ddl-connection + projected: + sources: + - secret: + name: substrate-database + items: + - key: ddlUrl + path: connection-string + +- it: mounts runtime and DDL connections from different Secrets + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionStringSecretRef: + name: runtime-database + key: url + ddlConnectionStringSecretRef: + name: ddl-database + key: url + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name == "postgres-runtime-connection")].projected.sources[0].secret.name + value: runtime-database + - equal: + path: spec.template.spec.volumes[?(@.name == "postgres-ddl-connection")].projected.sources[0].secret.name + value: ddl-database + +- it: supports a runtime Secret with an inline DDL connection + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionStringSecretRef: + name: runtime-database + ddlConnectionString: postgresql://owner@database:5432/agents + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-connection-string=@file:/etc/ateapi/postgres/runtime/connection-string + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-ddl-connection-string=@env + +- it: keeps only inline DDL credentials in the ConfigMap for mixed mode + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + connectionStringSecretRef: + name: runtime-database + ddlConnectionString: postgresql://owner@database:5432/agents + asserts: + - notExists: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + - equal: + path: data.ATE_API_POSTGRES_DDL_CONNECTION_STRING + value: postgresql://owner@database:5432/agents + +- it: supports an inline runtime connection with a DDL Secret + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionString: postgresql://runtime@database:5432/agents + ddlConnectionStringSecretRef: + name: ddl-database + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-connection-string=@env + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-ddl-connection-string=@file:/etc/ateapi/postgres/ddl/connection-string + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: postgres-ddl-connection + mountPath: /etc/ateapi/postgres/ddl + readOnly: true + - notExists: + path: spec.template.spec.containers[0].volumeMounts[?(@.name == "postgres-ddl-connection")].subPath + +- it: keeps only inline runtime credentials in the ConfigMap for mixed mode + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + connectionString: postgresql://runtime@database:5432/agents + ddlConnectionStringSecretRef: + name: ddl-database + asserts: + - equal: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + value: postgresql://runtime@database:5432/agents + - notExists: + path: data.ATE_API_POSTGRES_DDL_CONNECTION_STRING + +- it: derives a release-scoped Secret name when enabled + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + enabled: false + connectionStringSecretRef: + enabled: true + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: postgres-runtime-connection + projected: + sources: + - secret: + name: RELEASE-NAME-postgres-connection + items: + - key: connectionString + path: connection-string + +- it: keeps credentials out of the ConfigMap in Secret mode + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + schema: substrate + connectionStringSecretRef: + name: shared-database + asserts: + - notExists: + path: data.ATE_API_POSTGRES_CONNECTION_STRING + - equal: + path: data.ATE_API_POSTGRES_SCHEMA + value: substrate + +- it: configures the maximum connection lifetime + template: ate-api-server.yaml + documentIndex: 3 + set: + postgres: + pool: + maxConnLifetime: 10m + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --postgres-max-conn-lifetime=10m + +- it: rejects disabling bundled PostgreSQL without a connection + template: ate-api-server-envvars.yaml + set: + postgres: + enabled: false + asserts: + - failedTemplate: + errorMessage: postgres.connectionString or postgres.connectionStringSecretRef is required when postgres.enabled=false + +- it: rejects configuring both a literal connection and a Secret + template: ate-api-server-envvars.yaml + set: + postgres: + connectionString: postgresql://user:pass@database:5432/agents + connectionStringSecretRef: + name: shared-database + asserts: + - failedTemplate: + errorMessage: postgres.connectionString and postgres.connectionStringSecretRef are mutually exclusive + +- it: rejects a DDL connection without an explicit runtime connection + template: ate-api-server-envvars.yaml + set: + postgres: + ddlConnectionString: postgresql://owner@database:5432/agents + asserts: + - failedTemplate: + errorMessage: postgres.connectionString or postgres.connectionStringSecretRef is required when a DDL connection is configured + +- it: rejects both a literal and Secret-backed DDL connection + template: ate-api-server-envvars.yaml + set: + postgres: + connectionString: postgresql://runtime@database:5432/agents + ddlConnectionString: postgresql://owner@database:5432/agents + ddlConnectionStringSecretRef: + name: substrate-database + asserts: + - failedTemplate: + errorMessage: postgres.ddlConnectionString and postgres.ddlConnectionStringSecretRef are mutually exclusive diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 618be970b..eb284e28a 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -26,7 +26,24 @@ createNamespace: false postgres: enabled: true storageSize: 1Gi + # Runtime/DML connection. Also used for DDL when no DDL connection is set. connectionString: "" + # Read the connection string from a Secret instead of values/ConfigMap. + connectionStringSecretRef: + # When enabled with no name, use the release-scoped postgres-connection name. + enabled: false + name: "" + key: connectionString + # Optional DDL and maintenance connection. Defaults to connectionString. + ddlConnectionString: "" + ddlConnectionStringSecretRef: + enabled: false + name: "" + key: ddlConnectionString + pool: + # Bounds how long established connections can keep an old credential. + # Leave empty to use the upstream pgx default. + maxConnLifetime: "" schema: public resources: requests: