diff --git a/charts/sourcegraph/CHANGELOG.md b/charts/sourcegraph/CHANGELOG.md index b13318435..225a58ded 100644 --- a/charts/sourcegraph/CHANGELOG.md +++ b/charts/sourcegraph/CHANGELOG.md @@ -8,6 +8,7 @@ Use `**BREAKING**:` to denote a breaking change ## Unreleased +- Added configurable pre-shutdown pauses, graceful-shutdown timeouts, and termination grace periods for application services - Added optional `syntectServer.podDisruptionBudget` support - Added optional `searcher.podDisruptionBudget` support - Set `DEPLOY_TYPE=helm` consistently for all Sourcegraph application containers diff --git a/charts/sourcegraph/README.md b/charts/sourcegraph/README.md index cdd2a2d9e..870239521 100644 --- a/charts/sourcegraph/README.md +++ b/charts/sourcegraph/README.md @@ -20,6 +20,9 @@ In addition to the documented values, all services also support the following va - `.nodeSelector` - [learn more](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) - `.tolerations` - [learn more](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) - `.podSecurityContext` - [learn more](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) +- `.preShutdownPause` - override `sourcegraph.preShutdownPause` for this service +- `.gracefulShutdownTimeout` - override `sourcegraph.gracefulShutdownTimeout` for this service +- `.terminationGracePeriodSeconds` - override `sourcegraph.terminationGracePeriodSeconds` for this service - `.args` - override default container args - `.env` - consult `values.yaml` file - `.serivceAccount.create` - create service account for service @@ -343,6 +346,7 @@ In addition to the documented values, all services also support the following va | sgTestConnection | object | `{"enabled":true}` | Enable the busybox connection test after deployment | | sourcegraph.affinity | object | `{}` | Global Affinity, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | | sourcegraph.disableKubernetesSecrets | bool | `false` | Disable the creation of Kubernetes secrets objects | +| sourcegraph.gracefulShutdownTimeout | string | `"15s"` | Total time Sourcegraph services have for the pre-shutdown pause and graceful drain | | sourcegraph.image.defaultTag | string | `"{{ .Chart.AppVersion }}"` | Global docker image tag | | sourcegraph.image.pullPolicy | string | `"IfNotPresent"` | Global docker image pull policy | | sourcegraph.image.repository | string | `"index.docker.io/sourcegraph"` | Global docker image registry or prefix | @@ -354,9 +358,11 @@ In addition to the documented values, all services also support the following va | sourcegraph.nodeSelector | object | `{}` | Global NodeSelector, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) | | sourcegraph.podAnnotations | object | `{}` | Add extra annotations to attach to all pods | | sourcegraph.podLabels | object | `{}` | Add extra labels to attach to all pods | +| sourcegraph.preShutdownPause | string | `"5s"` | Time Sourcegraph services continue accepting requests after shutdown begins so Kubernetes endpoint changes can propagate | | sourcegraph.priorityClassName | string | `""` | Assign a priorityClass to all pods (daemonSets, deployments, and statefulSets) | | sourcegraph.revisionHistoryLimit | int | `10` | Global deployment clean up policy, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#clean-up-policy) | | sourcegraph.serviceLabels | object | `{}` | Add extra labels to all services | +| sourcegraph.terminationGracePeriodSeconds | int | `30` | Time Kubernetes allows Sourcegraph services to shut down before forced termination | | sourcegraph.tolerations | list | `[]` | Global Tolerations, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) | | storageClass.allowedTopologies | list | `[]` | Persistent volumes topology configuration, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/storage/storage-classes/#allowed-topologies) | | storageClass.create | bool | `true` | Enable creation of storageClass. Disable if you have your own existing storage class | diff --git a/charts/sourcegraph/README.md.gotmpl b/charts/sourcegraph/README.md.gotmpl index 9e20f45e1..06ed43c99 100644 --- a/charts/sourcegraph/README.md.gotmpl +++ b/charts/sourcegraph/README.md.gotmpl @@ -20,6 +20,9 @@ In addition to the documented values, all services also support the following va - `.nodeSelector` - [learn more](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) - `.tolerations` - [learn more](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) - `.podSecurityContext` - [learn more](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) +- `.preShutdownPause` - override `sourcegraph.preShutdownPause` for this service +- `.gracefulShutdownTimeout` - override `sourcegraph.gracefulShutdownTimeout` for this service +- `.terminationGracePeriodSeconds` - override `sourcegraph.terminationGracePeriodSeconds` for this service - `.args` - override default container args - `.env` - consult `values.yaml` file - `.serivceAccount.create` - create service account for service diff --git a/charts/sourcegraph/templates/_helpers/_gracefulShutdown.tpl b/charts/sourcegraph/templates/_helpers/_gracefulShutdown.tpl new file mode 100644 index 000000000..4a9bff41e --- /dev/null +++ b/charts/sourcegraph/templates/_helpers/_gracefulShutdown.tpl @@ -0,0 +1,29 @@ +{{/* +Render the standard graceful-shutdown environment variables for a service. +Service-specific values override global defaults. Explicit entries in the service's env map +take precedence so existing configurations remain valid. +*/}} +{{- define "sourcegraph.gracefulShutdownEnv" -}} +{{- $top := index . 0 -}} +{{- $serviceName := index . 1 -}} +{{- $service := index $top.Values $serviceName -}} +{{- if not (hasKey $service.env "SRC_PRE_SHUTDOWN_PAUSE") }} +- name: SRC_PRE_SHUTDOWN_PAUSE + value: {{ default $top.Values.sourcegraph.preShutdownPause $service.preShutdownPause | quote }} +{{- end }} +{{- if not (hasKey $service.env "SRC_GRACEFUL_SHUTDOWN_TIMEOUT") }} +- name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT + value: {{ default $top.Values.sourcegraph.gracefulShutdownTimeout $service.gracefulShutdownTimeout | quote }} +{{- end }} +{{- end }} + +{{/* Render the pod termination deadline for a service. */}} +{{- define "sourcegraph.terminationGracePeriodSeconds" -}} +{{- $top := index . 0 -}} +{{- $service := index $top.Values (index . 1) -}} +{{- $terminationGracePeriodSeconds := $top.Values.sourcegraph.terminationGracePeriodSeconds -}} +{{- if and (hasKey $service "terminationGracePeriodSeconds") (ne $service.terminationGracePeriodSeconds nil) -}} +{{- $terminationGracePeriodSeconds = $service.terminationGracePeriodSeconds -}} +{{- end -}} +terminationGracePeriodSeconds: {{ $terminationGracePeriodSeconds }} +{{- end }} diff --git a/charts/sourcegraph/templates/_worker.tpl b/charts/sourcegraph/templates/_worker.tpl index 73429d300..2b087e7f2 100644 --- a/charts/sourcegraph/templates/_worker.tpl +++ b/charts/sourcegraph/templates/_worker.tpl @@ -64,9 +64,11 @@ spec: worker-replica: {{ $name | quote }} {{- end }} spec: + {{- include "sourcegraph.terminationGracePeriodSeconds" (list $top "worker") | nindent 6 }} containers: - name: worker env: + {{- include "sourcegraph.gracefulShutdownEnv" (list $top "worker") | nindent 8 }} {{- include "sourcegraph.redisConnection" $top | nindent 8 }} {{- if $allowlist }} - name: WORKER_JOB_ALLOWLIST diff --git a/charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml b/charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml index 1696644e5..41575281d 100644 --- a/charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml +++ b/charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml @@ -47,6 +47,7 @@ spec: app: sourcegraph-frontend deploy: sourcegraph spec: + {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "frontend") | nindent 6 }} initContainers: {{- if .Values.migrator.enabled }} - name: migrator @@ -86,6 +87,7 @@ spec: {{- end }} args: {{- default (list "serve") .Values.frontend.args | toYaml | nindent 8 }} env: + {{- include "sourcegraph.gracefulShutdownEnv" (list . "frontend") | nindent 8 }} {{- if not .Values.sourcegraph.disableKubernetesSecrets }} {{- include "sourcegraph.databaseAuth" (list . "pgsql" "PG") | nindent 8 }} {{- include "sourcegraph.databaseAuth" (list . "codeIntelDB" "CODEINTEL_PG") | nindent 8 }} diff --git a/charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml b/charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml index 99d1eca77..3e093ce57 100644 --- a/charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml +++ b/charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml @@ -43,12 +43,14 @@ spec: type: gitserver deploy: sourcegraph spec: + {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "gitserver") | nindent 6 }} containers: - name: gitserver args: {{- default (list "run") .Values.gitserver.args | toYaml | nindent 8 }} image: {{ include "sourcegraph.image" (list . "gitserver") }} imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }} env: + {{- include "sourcegraph.gracefulShutdownEnv" (list . "gitserver") | nindent 8 }} {{- include "sourcegraph.redisConnection" .| nindent 8 }} {{- range $name, $item := .Values.gitserver.env}} - name: {{ $name }} diff --git a/charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml b/charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml index bb15fbf7c..3fc4f37b9 100644 --- a/charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml +++ b/charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml @@ -45,9 +45,11 @@ spec: deploy: sourcegraph app: precise-code-intel-worker spec: + {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "preciseCodeIntel") | nindent 6 }} containers: - name: precise-code-intel-worker env: + {{- include "sourcegraph.gracefulShutdownEnv" (list . "preciseCodeIntel") | nindent 8 }} {{- range $name, $item := .Values.preciseCodeIntel.env}} - name: {{ $name }} {{- $item | toYaml | nindent 10 }} diff --git a/charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml b/charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml index f8968a54d..60bc6344a 100644 --- a/charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml +++ b/charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml @@ -43,6 +43,7 @@ spec: deploy: sourcegraph app: searcher spec: + {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "searcher") | nindent 6 }} containers: - name: searcher {{- with .Values.searcher.args }} @@ -50,6 +51,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} env: + {{- include "sourcegraph.gracefulShutdownEnv" (list . "searcher") | nindent 8 }} {{- include "sourcegraph.redisConnection" .| nindent 8 }} {{- range $name, $item := .Values.searcher.env}} - name: {{ $name }} @@ -154,4 +156,4 @@ spec: resources: requests: storage: {{ .Values.searcher.storageSize | default "26Gi" }} - storageClassName: {{ .Values.storageClass.name }} \ No newline at end of file + storageClassName: {{ .Values.storageClass.name }} diff --git a/charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml b/charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml index d7a63f379..b68c89c3b 100644 --- a/charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml +++ b/charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml @@ -46,9 +46,11 @@ spec: deploy: sourcegraph app: syntactic-code-intel-worker spec: + {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "syntacticCodeIntel") | nindent 6 }} containers: - name: syntactic-code-intel-worker env: + {{- include "sourcegraph.gracefulShutdownEnv" (list . "syntacticCodeIntel") | nindent 8 }} {{- range $name, $item := .Values.syntacticCodeIntel.env}} - name: {{ $name }} {{- $item | toYaml | nindent 10 }} diff --git a/charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml b/charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml index 9a66ae5fc..1195f6ad9 100644 --- a/charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml +++ b/charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml @@ -45,9 +45,11 @@ spec: deploy: sourcegraph app: syntect-server spec: + {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "syntectServer") | nindent 6 }} containers: - name: syntect-server env: + {{- include "sourcegraph.gracefulShutdownEnv" (list . "syntectServer") | nindent 8 }} {{- range $name, $item := .Values.syntectServer.env}} - name: {{ $name }} {{- $item | toYaml | nindent 10 }} diff --git a/charts/sourcegraph/tests/gracefulShutdown_test.yaml b/charts/sourcegraph/tests/gracefulShutdown_test.yaml new file mode 100644 index 000000000..63691456c --- /dev/null +++ b/charts/sourcegraph/tests/gracefulShutdown_test.yaml @@ -0,0 +1,125 @@ +suite: graceful shutdown +release: + name: sourcegraph + namespace: sourcegraph +tests: +- it: should render global defaults for frontend + template: frontend/sourcegraph-frontend.Deployment.yaml + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 30 + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_PRE_SHUTDOWN_PAUSE + value: 5s + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT + value: 15s +- it: should render global defaults for gitserver + template: gitserver/gitserver.StatefulSet.yaml + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 30 + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_PRE_SHUTDOWN_PAUSE + value: 5s + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT + value: 15s +- it: should render service overrides for searcher + template: searcher/searcher.StatefulSet.yaml + set: + searcher: + preShutdownPause: 7s + gracefulShutdownTimeout: 25s + terminationGracePeriodSeconds: 35 + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 35 + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_PRE_SHUTDOWN_PAUSE + value: 7s + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT + value: 25s +- it: should preserve explicit env overrides + template: precise-code-intel/worker.Deployment.yaml + set: + preciseCodeIntel: + env: + SRC_PRE_SHUTDOWN_PAUSE: + value: 8s + SRC_GRACEFUL_SHUTDOWN_TIMEOUT: + value: 28s + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_PRE_SHUTDOWN_PAUSE + value: 8s + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT + value: 28s +- it: should preserve a zero-second service termination override + template: precise-code-intel/worker.Deployment.yaml + set: + preciseCodeIntel.terminationGracePeriodSeconds: 0 + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 0 +- it: should render defaults for syntactic code intel + template: syntactic-code-intel/worker.Deployment.yaml + set: + syntacticCodeIntel.enabled: true + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 30 + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_PRE_SHUTDOWN_PAUSE + value: 5s + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT + value: 15s +- it: should render defaults for worker replicas + template: worker/worker.Deployment.yaml + set: + worker: + replicas: + - jobs: [job1] + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 30 + documentIndex: 0 + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 30 + documentIndex: 1 + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT + value: 15s + documentIndex: 0 diff --git a/charts/sourcegraph/tests/syntectServer_test.yaml b/charts/sourcegraph/tests/syntectServer_test.yaml new file mode 100644 index 000000000..8d144d815 --- /dev/null +++ b/charts/sourcegraph/tests/syntectServer_test.yaml @@ -0,0 +1,43 @@ +suite: syntect server +release: + name: sourcegraph + namespace: sourcegraph +templates: +- syntect-server/syntect-server.Deployment.yaml +tests: +- it: should render rollout-safe defaults + asserts: + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 30 + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_PRE_SHUTDOWN_PAUSE + value: 5s + - contains: + path: spec.template.spec.containers[0].env + content: + name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT + value: 15s +- it: should render replica and termination grace overrides + set: + syntectServer: + replicaCount: 2 + terminationGracePeriodSeconds: 60 + asserts: + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 60 diff --git a/charts/sourcegraph/values.yaml b/charts/sourcegraph/values.yaml index e183f5fe8..db31d8563 100644 --- a/charts/sourcegraph/values.yaml +++ b/charts/sourcegraph/values.yaml @@ -38,6 +38,12 @@ sourcegraph: # -- Global deployment clean up policy, # learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#clean-up-policy) revisionHistoryLimit: 10 + # -- Time Sourcegraph services continue accepting requests after shutdown begins so Kubernetes endpoint changes can propagate + preShutdownPause: 5s + # -- Total time Sourcegraph services have for the pre-shutdown pause and graceful drain + gracefulShutdownTimeout: 15s + # -- Time Kubernetes allows Sourcegraph services to shut down before forced termination + terminationGracePeriodSeconds: 30 # -- Add extra labels to all services serviceLabels: {} # -- Disable the creation of Kubernetes secrets objects @@ -65,6 +71,10 @@ sourcegraph: # serviceAnnotations: {} # # Provide custom environment variables # env: {} +# # Override the global graceful shutdown settings +# preShutdownPause: "" +# gracefulShutdownTimeout: "" +# terminationGracePeriodSeconds: null # # Set resource requests / limits # resources: {} # containerSecurityContext: {}