Add configuration for graceful shutdown of application services

charts/sourcegraph/CHANGELOG.md

@@ -8,6 +8,7 @@ Use `**BREAKING**:` to denote a breaking change
 
 ## Unreleased
 
+- Added configurable pre-shutdown pauses, graceful-shutdown timeouts, and termination grace periods for application services
 - Added optional `syntectServer.podDisruptionBudget` support
 - Added optional `searcher.podDisruptionBudget` support
 - Set `DEPLOY_TYPE=helm` consistently for all Sourcegraph application containers

charts/sourcegraph/README.md

@@ -20,6 +20,9 @@ In addition to the documented values, all services also support the following va
 - `<serviceName>.nodeSelector` - [learn more](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector)
 - `<serviceName>.tolerations` - [learn more](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)
 - `<serviceName>.podSecurityContext` - [learn more](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod)
+- `<serviceName>.preShutdownPause` - override `sourcegraph.preShutdownPause` for this service
+- `<serviceName>.gracefulShutdownTimeout` - override `sourcegraph.gracefulShutdownTimeout` for this service
+- `<serviceName>.terminationGracePeriodSeconds` - override `sourcegraph.terminationGracePeriodSeconds` for this service
 - `<serviceName>.args` - override default container args
 - `<serviceName>.env` - consult `values.yaml` file
 - `<serviceName>.serivceAccount.create` - create service account for service
@@ -343,6 +346,7 @@ In addition to the documented values, all services also support the following va
 | sgTestConnection | object | `{"enabled":true}` | Enable the busybox connection test after deployment |
 | sourcegraph.affinity | object | `{}` | Global Affinity, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) |
 | sourcegraph.disableKubernetesSecrets | bool | `false` | Disable the creation of Kubernetes secrets objects |
+| sourcegraph.gracefulShutdownTimeout | string | `"15s"` | Total time Sourcegraph services have for the pre-shutdown pause and graceful drain |
 | sourcegraph.image.defaultTag | string | `"{{ .Chart.AppVersion }}"` | Global docker image tag |
 | sourcegraph.image.pullPolicy | string | `"IfNotPresent"` | Global docker image pull policy |
 | sourcegraph.image.repository | string | `"index.docker.io/sourcegraph"` | Global docker image registry or prefix |
@@ -354,9 +358,11 @@ In addition to the documented values, all services also support the following va
 | sourcegraph.nodeSelector | object | `{}` | Global NodeSelector, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) |
 | sourcegraph.podAnnotations | object | `{}` | Add extra annotations to attach to all pods |
 | sourcegraph.podLabels | object | `{}` | Add extra labels to attach to all pods |
+| sourcegraph.preShutdownPause | string | `"5s"` | Time Sourcegraph services continue accepting requests after shutdown begins so Kubernetes endpoint changes can propagate |
 | sourcegraph.priorityClassName | string | `""` | Assign a priorityClass to all pods (daemonSets, deployments, and statefulSets) |
 | sourcegraph.revisionHistoryLimit | int | `10` | Global deployment clean up policy, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#clean-up-policy) |
 | sourcegraph.serviceLabels | object | `{}` | Add extra labels to all services |
+| sourcegraph.terminationGracePeriodSeconds | int | `30` | Time Kubernetes allows Sourcegraph services to shut down before forced termination |
 | sourcegraph.tolerations | list | `[]` | Global Tolerations, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) |
 | storageClass.allowedTopologies | list | `[]` | Persistent volumes topology configuration, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/storage/storage-classes/#allowed-topologies) |
 | storageClass.create | bool | `true` | Enable creation of storageClass. Disable if you have your own existing storage class |

charts/sourcegraph/README.md.gotmpl

@@ -20,6 +20,9 @@ In addition to the documented values, all services also support the following va
 - `<serviceName>.nodeSelector` - [learn more](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector)
 - `<serviceName>.tolerations` - [learn more](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)
 - `<serviceName>.podSecurityContext` - [learn more](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod)
+- `<serviceName>.preShutdownPause` - override `sourcegraph.preShutdownPause` for this service
+- `<serviceName>.gracefulShutdownTimeout` - override `sourcegraph.gracefulShutdownTimeout` for this service
+- `<serviceName>.terminationGracePeriodSeconds` - override `sourcegraph.terminationGracePeriodSeconds` for this service
 - `<serviceName>.args` - override default container args
 - `<serviceName>.env` - consult `values.yaml` file
 - `<serviceName>.serivceAccount.create` - create service account for service

charts/sourcegraph/templates/_helpers/_gracefulShutdown.tpl

@@ -0,0 +1,29 @@
+{{/*
+Render the standard graceful-shutdown environment variables for a service.
+Service-specific values override global defaults. Explicit entries in the service's env map
+take precedence so existing configurations remain valid.
+*/}}
+{{- define "sourcegraph.gracefulShutdownEnv" -}}
+{{- $top := index . 0 -}}
+{{- $serviceName := index . 1 -}}
+{{- $service := index $top.Values $serviceName -}}
+{{- if not (hasKey $service.env "SRC_PRE_SHUTDOWN_PAUSE") }}
+- name: SRC_PRE_SHUTDOWN_PAUSE
+  value: {{ default $top.Values.sourcegraph.preShutdownPause $service.preShutdownPause | quote }}
+{{- end }}
+{{- if not (hasKey $service.env "SRC_GRACEFUL_SHUTDOWN_TIMEOUT") }}
+- name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT
+  value: {{ default $top.Values.sourcegraph.gracefulShutdownTimeout $service.gracefulShutdownTimeout | quote }}
+{{- end }}
+{{- end }}
+
+{{/* Render the pod termination deadline for a service. */}}
+{{- define "sourcegraph.terminationGracePeriodSeconds" -}}
+{{- $top := index . 0 -}}
+{{- $service := index $top.Values (index . 1) -}}
+{{- $terminationGracePeriodSeconds := $top.Values.sourcegraph.terminationGracePeriodSeconds -}}
+{{- if and (hasKey $service "terminationGracePeriodSeconds") (ne $service.terminationGracePeriodSeconds nil) -}}
+{{- $terminationGracePeriodSeconds = $service.terminationGracePeriodSeconds -}}
+{{- end -}}
+terminationGracePeriodSeconds: {{ $terminationGracePeriodSeconds }}
+{{- end }}

charts/sourcegraph/templates/_worker.tpl

@@ -64,9 +64,11 @@ spec:
         worker-replica: {{ $name | quote }}
         {{- end }}
     spec:
+      {{- include "sourcegraph.terminationGracePeriodSeconds" (list $top "worker") | nindent 6 }}
       containers:
       - name: worker
         env:
+        {{- include "sourcegraph.gracefulShutdownEnv" (list $top "worker") | nindent 8 }}
         {{- include "sourcegraph.redisConnection" $top | nindent 8 }}
         {{- if $allowlist }}
         - name: WORKER_JOB_ALLOWLIST

charts/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml

@@ -47,6 +47,7 @@ spec:
         app: sourcegraph-frontend
         deploy: sourcegraph
     spec:
+      {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "frontend") | nindent 6 }}
       initContainers:
       {{- if .Values.migrator.enabled }}
       - name: migrator
@@ -86,6 +87,7 @@ spec:
         {{- end }}
         args: {{- default (list "serve") .Values.frontend.args | toYaml | nindent 8 }}
         env:
+        {{- include "sourcegraph.gracefulShutdownEnv" (list . "frontend") | nindent 8 }}
         {{- if not .Values.sourcegraph.disableKubernetesSecrets }}
         {{- include "sourcegraph.databaseAuth" (list . "pgsql" "PG") | nindent 8 }}
         {{- include "sourcegraph.databaseAuth" (list . "codeIntelDB" "CODEINTEL_PG") | nindent 8 }}

charts/sourcegraph/templates/gitserver/gitserver.StatefulSet.yaml

@@ -43,12 +43,14 @@ spec:
         type: gitserver
         deploy: sourcegraph
     spec:
+      {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "gitserver") | nindent 6 }}
       containers:
       - name: gitserver
         args: {{- default (list "run") .Values.gitserver.args | toYaml | nindent 8 }}
         image: {{ include "sourcegraph.image" (list . "gitserver") }}
         imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }}
         env:
+        {{- include "sourcegraph.gracefulShutdownEnv" (list . "gitserver") | nindent 8 }}
         {{- include "sourcegraph.redisConnection" .| nindent 8 }}
         {{- range $name, $item := .Values.gitserver.env}}
         - name: {{ $name }}

charts/sourcegraph/templates/precise-code-intel/worker.Deployment.yaml

@@ -45,9 +45,11 @@ spec:
         deploy: sourcegraph
         app: precise-code-intel-worker
     spec:
+      {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "preciseCodeIntel") | nindent 6 }}
       containers:
       - name: precise-code-intel-worker
         env:
+        {{- include "sourcegraph.gracefulShutdownEnv" (list . "preciseCodeIntel") | nindent 8 }}
         {{- range $name, $item := .Values.preciseCodeIntel.env}}
         - name: {{ $name }}
           {{- $item | toYaml | nindent 10 }}

charts/sourcegraph/templates/searcher/searcher.StatefulSet.yaml

@@ -43,13 +43,15 @@ spec:
         deploy: sourcegraph
         app: searcher
     spec:
+      {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "searcher") | nindent 6 }}
       containers:
       - name: searcher
         {{- with .Values.searcher.args }}
         args:
           {{- toYaml . | nindent 8 }}
         {{- end }}
         env:
+        {{- include "sourcegraph.gracefulShutdownEnv" (list . "searcher") | nindent 8 }}
         {{- include "sourcegraph.redisConnection" .| nindent 8 }}
         {{- range $name, $item := .Values.searcher.env}}
         - name: {{ $name }}
@@ -154,4 +156,4 @@ spec:
       resources:
         requests:
           storage: {{ .Values.searcher.storageSize | default "26Gi" }}
-      storageClassName: {{ .Values.storageClass.name }}
+      storageClassName: {{ .Values.storageClass.name }}

charts/sourcegraph/templates/syntactic-code-intel/worker.Deployment.yaml

@@ -46,9 +46,11 @@ spec:
         deploy: sourcegraph
         app: syntactic-code-intel-worker
     spec:
+      {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "syntacticCodeIntel") | nindent 6 }}
       containers:
       - name: syntactic-code-intel-worker
         env:
+        {{- include "sourcegraph.gracefulShutdownEnv" (list . "syntacticCodeIntel") | nindent 8 }}
         {{- range $name, $item := .Values.syntacticCodeIntel.env}}
         - name: {{ $name }}
           {{- $item | toYaml | nindent 10 }}

charts/sourcegraph/templates/syntect-server/syntect-server.Deployment.yaml

@@ -45,9 +45,11 @@ spec:
         deploy: sourcegraph
         app: syntect-server
     spec:
+      {{- include "sourcegraph.terminationGracePeriodSeconds" (list . "syntectServer") | nindent 6 }}
       containers:
       - name: syntect-server
         env:
+        {{- include "sourcegraph.gracefulShutdownEnv" (list . "syntectServer") | nindent 8 }}
         {{- range $name, $item := .Values.syntectServer.env}}
         - name: {{ $name }}
           {{- $item | toYaml | nindent 10 }}

charts/sourcegraph/tests/gracefulShutdown_test.yaml

@@ -0,0 +1,125 @@
+suite: graceful shutdown
+release:
+  name: sourcegraph
+  namespace: sourcegraph
+tests:
+- it: should render global defaults for frontend
+  template: frontend/sourcegraph-frontend.Deployment.yaml
+  asserts:
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 30
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_PRE_SHUTDOWN_PAUSE
+        value: 5s
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT
+        value: 15s
+- it: should render global defaults for gitserver
+  template: gitserver/gitserver.StatefulSet.yaml
+  asserts:
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 30
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_PRE_SHUTDOWN_PAUSE
+        value: 5s
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT
+        value: 15s
+- it: should render service overrides for searcher
+  template: searcher/searcher.StatefulSet.yaml
+  set:
+    searcher:
+      preShutdownPause: 7s
+      gracefulShutdownTimeout: 25s
+      terminationGracePeriodSeconds: 35
+  asserts:
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 35
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_PRE_SHUTDOWN_PAUSE
+        value: 7s
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT
+        value: 25s
+- it: should preserve explicit env overrides
+  template: precise-code-intel/worker.Deployment.yaml
+  set:
+    preciseCodeIntel:
+      env:
+        SRC_PRE_SHUTDOWN_PAUSE:
+          value: 8s
+        SRC_GRACEFUL_SHUTDOWN_TIMEOUT:
+          value: 28s
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_PRE_SHUTDOWN_PAUSE
+        value: 8s
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT
+        value: 28s
+- it: should preserve a zero-second service termination override
+  template: precise-code-intel/worker.Deployment.yaml
+  set:
+    preciseCodeIntel.terminationGracePeriodSeconds: 0
+  asserts:
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 0
+- it: should render defaults for syntactic code intel
+  template: syntactic-code-intel/worker.Deployment.yaml
+  set:
+    syntacticCodeIntel.enabled: true
+  asserts:
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 30
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_PRE_SHUTDOWN_PAUSE
+        value: 5s
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT
+        value: 15s
+- it: should render defaults for worker replicas
+  template: worker/worker.Deployment.yaml
+  set:
+    worker:
+      replicas:
+      - jobs: [job1]
+  asserts:
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 30
+    documentIndex: 0
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 30
+    documentIndex: 1
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT
+        value: 15s
+    documentIndex: 0

charts/sourcegraph/tests/syntectServer_test.yaml

@@ -0,0 +1,43 @@
+suite: syntect server
+release:
+  name: sourcegraph
+  namespace: sourcegraph
+templates:
+- syntect-server/syntect-server.Deployment.yaml
+tests:
+- it: should render rollout-safe defaults
+  asserts:
+  - equal:
+      path: spec.strategy.type
+      value: RollingUpdate
+  - equal:
+      path: spec.strategy.rollingUpdate.maxSurge
+      value: 1
+  - equal:
+      path: spec.strategy.rollingUpdate.maxUnavailable
+      value: 0
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 30
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_PRE_SHUTDOWN_PAUSE
+        value: 5s
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: SRC_GRACEFUL_SHUTDOWN_TIMEOUT
+        value: 15s
+- it: should render replica and termination grace overrides
+  set:
+    syntectServer:
+      replicaCount: 2
+      terminationGracePeriodSeconds: 60
+  asserts:
+  - equal:
+      path: spec.replicas
+      value: 2
+  - equal:
+      path: spec.template.spec.terminationGracePeriodSeconds
+      value: 60

charts/sourcegraph/values.yaml

@@ -38,6 +38,12 @@ sourcegraph:
   # -- Global deployment clean up policy,
   # learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#clean-up-policy)
   revisionHistoryLimit: 10
+  # -- Time Sourcegraph services continue accepting requests after shutdown begins so Kubernetes endpoint changes can propagate
+  preShutdownPause: 5s
+  # -- Total time Sourcegraph services have for the pre-shutdown pause and graceful drain
+  gracefulShutdownTimeout: 15s
+  # -- Time Kubernetes allows Sourcegraph services to shut down before forced termination
+  terminationGracePeriodSeconds: 30
   # -- Add extra labels to all services
   serviceLabels: {}
   # -- Disable the creation of Kubernetes secrets objects
@@ -65,6 +71,10 @@ sourcegraph:
 #   serviceAnnotations: {}
 #   # Provide custom environment variables
 #   env: {}
+#   # Override the global graceful shutdown settings
+#   preShutdownPause: ""
+#   gracefulShutdownTimeout: ""
+#   terminationGracePeriodSeconds: null
 #   # Set resource requests / limits
 #   resources: {}
 #   containerSecurityContext: {}