Refactor ssh known hosts

Dockerfile

@@ -5,7 +5,7 @@ ARG TARGETARCH
 ENV ASSETS_DIR=/usr/local/bin \
     HOME=/home/codebase-operator \
     OPERATOR=/usr/local/bin/codebase-operator \
-    SSH_KNOWN_HOSTS=/home/codebase-operator/.ssh/known_hosts \
+    SSH_KNOWN_HOSTS=/etc/codebase-operator/ssh/ssh_known_hosts \
     USER_NAME=codebase-operator \
     USER_UID=1001
 

api/v1/git_server_types.go

@@ -32,6 +32,9 @@ type GitServerSpec struct {
 	//   - secretString: Webhook secret for validating webhook requests
 	//   - username: Git username to override the default GitUser
 	// For Gerrit provider, only id_rsa key is required and used.
+	// When id_rsa is present the operator connects over SSH and verifies the server's
+	// host key against the operator's ssh-known-hosts ConfigMap; hosts other than
+	// github.com, gitlab.com and bitbucket.org must be added there first.
 	// +kubebuilder:example:=my-git-credentials
 	// +required
 	NameSshKeySecret string `json:"nameSshKeySecret"`

config/crd/bases/v2.edp.epam.com_gitservers.yaml

@@ -83,6 +83,9 @@ spec:
                     - secretString: Webhook secret for validating webhook requests
                     - username: Git username to override the default GitUser
                   For Gerrit provider, only id_rsa key is required and used.
+                  When id_rsa is present the operator connects over SSH and verifies the server's
+                  host key against the operator's ssh-known-hosts ConfigMap; hosts other than
+                  github.com, gitlab.com and bitbucket.org must be added there first.
                 example: my-git-credentials
                 type: string
               skipWebhookSSLVerification:

controllers/gitserver/ssh.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/epam/edp-codebase-operator/v2/pkg/gerrit"
 	"github.com/epam/edp-codebase-operator/v2/pkg/model"
+	"github.com/epam/edp-codebase-operator/v2/pkg/sshhostkey"
 	"github.com/epam/edp-codebase-operator/v2/pkg/util"
 )
 
@@ -33,6 +34,8 @@ func checkGitServerConnection(data gitSshData, log logr.Logger) error {
 		c *ssh.Client
 	)
 
+	// NewSession already enriches host key failures; enriching again here would
+	// repeat the whole remediation sentence inside GitServer .status.error.
 	if s, c, err = sshClient.NewSession(); err != nil {
 		return fmt.Errorf("failed to create ssh session: %w", err)
 	}
@@ -61,12 +64,18 @@ func sshInitFromSecret(data gitSshData, logger logr.Logger) (*gerrit.SSHClient,
 		return nil, err
 	}
 
+	hostKeyCallback, hostKeyAlgorithms, err := sshhostkey.ClientConfig(data.Host, data.Port)
+	if err != nil {
+		return nil, err
+	}
+
 	sshConfig := &ssh.ClientConfig{
 		User: data.User,
 		Auth: []ssh.AuthMethod{
 			sshAuth,
 		},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		HostKeyCallback:   hostKeyCallback,
+		HostKeyAlgorithms: hostKeyAlgorithms,
 	}
 
 	cl := &gerrit.SSHClient{

controllers/gitserver/ssh_test.go

@@ -1,11 +1,53 @@
 package gitserver
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/go-logr/logr"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func Test_sshInitFromSecret_verifiesHostKeys(t *testing.T) {
+	knownHosts := filepath.Join(t.TempDir(), "ssh_known_hosts")
+	require.NoError(t, os.WriteFile(knownHosts, []byte(
+		"[git.example.com]:2222 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\n",
+	), 0o600))
+	t.Setenv("SSH_KNOWN_HOSTS", knownHosts)
+
+	client, err := sshInitFromSecret(gitSshData{
+		Host: "git.example.com",
+		User: "git",
+		Key:  testKey,
+		Port: 2222,
+	}, logr.Discard())
+	require.NoError(t, err)
+
+	// The connection must never be established without a host key check.
+	require.NotNil(t, client.Config.HostKeyCallback)
+	// Algorithms must be narrowed to what is on file, otherwise a server offering
+	// an unpinned key type fails as a mismatch rather than as a missing entry.
+	assert.Equal(t, []string{"ssh-ed25519"}, client.Config.HostKeyAlgorithms)
+}
+
+func Test_sshInitFromSecret_failsWithoutKnownHosts(t *testing.T) {
+	t.Setenv("SSH_KNOWN_HOSTS", filepath.Join(t.TempDir(), "absent"))
+
+	client, err := sshInitFromSecret(gitSshData{
+		Host: "git.example.com",
+		User: "git",
+		Key:  testKey,
+		Port: 22,
+	}, logr.Discard())
+
+	require.Error(t, err)
+	assert.Nil(t, client)
+	// This message reaches the user through GitServer .status.error.
+	assert.Contains(t, err.Error(), "known_hosts")
+}
+
 func Test_publicKey(t *testing.T) {
 	tests := []struct {
 		name    string

deploy-templates/README.md

@@ -44,6 +44,8 @@ A Helm chart for KubeRocketCI Codebase Operator
 | jira.name | string | `"jira"` | JiraServer CR name |
 | jira.quickLink | object | `{"enabled":true}` | Enable creation of QuickLink for Jira |
 | jira.rootUrl | string | `"https://jiraeu.example.com"` | URL to Jira server |
+| knownHosts.entries | string | `""` (no self-hosted servers pinned) | Host keys for self-hosted git servers, in known_hosts format, one per line. Obtain them with `ssh-keyscan -t rsa,ecdsa,ed25519 -p <port> <host>` and verify the fingerprints out-of-band before trusting them. Servers on a port other than 22 must use the bracket form, e.g. `[git.example.com]:2222 ssh-ed25519 AAAA...`. |
+| knownHosts.includeDefaultProviders | bool | `true` | Include the shipped host keys for github.com, gitlab.com and bitbucket.org. Disable only if you pin these hosts yourself through `entries`. |
 | name | string | `"codebase-operator"` | component name |
 | nodeSelector | object | `{}` |  |
 | podLabels | object | `{}` | Labels to be added to the pod |

deploy-templates/crds/v2.edp.epam.com_gitservers.yaml

@@ -83,6 +83,9 @@ spec:
                     - secretString: Webhook secret for validating webhook requests
                     - username: Git username to override the default GitUser
                   For Gerrit provider, only id_rsa key is required and used.
+                  When id_rsa is present the operator connects over SSH and verifies the server's
+                  host key against the operator's ssh-known-hosts ConfigMap; hosts other than
+                  github.com, gitlab.com and bitbucket.org must be added there first.
                 example: my-git-credentials
                 type: string
               skipWebhookSSLVerification:

deploy-templates/templates/deployment.yaml

@@ -42,11 +42,19 @@ spec:
             - containerPort: 9443
               name: webhook-server
               protocol: TCP
+          {{- end }}
           volumeMounts:
+            {{- if .Values.enableWebhooks }}
             - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: cert
               readOnly: true
-          {{- end }}
+            {{- end }}
+            # A directory mount, not subPath: the kubelet only refreshes in-place
+            # directory mounts, so host keys added to the ConfigMap take effect
+            # without restarting the operator.
+            - mountPath: /etc/codebase-operator/ssh
+              name: ssh-known-hosts
+              readOnly: true
           imagePullPolicy: "{{ .Values.imagePullPolicy }}"
           {{- if .Values.securityContext }}
           securityContext: {{ toYaml .Values.securityContext | nindent 12 }}
@@ -76,6 +84,8 @@ spec:
               value: {{ .Values.enableWebhooks | quote }}
             - name: BRANCH_STALE_CHECK_INTERVAL
               value: {{ .Values.branchStaleCheckInterval | quote }}
+            - name: SSH_KNOWN_HOSTS
+              value: /etc/codebase-operator/ssh/ssh_known_hosts
 {{ toYaml .Values.envs | indent 12 }}
           resources:
 {{ toYaml .Values.resources | indent 12 }}
@@ -91,10 +101,13 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.enableWebhooks }}
       volumes:
+        {{- if .Values.enableWebhooks }}
         - name: cert
           secret:
             defaultMode: 420
             secretName: {{ .Values.name }}-webhook-certs
-      {{- end }}
+        {{- end }}
+        - name: ssh-known-hosts
+          configMap:
+            name: {{ .Values.name }}-ssh-known-hosts

deploy-templates/templates/ssh-known-hosts-cm.yaml

@@ -0,0 +1,33 @@
+{{/*
+Always rendered, even when it holds no entries: the operator resolves
+SSH_KNOWN_HOSTS to a file inside it, and a missing file produces a generic
+"unable to find any valid known_hosts file" error instead of the actionable
+per-host message.
+
+Edits are picked up by a running operator without a restart - see the
+volumeMount comment in deployment.yaml.
+*/}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.name }}-ssh-known-hosts
+  labels:
+    {{- include "codebase-operator.labels" . | nindent 4 }}
+data:
+  ssh_known_hosts: |
+    # Host keys for the public git providers, verified against the keys published
+    # by each vendor. Regenerate with: ssh-keyscan -t rsa,ecdsa,ed25519 <host>
+    {{- if .Values.knownHosts.includeDefaultProviders }}
+    github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+    github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+    github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
+    gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY=
+    gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf
+    gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9
+    bitbucket.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPIQmuzMBuKdWeF4+a2sjSSpBK0iqitSQ+5BM9KhpexuGt20JpTVM7u5BDZngncgrqDMbWdxMWWOGtZ9UgbqgZE=
+    bitbucket.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIazEu89wgQZ4bqs3d63QSMzYVa0MuJ2e2gKTKqu+UUO
+    bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQeJzhupRu0u0cdegZIa8e86EG2qOCsIsD1Xw0xSeiPDlCr7kq97NLmMbpKTX6Esc30NuoqEEHCuc7yWtwp8dI76EEEB1VqY9QJq6vk+aySyboD5QF61I/1WeTwu+deCbgKMGbUijeXhtfbxSxm6JwGrXrhBdofTsbKRUsrN1WoNgUa8uqN1Vx6WAJw1JHPhglEGGHea6QICwJOAr/6mrui/oB7pkaWKHj3z7d1IC4KWLtY47elvjbaTlkN04Kc/5LFEirorGYVbt15kAUlqGM65pk6ZBxtaO3+30LVlORZkxOh+LKL/BvbZ/iRNhItLqNyieoQj/uh/7Iv4uyH/cV/0b4WDSd3DptigWq84lJubb9t/DnZlrJazxyDCulTmKdOR7vs9gMTo+uoIrPSb8ScTtvw65+odKAlBj59dhnVp9zd7QUojOpXlL62Aw56U4oO+FALuevvMjiWeavKhJqlR7i5n9srYcrNV7ttmDw7kf/97P5zauIhxcjX+xHv4M=
+    {{- end }}
+    {{- with .Values.knownHosts.entries }}
+    {{- tpl . $ | nindent 4 }}
+    {{- end }}

deploy-templates/values.yaml

@@ -88,3 +88,17 @@ enableWebhooks: true
 # marking missing ones with the Stale condition and the app.edp.epam.com/stale label.
 # Accepts Go duration strings (e.g. 24h, 30m); "0" disables the check.
 branchStaleCheckInterval: 24h
+
+# SSH host key verification. Every SSH connection the operator makes is verified
+# against these entries; there is no way to disable verification. GitServers that
+# authenticate with a token over HTTPS are unaffected.
+knownHosts:
+  # -- Include the shipped host keys for github.com, gitlab.com and bitbucket.org.
+  # Disable only if you pin these hosts yourself through `entries`.
+  includeDefaultProviders: true
+  # -- Host keys for self-hosted git servers, in known_hosts format, one per line.
+  # Obtain them with `ssh-keyscan -t rsa,ecdsa,ed25519 -p <port> <host>` and verify
+  # the fingerprints out-of-band before trusting them. Servers on a port other than
+  # 22 must use the bracket form, e.g. `[git.example.com]:2222 ssh-ed25519 AAAA...`.
+  # @default -- `""` (no self-hosted servers pinned)
+  entries: ""

docs/api.md

@@ -1346,7 +1346,10 @@ Optional keys:
   - id_rsa: SSH private key for Git operations over SSH
   - secretString: Webhook secret for validating webhook requests
   - username: Git username to override the default GitUser
-For Gerrit provider, only id_rsa key is required and used.<br/>
+For Gerrit provider, only id_rsa key is required and used.
+When id_rsa is present the operator connects over SSH and verifies the server's
+host key against the operator's ssh-known-hosts ConfigMap; hosts other than
+github.com, gitlab.com and bitbucket.org must be added there first.<br/>
         </td>
         <td>true</td>
       </tr><tr>

docs/ssh-known-hosts.md

@@ -0,0 +1,130 @@
+# SSH host key verification
+
+Every SSH connection the codebase-operator makes to a git server is verified
+against a `known_hosts` file. This includes repository clone, fetch and push,
+the packless branch operations, the GitServer connectivity check, and Gerrit
+administrative commands.
+
+**Verification cannot be disabled.** An operator that accepts any host key can be
+made to hand source code and credentials to whoever controls the network path.
+
+GitServers that authenticate with a token over HTTPS are not affected — they
+never open an SSH connection. Only GitServers whose credentials secret contains
+an `id_rsa` key are.
+
+## Where the host keys live
+
+The Helm chart creates a ConfigMap named `<release>-ssh-known-hosts` and mounts
+it into the operator at `/etc/codebase-operator/ssh`. The operator reads
+`/etc/codebase-operator/ssh/ssh_known_hosts`, via the `SSH_KNOWN_HOSTS`
+environment variable set by the chart.
+
+The ConfigMap ships with host keys for `github.com`, `gitlab.com` and
+`bitbucket.org`, so SSH GitServers pointing at those providers work with no
+further configuration.
+
+The ConfigMap is mounted as a directory rather than with `subPath`, so the
+kubelet refreshes it in place and the operator re-reads it on the next
+connection. **Adding a host key does not require restarting the operator.**
+
+## Adding a self-hosted git server
+
+Collect the server's host keys:
+
+```sh
+ssh-keyscan -t rsa,ecdsa,ed25519 git.example.com
+```
+
+For a server on a port other than 22, pass `-p` and note that the output uses
+the bracket form, which is what `known_hosts` requires:
+
+```sh
+ssh-keyscan -t rsa,ecdsa,ed25519 -p 2222 git.example.com
+# [git.example.com]:2222 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA...
+```
+
+Verify the fingerprints against a source other than the connection you just
+made — your git server's documentation or its administrator:
+
+```sh
+ssh-keyscan -t rsa,ecdsa,ed25519 git.example.com | ssh-keygen -lf -
+```
+
+`ssh-keyscan` output is only as trustworthy as the network it ran over. Pinning
+a key you scanned through a compromised path pins the attacker's key.
+
+Then add the lines to the chart values:
+
+```yaml
+knownHosts:
+  entries: |
+    [git.example.com]:2222 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA...
+    [git.example.com]:2222 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAB...
+```
+
+Or, for an immediate fix without a Helm upgrade, edit the ConfigMap directly —
+note that a subsequent `helm upgrade` will overwrite it, so fold the entry into
+your values afterwards:
+
+```sh
+kubectl edit configmap <release>-ssh-known-hosts
+```
+
+## Which port to pin
+
+Pin the port the operator actually connects on, which is not always
+`spec.sshPort`:
+
+| Operation | Port used | known_hosts entry |
+|---|---|---|
+| GitServer connectivity check, and all Gerrit access | `spec.sshPort` | `[git.example.com]:2222 ...` |
+| Repository clone, fetch, push and branch operations for github, gitlab and bitbucket providers | always 22 | `git.example.com ...` |
+
+The second row is a consequence of the repository URL format for those
+providers: `GetSSHUrl` builds the scp-style `git@host:path.git`, which carries
+no port, so the SSH client uses 22. `spec.sshPort` is honoured only for Gerrit
+URLs and for the connectivity check.
+
+**If your git server's `sshPort` is not 22, pin both forms**, or the GitServer
+will report healthy while codebase operations fail (or the reverse). Scan each
+port separately:
+
+```sh
+ssh-keyscan -t rsa,ecdsa,ed25519 -p 2222 git.example.com   # [git.example.com]:2222 ...
+ssh-keyscan -t rsa,ecdsa,ed25519 git.example.com           # git.example.com ...
+```
+
+Both usually return the same keys, since it is normally the same server reached
+on two ports.
+
+## Diagnosing failures
+
+A GitServer whose host key is missing or wrong reports the reason in its status:
+
+```sh
+kubectl get gitserver <name> -o jsonpath='{.status.error}'
+```
+
+| Message | Meaning | Action |
+|---|---|---|
+| `SSH host key for <host> is not present in known_hosts` | The server is not pinned | Add its keys as above |
+| `SSH host key mismatch for <host>` | The server presented a key that differs from the pinned one | **Do not simply replace the entry.** Either the server was rekeyed or the connection is being intercepted. Confirm the new key with the server's administrator first |
+| `failed to load SSH known_hosts` | The file is missing or unreadable | Check that the ConfigMap exists and is mounted |
+
+The same messages appear on `Codebase` and `CodebaseBranch` status when the
+failure happens during a repository operation.
+
+## Upgrading from a release without host key verification
+
+Before upgrading, list the GitServers that use SSH:
+
+```sh
+kubectl get gitservers -o json | jq -r '.items[] | select(.spec.gitProvider) |
+  "\(.metadata.name) \(.spec.gitHost):\(.spec.sshPort) secret=\(.spec.nameSshKeySecret)"'
+```
+
+For each one whose secret contains an `id_rsa` key and whose host is not
+github.com, gitlab.com or bitbucket.org, add its host keys to
+`knownHosts.entries` as part of the upgrade. GitServers left unpinned will
+report `connected: false` until their keys are added; no data is lost and the
+operator recovers on the next reconcile once the entry is present.