feat(nvca): extend control-plane validator with HA checks, DaemonSet n2n, and route CR type check

deploy/helm/nvca-operator/nvca-operator/README.md

@@ -234,6 +234,12 @@ This release does not wire the catalog into backend selection. Runtime use requi
 | `clusterValidator.image.repository`                   | Cluster Validator container registry path, without tag                            | `""`                                   |
 | `clusterValidator.image.tag`                          | Cluster Validator container image tag                                             | `v2.0.0`                               |
 | `clusterValidator.image.pullPolicy`                   | K8s ImagePullPolicy for cluster-validator                                         | `IfNotPresent`                         |
+| `clusterValidator.role`                               | Check set: `control-plane`, or `compute-plane` or empty for the GPU checks        | `""`                                   |
+| `clusterValidator.openBaoNamespace`                   | Namespace holding OpenBao when it is not `vault-system`                           | `""`                                   |
+| `clusterValidator.envoyGatewayNamespace`              | Namespace holding Envoy Gateway when it is not `envoy-gateway-system`             | `""`                                   |
+| `clusterValidator.gatewayNames`                       | Every NVCF Gateway as `namespace/name`; replaces discovery for the LB and Tier-1 checks | `[]` (discovered from NVCF routes)     |
+| `clusterValidator.nodeToNodeProbeImage`               | Overlay probe image; needs `sh` and busybox-style `nc`, pullable without `imagePullSecrets` | `""` (`busybox:1.36`)                  |
+| `clusterValidator.tolerations`                        | Extra tolerations for the validator Job pods, added to the control-plane ones     | `[]`                                   |
 | `clusterValidator.schedule`                           | CronJob schedule (cron expression)                                                | `0 */3 * * *`                          |
 | `clusterValidator.configMapName`                      | ConfigMap name for user-defined network checks                                    | `cluster-validator-network-checks`     |
 | `clusterValidator.networkChecks`                      | Network check configuration (creates the ConfigMap automatically when set)        | `{}`                                   |

deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl

@@ -429,6 +429,12 @@ Usage: {{- $cv := include "nvcaop.clusterValidatorConfig" . | fromYaml -}}
     "image" (dict "repository" "" "tag" "" "pullPolicy" "IfNotPresent")
     "schedule" "0 */3 * * *"
     "configMapName" "cluster-validator-network-checks"
+    "role" ""
+    "openBaoNamespace" ""
+    "envoyGatewayNamespace" ""
+    "gatewayNames" (list)
+    "nodeToNodeProbeImage" ""
+    "tolerations" (list)
     "networkChecks" (dict)
     "resources" (dict
       "requests" (dict "cpu" "100m" "memory" "64Mi")
@@ -459,3 +465,119 @@ stg.nvcr.io/nvidia/nvcf-byoc/cluster-validator
 nvcr.io/nvidia/nvcf-byoc/cluster-validator
 {{- end -}}
 {{- end -}}
+
+{{/*
+The cluster-validator CronJob's Job spec. Under the control-plane role the
+operator also runs it once at startup, from the CronJob itself, so the two
+cannot drift.
+*/}}
+{{- define "nvcaop.clusterValidatorJobSpec" -}}
+{{- $cv := include "nvcaop.clusterValidatorConfig" . | fromYaml -}}
+parallelism: 1
+completions: 1
+backoffLimit: 2
+activeDeadlineSeconds: 600
+template:
+  metadata:
+    labels:
+      {{- include "nvcaop.baseSelectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: validation
+  spec:
+    serviceAccountName: {{ include "nvcaop.fullname" . }}-cluster-validator
+    automountServiceAccountToken: true
+    restartPolicy: Never
+    securityContext:
+      runAsUser: 65534
+      runAsGroup: 65534
+      fsGroup: 65534
+    {{- if or .Values.generateImagePullSecret (gt (len .Values.imagePullSecrets) 0) }}
+    imagePullSecrets:
+    {{- if .Values.generateImagePullSecret }}
+    - name: {{ (.Values.imagePullSecretName) | default "nvca-operator-image-pull" | quote }}
+    {{- end }}
+    {{- range .Values.imagePullSecrets }}
+    - name: {{ .name | quote }}
+    {{- end }}
+    {{- end }}
+    containers:
+      - name: cluster-validator
+        image: {{ include "nvcaop.clusterValidatorRepository" (dict "imageRepository" $cv.image.repository "defaultRepository" .Values.image.repository) }}:{{ default .Chart.AppVersion $cv.image.tag }}
+        imagePullPolicy: {{ $cv.image.pullPolicy }}
+        env:
+          - name: VALIDATOR_CONFIG_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: VALIDATOR_CONFIG_NAME
+            value: {{ $cv.configMapName | quote }}
+          # Namespace the NVCA agent watches for the metrics summary; kept
+          # separate from the config namespace so a config-namespace
+          # override can't redirect metrics.
+          - name: VALIDATOR_SUMMARY_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          # Selects the check set: "control-plane" runs the gateway,
+          # storage, overlay and HA checks; "compute-plane" or unset runs
+          # the GPU set, and the schema rejects anything else. Without
+          # this the control-plane checks are unreachable from the chart.
+          - name: VALIDATOR_ROLE
+            value: {{ $cv.role | quote }}
+          # The CronJob runs beside an installed stack, so an empty
+          # control plane is a failure here rather than pre-install.
+          - name: VALIDATOR_POST_INSTALL
+            value: "true"
+          {{- if $cv.openBaoNamespace }}
+          # Relocated OpenBao: without this the Tier-2 quorum check
+          # silently skips its StatefulSet.
+          - name: NVCF_OPENBAO_NAMESPACE
+            value: {{ $cv.openBaoNamespace | quote }}
+          {{- end }}
+          {{- if $cv.envoyGatewayNamespace }}
+          # Set when the stack's controllerNamespace differs from the
+          # Envoy Gateway chart default.
+          - name: NVCF_ENVOY_GATEWAY_NAMESPACE
+            value: {{ $cv.envoyGatewayNamespace | quote }}
+          {{- end }}
+          {{- with $cv.gatewayNames }}
+          # Replaces route-based discovery of the NVCF Gateways.
+          - name: NVCF_GATEWAY_NAMES
+            value: {{ join "," . | quote }}
+          {{- end }}
+          {{- if $cv.nodeToNodeProbeImage }}
+          - name: NVCF_N2N_PROBE_IMAGE
+            value: {{ $cv.nodeToNodeProbeImage | quote }}
+          {{- end }}
+        resources:
+          requests:
+            cpu: {{ $cv.resources.requests.cpu | quote }}
+            memory: {{ $cv.resources.requests.memory | quote }}
+          limits:
+            cpu: {{ $cv.resources.limits.cpu | quote }}
+            memory: {{ $cv.resources.limits.memory | quote }}
+        securityContext:
+          runAsNonRoot: true
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      {{- with $cv.tolerations }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+      {{- /* The operator's own tolerations go with its nodeSelector: pinned to
+             tainted nodes, the Job would otherwise never schedule. */}}
+      {{- with .Values.tolerations }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+    {{- if .Values.nodeSelector.value }}
+    nodeSelector:
+      {{ .Values.nodeSelector.key }}: {{ .Values.nodeSelector.value }}
+    {{- end }}
+{{- end }}

deploy/helm/nvca-operator/nvca-operator/templates/cronjob.yaml

@@ -36,72 +36,5 @@ spec:
         {{- include "nvcaop.baseSelectorLabels" . | nindent 8 }}
         app.kubernetes.io/component: validation
     spec:
-      parallelism: 1
-      completions: 1
-      backoffLimit: 2
-      activeDeadlineSeconds: 600
-      template:
-        metadata:
-          labels:
-            {{- include "nvcaop.baseSelectorLabels" . | nindent 12 }}
-            app.kubernetes.io/component: validation
-        spec:
-          serviceAccountName: {{ include "nvcaop.fullname" . }}-cluster-validator
-          automountServiceAccountToken: true
-          restartPolicy: Never
-          securityContext:
-            runAsUser: 65534
-            runAsGroup: 65534
-            fsGroup: 65534
-          {{- if or .Values.generateImagePullSecret (gt (len .Values.imagePullSecrets) 0) }}
-          imagePullSecrets:
-          {{- if .Values.generateImagePullSecret }}
-          - name: {{ (.Values.imagePullSecretName) | default "nvca-operator-image-pull" | quote }}
-          {{- end }}
-          {{- range .Values.imagePullSecrets }}
-          - name: {{ .name | quote }}
-          {{- end }}
-          {{- end }}
-          containers:
-            - name: cluster-validator
-              image: {{ include "nvcaop.clusterValidatorRepository" (dict "imageRepository" $cv.image.repository "defaultRepository" .Values.image.repository) }}:{{ default .Chart.AppVersion $cv.image.tag }}
-              imagePullPolicy: {{ $cv.image.pullPolicy }}
-              env:
-                - name: VALIDATOR_CONFIG_NAMESPACE
-                  valueFrom:
-                    fieldRef:
-                      fieldPath: metadata.namespace
-                - name: VALIDATOR_CONFIG_NAME
-                  value: {{ $cv.configMapName | quote }}
-                # Namespace the NVCA agent watches for the metrics summary; kept
-                # separate from the config namespace so a config-namespace
-                # override can't redirect metrics.
-                - name: VALIDATOR_SUMMARY_NAMESPACE
-                  valueFrom:
-                    fieldRef:
-                      fieldPath: metadata.namespace
-              resources:
-                requests:
-                  cpu: {{ $cv.resources.requests.cpu | quote }}
-                  memory: {{ $cv.resources.requests.memory | quote }}
-                limits:
-                  cpu: {{ $cv.resources.limits.cpu | quote }}
-                  memory: {{ $cv.resources.limits.memory | quote }}
-              securityContext:
-                runAsNonRoot: true
-                readOnlyRootFilesystem: true
-                allowPrivilegeEscalation: false
-                capabilities:
-                  drop: ["ALL"]
-          tolerations:
-            - key: node-role.kubernetes.io/control-plane
-              operator: Exists
-              effect: NoSchedule
-            - key: node-role.kubernetes.io/master
-              operator: Exists
-              effect: NoSchedule
-          {{- if .Values.nodeSelector.value }}
-          nodeSelector:
-            {{ .Values.nodeSelector.key }}: {{ .Values.nodeSelector.value }}
-          {{- end }}
+      {{- include "nvcaop.clusterValidatorJobSpec" . | nindent 6 }}
 {{- end }}

deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml

@@ -30,6 +30,16 @@ spec:
     metadata:
       labels:
         {{- include "nvcaop.baseSelectorLabels" . | nindent 8 }}
+      {{- if (include "nvcaop.clusterValidatorEnabled" .) }}
+      {{- $cv := include "nvcaop.clusterValidatorConfig" . | fromYaml }}
+      {{- if eq (lower (trim (toString $cv.role))) "control-plane" }}
+      annotations:
+        # The operator starts the validator's first run when it starts, once
+        # per Job spec. A change to that spec restarts the operator, so an
+        # upgrade that changes only the validator's values runs it too.
+        checksum/cluster-validator-job: {{ include "nvcaop.clusterValidatorJobSpec" . | sha256sum }}
+      {{- end }}
+      {{- end }}
     spec:
       serviceAccountName: {{ include "nvcaop.serviceAccountName" . }}
       automountServiceAccountToken: true
@@ -95,6 +105,25 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        # Deliberately pinned to the compute-plane check set, and deliberately
+        # NOT wired to clusterValidator.role. This chart is only installed on
+        # compute-plane clusters, and a failed validation is fatal here, so
+        # running the control-plane set would gate operator startup on
+        # control-plane HA: a single OpenBao pod stuck Terminating, or the
+        # Gateway API CRDs simply being absent on a compute-only cluster, would
+        # leave the operator in Init:CrashLoopBackOff.
+        - name: VALIDATOR_ROLE
+          value: "compute-plane"
+        {{- /* Normalized like the validator's own parseRole, so a role of
+               "Control-Plane" still keeps this container off the summary. */}}
+        {{- if eq (lower (trim (toString $cv.role))) "control-plane" }}
+        # The CronJob owns the summary under the control-plane role, so this
+        # container must not republish a compute-plane summary over it: the GPU
+        # keys would reappear and the control-plane keys be pruned on every
+        # operator restart. Preflight mode keeps the checks and drops the write.
+        - name: VALIDATOR_PREFLIGHT
+          value: "true"
+        {{- end }}
         resources:
           requests:
             cpu: {{ $cv.resources.requests.cpu | quote }}
@@ -123,6 +152,20 @@ spec:
               fieldPath: metadata.namespace
         - name: DEPLOYMENT_NAME
           value: {{ include "nvcaop.fullname" . | quote }}
+        {{- if (include "nvcaop.clusterValidatorEnabled" .) }}
+        {{- $cv := include "nvcaop.clusterValidatorConfig" . | fromYaml }}
+        # The agent publishes the cluster-validator metrics baseline only
+        # where the validator runs.
+        - name: NVCA_CLUSTER_VALIDATOR_ENABLED
+          value: "true"
+        {{- if eq (lower (trim (toString $cv.role))) "control-plane" }}
+        # The init container publishes no summary under the control-plane
+        # role, so the operator runs the CronJob once at startup instead. It is
+        # not a release resource, so the install does not wait on it.
+        - name: NVCA_CLUSTER_VALIDATOR_CRONJOB
+          value: {{ printf "%s-cluster-validator" (include "nvcaop.fullname" .) | quote }}
+        {{- end }}
+        {{- end }}
         - name: NGC_API_URL
           value: {{ .Values.ngcConfig.apiURL }}
         - name: NGC_SERVICE_KEY_FILE

deploy/helm/nvca-operator/nvca-operator/templates/rbac.yaml

@@ -43,6 +43,12 @@ rules:
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["get", "list", "create", "delete", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    # list: the node-to-node probe reads its pods' events to tell a slow
+    # first image pull, which publishes no pod IP until it returns, from a
+    # node that could not network the pod.
+    verbs: ["list"]
   - apiGroups: [""]
     resources: ["configmaps"]
     # get: read the network-checks config ConfigMap (its name is overridable
@@ -57,9 +63,23 @@ rules:
     # limit blast radius if the validator ServiceAccount is compromised.
     resourceNames: ["cluster-validator-summary"]
     verbs: ["update"]
+  - apiGroups: ["gateway.networking.k8s.io"]
+    # Read-only: the LoadBalancer and Tier-1 checks follow the NVCF routes'
+    # parentRefs to learn which Gateways are NVCF's, so another team's proxy
+    # in the shared Envoy namespace is not blamed on NVCF. gateways gives
+    # their GatewayClass, which is how a merged-gateways proxy is labelled.
+    resources: ["httproutes", "grpcroutes", "tcproutes", "udproutes", "gateways"]
+    verbs: ["get", "list"]
   - apiGroups: ["apps"]
-    resources: ["deployments", "daemonsets"]
+    resources: ["deployments", "statefulsets"]
+    # statefulsets: the Tier-2 quorum and placement check. Without it every
+    # control-plane namespace 403s and the critical row is permanently unknown.
     verbs: ["get", "list"]
+  - apiGroups: ["apps"]
+    resources: ["daemonsets"]
+    # create/delete: the node-to-node overlay probe runs a short-lived
+    # DaemonSet in a per-run namespace and deletes it in the same run.
+    verbs: ["get", "list", "create", "delete"]
   - apiGroups: ["admissionregistration.k8s.io"]
     resources:
       - mutatingwebhookconfigurations
@@ -69,7 +89,9 @@ rules:
     resources: ["networkpolicies"]
     verbs: ["get", "list", "create", "update", "delete"]
   - apiGroups: ["storage.k8s.io"]
-    resources: ["csidrivers"]
+    # storageclasses: the default-StorageClass check. Without it the critical
+    # row is permanently unknown on every run.
+    resources: ["csidrivers", "storageclasses"]
     verbs: ["get", "list"]
   - apiGroups: ["nvidia.com"]
     resources: ["clusterpolicies"]

deploy/helm/nvca-operator/nvca-operator/values.schema.json

@@ -1237,6 +1237,44 @@
                     "description": "ConfigMap name for user-defined network checks (reachability + network policy validation)",
                     "default": "cluster-validator-network-checks"
                 },
+                "role": {
+                    "type": "string",
+                    "description": "Check set: control-plane runs the gateway, storage, overlay and HA checks; compute-plane or empty runs the compute-plane checks. Matched case-insensitively; other values are rejected.",
+                    "pattern": "^\\s*([Cc][Oo][Nn][Tt][Rr][Oo][Ll]-[Pp][Ll][Aa][Nn][Ee]|[Cc][Oo][Mm][Pp][Uu][Tt][Ee]-[Pp][Ll][Aa][Nn][Ee])?\\s*$",
+                    "default": ""
+                },
+                "openBaoNamespace": {
+                    "type": "string",
+                    "description": "Namespace holding OpenBao when it is not vault-system",
+                    "default": ""
+                },
+                "envoyGatewayNamespace": {
+                    "type": "string",
+                    "description": "Namespace holding Envoy Gateway when it is not envoy-gateway-system",
+                    "default": ""
+                },
+                "gatewayNames": {
+                    "type": "array",
+                    "items": {
+                        "type": "string",
+                        "pattern": "^[^/\\s]+/[^/\\s]+$"
+                    },
+                    "description": "Every NVCF Gateway as namespace/name; replaces discovery from the NVCF routes for the LoadBalancer and Tier-1 checks",
+                    "default": []
+                },
+                "nodeToNodeProbeImage": {
+                    "type": "string",
+                    "description": "Image for the node-to-node overlay probe; needs sh and busybox-style nc and must be pullable without imagePullSecrets",
+                    "default": ""
+                },
+                "tolerations": {
+                    "type": "array",
+                    "items": {
+                        "type": "object"
+                    },
+                    "description": "Extra tolerations for the validator CronJob pods, added to the control-plane ones",
+                    "default": []
+                },
                 "networkChecks": {
                     "type": [
                         "object",