From effedfe04ce3af3cf571a5a728a4a2320e0fa943 Mon Sep 17 00:00:00 2001 From: Yuva Shankar <11082310+yuva29@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:06:01 +0000 Subject: [PATCH] docs: restructure OpenShift guide to align with GPU operator pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move OpenShift installation guide from docs/openshift/ to docs/installation/openshift-olm.md, aligning with the GPU operator's documentation structure. The guide now follows the standard Prerequisites → Installation → Configuration → Uninstallation flow. Changes: - Move and restructure OpenShift guide under Installation section - Add CVF fixes: pods/exec RBAC create verb, SYS_CHROOT capability, OpenShift SCC ClusterRoleBinding - Add known limitations page (modinfo srcversion verification) - Add driver verification to troubleshooting guide - Add OpenShift-specific requirements to CVF README (iommu=pt) - Add Workload Configuration section (iommu=pt MachineConfig, NMState) - Update cross-references in drivers.md - Add spelling dictionary entries for OpenShift terms - Remove dev install, implementation details, and other internal content from user-facing guide Co-Authored-By: Claude Opus 4 (1M context) --- .wordlist.txt | 13 +- docs/_static/cluster-validation-job.yaml | 26 +- docs/cluster_validation_framework/README.md | 15 + docs/drivers/drivers.md | 2 +- .../openshift-olm.md} | 796 +++++------------- docs/sphinx/_toc.yml.in | 18 +- docs/troubleshooting.md | 22 + 7 files changed, 303 insertions(+), 589 deletions(-) rename docs/{openshift/installation-guide.md => installation/openshift-olm.md} (60%) diff --git a/.wordlist.txt b/.wordlist.txt index 01950b74..0c9f94ff 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -97,6 +97,7 @@ GPUs Grafana gRPC Gurunathan +hoc honorLabels honorTimestamps host-device @@ -116,10 +117,13 @@ insecureSkipVerify installdefaultNFDRule intra io +iommu +IOMMU IOV ipam ipc IPC +IPs IPv JPG json @@ -141,8 +145,8 @@ Kustomize labeller Labeller learnings -LIF libionic +LIF lifecycle LSP LTS @@ -176,6 +180,8 @@ NFD nic NICCTL NICs +NMState +NNCP nodeAffinity NodeFeatureDiscovery nodelabeller @@ -202,12 +208,13 @@ OpenShift's OperatorGroup OperatorHub oyaml +passthrough Passthrough pci pds PDS -perf Pensando +perf Podman Pollara PostToolUse @@ -231,6 +238,7 @@ relatedImageSignPullSecret relatedImageWorker relatedImageWorkerPullSecret repo +RHCOS RHEL roce RoCE @@ -253,6 +261,7 @@ ServiceMonitor serviceType SessionEnd SessionStart +srcversion SR-IOV sriov staticAuthorization diff --git a/docs/_static/cluster-validation-job.yaml b/docs/_static/cluster-validation-job.yaml index 2b407a8a..0a79d7b1 100644 --- a/docs/_static/cluster-validation-job.yaml +++ b/docs/_static/cluster-validation-job.yaml @@ -164,7 +164,7 @@ data: imagePullPolicy: Always securityContext: capabilities: - add: ["IPC_LOCK"] + add: ["IPC_LOCK", "SYS_CHROOT"] envFrom: - configMapRef: name: cluster-validation-config @@ -290,10 +290,13 @@ rules: resources: ["events"] verbs: ["get", "list", "watch", "create", "update"] - # Allow listing pods by MPIJob controller wait steps + # Allow listing and managing pods, exec for SSH checks, logs for debugging - apiGroups: [""] - resources: ["pods", "pods/exec", "pods/log"] - verbs: ["get", "list", "watch"] + resources: ["pods", "pods/log"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["get", "list", "watch", "create"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -309,6 +312,21 @@ roleRef: name: cluster-validation-role apiGroup: rbac.authorization.k8s.io --- +# OpenShift only: grant privileged SCC to allow SYS_CHROOT capability for sshd in MPI workers. +# On vanilla Kubernetes this ClusterRoleBinding is created but has no effect (the referenced ClusterRole does not exist). +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-validation-scc-privileged +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:privileged +subjects: + - kind: ServiceAccount + name: cluster-validation-sa + namespace: default +--- apiVersion: batch/v1 kind: CronJob diff --git a/docs/cluster_validation_framework/README.md b/docs/cluster_validation_framework/README.md index 272fe2bf..15a8957e 100644 --- a/docs/cluster_validation_framework/README.md +++ b/docs/cluster_validation_framework/README.md @@ -38,6 +38,21 @@ The following prerequisites need to be done on each new node which is intended t Back-end networking should be set up on the new node, and connectivity to other nodes in the cluster should be configured. +### OpenShift-Specific Requirements + +On OpenShift clusters, the following additional configuration is required: + +* **`iommu=pt` kernel parameter** must be set via MachineConfig. Without it, GPU IPC + fails and RCCL tests crash. See + [OpenShift (OLM)](../installation/openshift-olm.md#iommu-pt-kernel-parameter) for the + MachineConfig YAML. + +### Back-end RoCE Network Configuration + +The back-end RoCE network should use **L3 (routed) networking** with each AINIC interface on a separate subnet. RoCE interfaces should be configured with **jumbo frames** (MTU 9000–9216) for optimal performance — this can be set via the [tuning CNI plugin](https://www.cni.dev/plugins/current/meta/tuning/) chained after the AMD Host Device CNI in the NetworkAttachmentDefinition. + +For detailed NIC, switch, and routing configuration guidance, refer to the [RoCE Cluster Network Configuration](https://instinct.docs.amd.com/projects/cluster-documentation/latest/how-to/roce-network-config.html) guide. + --- ## Architecture diff --git a/docs/drivers/drivers.md b/docs/drivers/drivers.md index 5ca93120..b17915c9 100644 --- a/docs/drivers/drivers.md +++ b/docs/drivers/drivers.md @@ -45,7 +45,7 @@ Some operating systems may include an in-tree `ionic` kernel module. That module * sudo update-initramfs -u * sudo reboot -**OpenShift:** The `spec.driver.blacklist` field is not supported on OpenShift. Instead, use a MachineConfig resource to blacklist the in-tree ionic module. See the [OpenShift Installation Guide](../openshift/installation-guide.md#2-blacklist-in-tree-ionic-driver-recommended) for details. +**OpenShift:** The `spec.driver.blacklist` field is not supported on OpenShift. Instead, use a MachineConfig resource to blacklist the in-tree ionic module. See the [OpenShift Installation Guide](../installation/openshift-olm.md#2-blacklist-in-tree-ionic-driver-recommended) for details. ``` For example: diff --git a/docs/openshift/installation-guide.md b/docs/installation/openshift-olm.md similarity index 60% rename from docs/openshift/installation-guide.md rename to docs/installation/openshift-olm.md index 534dac79..bde17119 100644 --- a/docs/openshift/installation-guide.md +++ b/docs/installation/openshift-olm.md @@ -1,91 +1,45 @@ -# AMD Network Operator - Production Deployment Guide - -## Table of Contents - -1. [Overview](#overview) -2. [Prerequisites](#prerequisites) - - [Infrastructure Requirements](#infrastructure-requirements) - - [Required Operators Installation](#required-operators-installation) - - [Development Tools (for building)](#development-tools-for-building) -3. [Architecture](#architecture) -4. [Cluster Configuration](#cluster-configuration) -5. [Installing the AMD Network Operator](#installing-the-amd-network-operator) - - [Official Installation (Production)](#official-installation-production) - - [Development Installation (Build from Source)](#development-installation-build-from-source) -6. [Post-Installation Verification](#post-installation-verification) -7. [Preparing Pre-Compiled Driver Images (Optional)](#preparing-pre-compiled-driver-images-optional) - - [Two Driver Image Build Methods](#two-driver-image-build-methods) - - [Method 1: RPM-based Build (Recommended)](#method-1-rpm-based-build-recommended) - - [Method 2: Source Image Build (Advanced)](#method-2-source-image-build-advanced) -8. [Deploying NetworkConfig CR](#deploying-networkconfig-cr) -9. [Validation](#validation) -10. [Updating the Operator](#updating-the-operator) -11. [Cleanup](#cleanup) -12. [Troubleshooting](#troubleshooting) -13. [Key Implementation Details](#key-implementation-details) -14. [Production Checklist](#production-checklist) +# OpenShift (OLM) ---- - -## Overview - -This guide provides production-ready steps for deploying the AMD Network Operator on OpenShift clusters using OLM (Operator Lifecycle Manager). This operator manages AMD network drivers (ionic, ionic_rdma, pds_core, tawk_ipc) using Kernel Module Management (KMM). - -**What this operator does**: - -- Automatically loads AMD network drivers on OpenShift CoreOS nodes -- Manages kernel module lifecycle through KMM -- Deploys device plugins for GPU-NIC integration -- Provides metrics and monitoring capabilities -- Supports RDMA and high-performance networking - -### Quick Start Summary - -**Time Required**: 30-45 minutes (excluding build time) - -**High-Level Steps**: +This guide covers deploying the AMD Network Operator on OpenShift clusters using OLM +(Operator Lifecycle Manager). The operator manages AMD network drivers (ionic, ionic_rdma, +pds_core, tawk_ipc) using Kernel Module Management (KMM). -1. Install NFD and KMM operators from OperatorHub *(5 min)* -2. Configure insecure registry (if needed) *(2 min)* -3. **Install AMD Network Operator**: - - **Production**: Install from OperatorHub *(5 min)* - - **Development**: Build and deploy from source *(15 min)* -4. Create NetworkConfig CR *(2 min)* -5. Verify drivers loaded on nodes *(5 min)* - -> 💡 **Quick Start**: For connected environments, you can skip directly to step 4 after installing the operator. KMM will automatically build driver images in-cluster using the OpenShift internal registry. - -**Key Requirements**: +## Prerequisites -- OpenShift 4.16+ with CoreOS -- NFD and KMM operators installed -- Container registry (insecure registry configured if internal) -- AMD Pensando NICs installed on nodes +### Infrastructure Requirements ---- +- OpenShift 4.16+ cluster with CoreOS nodes +- AMD Pensando network hardware +- Container registry accessible from the cluster +- Administrative access to OpenShift cluster -## Important Notes +### Required OpenShift Operators -> **TIP**: This guide uses production-style versioning (`v1.0.0-netop-beta`). Replace with your actual version tags. +The following OpenShift operators must be enabled before installing the AMD Network Operator: - +**Service CA Operator** — required for certificate signing and KMM webhook authentication: -> **WARNING**: Only install KMM operator **ONCE** in `openshift-kmm` namespace. Multiple instances cause conflicts. +```bash +oc get pods -A | grep service-ca +``` - +**Operator Lifecycle Manager (OLM)** — required for operator installation and dependencies: -> **REGISTRY**: Configure insecure registries at cluster level before starting. Images won't pull otherwise. +```bash +oc get pods -A | grep operator-lifecycle +``` ---- +**MachineConfig Operator** — required for configuring the ionic driver blacklist: -## Prerequisites +```bash +oc get pods -A | grep machine-config +``` -### Infrastructure Requirements +**Cluster Image Registry Operator** — required for driver image building and storage: -- OpenShift 4.16+ cluster with CoreOS nodes -- AMD Pensando network hardware -- Container registry accessible from the cluster -- Administrative access to OpenShift cluster +```bash +oc get pods -A | grep image-registry +``` ### Required Operators Installation @@ -160,7 +114,9 @@ kubectl get pods -n openshift-nfd | grep worker # Expected: nfd-worker pods Running on each node ``` -> **Note**: Without a `NodeFeatureDiscovery` instance, the NFD operator is installed but idle — no node feature detection or labeling occurs. +```{note} + Without a `NodeFeatureDiscovery` instance, the NFD operator is installed but idle — no node feature detection or labeling occurs. +``` #### 2. Install Kernel Module Management (KMM) @@ -193,23 +149,9 @@ kubectl get deployment -n openshift-kmm # kmm-operator-webhook 1/1 1 1 ``` -**⚠️ IMPORTANT**: Only install KMM **once** in the `openshift-kmm` namespace. Multiple KMM instances cause conflicts and module loading failures. +**IMPORTANT**: Only install KMM **once** in the `openshift-kmm` namespace. Multiple KMM instances cause conflicts and module loading failures. -### Development Tools (for building) - -- Docker or Podman -- Go 1.23+ -- make -- operator-sdk v1.32.0+ -- Git - -## Architecture - -```text -NetworkConfig CR → AMD Network Operator → KMM Module CR → KMM Operator → Driver Pods → Node (drivers loaded) -``` - -## Cluster Configuration +## Installation ### 1. Configure Insecure Registry (if using internal registry) @@ -224,7 +166,7 @@ kubectl patch image.config.openshift.io/cluster --type=merge \ -p "{\"spec\":{\"registrySources\":{\"insecureRegistries\":[\"${REGISTRY_URL}\"]}}}" ``` -**Note**: This configuration allows all nodes to pull from the specified registry without TLS verification. +**Note:** This configuration allows all nodes to pull from the specified registry without TLS verification. ### 2. Blacklist In-Tree Ionic Driver (Recommended) @@ -258,7 +200,7 @@ Save the above manifest to a file and apply it: oc apply -f ionic-module-blacklist.yaml ``` -**Note**: Applying a `MachineConfig` will trigger a rolling reboot of the worker nodes managed by the Machine Config Operator (MCO). The base64 content decodes to: +**Note:** Applying a `MachineConfig` will trigger a rolling reboot of the worker nodes managed by the Machine Config Operator (MCO). The base64 content decodes to: ```text blacklist ionic @@ -292,7 +234,7 @@ export KERNEL_VERSION="5.14.0-570.76.1.el9_6.x86_64" export RHEL_VERSION="9.6" # Operator versions -export OPERATOR_VERSION="v1.0.0-netop-beta" +export OPERATOR_VERSION="v1.2.1" # Registry configuration export REGISTRY_URL="registry.test.pensando.io:5000" @@ -312,18 +254,7 @@ export KERNEL_VERSION=$(kubectl get nodes -o jsonpath='{.items[0].status.nodeInf export DTK_IMAGE=$(kubectl get is -n openshift driver-toolkit -o jsonpath="{.spec.tags[?(@.name=='${KERNEL_VERSION}')].from.name}") ``` ---- - -## Installing the AMD Network Operator - -Choose the appropriate installation method based on your use case: - -- **Official Installation**: Install published operator from OperatorHub (recommended for production) -- **Development Installation**: Build and install from source (for development and testing) - ---- - -### Official Installation (Production) +### Install from OperatorHub This method installs the AMD Network Operator from Red Hat OperatorHub. Use this for production deployments when the operator is officially published. @@ -358,131 +289,186 @@ kubectl logs -f deployment/amd-network-operator-controller-manager -n openshift- #### Step 3: Proceed to Deployment -Once the operator is installed, proceed to [Deploying NetworkConfig CR](#deploying-networkconfig-cr) to start using the operator. +Once the operator is installed, proceed to the **Create NetworkConfig** section below to start using the operator. -> 💡 **Optional**: If you need to pre-build driver images (for air-gapped environments or external registries), see [Preparing Pre-Compiled Driver Images](#preparing-pre-compiled-driver-images-optional). +```{tip} +If you need to pre-build driver images (for air-gapped environments or external registries), see [Preparing Pre-Compiled Driver Images](#preparing-pre-compiled-driver-images-optional). +``` --- -### Development Installation (Build from Source) - -This method is for developers and testers who need to build and deploy the operator from source code. - -#### Prerequisites +The operator creates multiple service accounts for different components: -Ensure you have the following tools installed: +```bash +kubectl get sa -n openshift-amd-network -- Docker or Podman -- Go 1.23+ -- make -- operator-sdk v1.32.0+ -- Git +# Expected service accounts: +# - amd-network-operator-controller-manager +# - amd-network-operator-device-plugin +# - amd-network-operator-kmm-module-loader +# - amd-network-operator-node-labeller +# - amd-network-operator-metrics-exporter +# - amd-network-operator-config-manager +# - amd-network-operator-utils-container +``` -#### Step 1: Build Operator Image +--- -```bash -# Clone repository -git clone https://github.com/ROCm/network-operator.git -cd network-operator -git checkout +## Configuration -# Set image tags -export OPERATOR_IMG=${REGISTRY_URL}/amd-network-operator:${OPERATOR_VERSION} -export BUNDLE_IMG=${REGISTRY_URL}/amd-network-operator-bundle:${OPERATOR_VERSION} +### 1. Create NFD Rule for NIC Detection -# Build operator image -make docker-build IMG=${OPERATOR_IMG} +Create a `NodeFeatureRule` to instruct NFD to automatically label nodes that have AMD Pensando NICs: -# Push to registry -docker push ${OPERATOR_IMG} +```bash +cat < 💡 **What `operator-sdk run bundle` does**: This command automatically creates the CatalogSource, OperatorGroup, and Subscription resources needed by OLM. You don't need to create them manually! + metricsExporter: + enable: true + image: docker.io/rocm/device-metrics-exporter:v1.2.0 +EOF +``` -#### Step 4: Verify Installation +**Option 2: Source Image Build** (for air-gapped environments or when using pre-built source images): ```bash -# Verify CSV is in Succeeded phase -kubectl get csv -n openshift-amd-network +cat < ⚠️ **THIS SECTION IS OPTIONAL**: For most users with connected clusters, you can **skip this entire section** and proceed directly to [Deploying NetworkConfig CR](#deploying-networkconfig-cr). When you create a NetworkConfig CR, KMM will automatically build driver images in-cluster using the OpenShift internal registry. +```{warning} +For most users with connected clusters, you can **skip this entire section** and proceed directly to the **Create NetworkConfig** section below. When you create a NetworkConfig CR, KMM will automatically build driver images in-cluster using the OpenShift internal registry. +``` **When to use this section**: @@ -497,7 +483,9 @@ kubectl get sa -n openshift-amd-network - **Quick start/trial**: Want the fastest path to running drivers - **Using internal registry**: OpenShift's built-in registry is sufficient -> 💡 **WORKFLOW TIP**: If you do choose to pre-build images, you can do this in parallel while the operator deploys. The operator will wait idle until you create a NetworkConfig CR. +```{tip} +If you do choose to pre-build images, you can do this in parallel while the operator deploys. The operator will wait idle until you create a NetworkConfig CR. +``` --- @@ -653,7 +641,9 @@ Why push to an external registry? This method first builds a source container image, then KMM compiles modules from that source against the specific kernel. -> **AIR-GAPPED ENVIRONMENTS**: This approach is designed for air-gapped or disconnected environments where direct access to external repositories (like `repo.radeon.com`) is restricted. By building a source image first, all required driver sources are packaged into a container that can be transferred and used in isolated environments without internet access during module compilation. +```{note} +Air-gapped environments: This approach is designed for air-gapped or disconnected environments where direct access to external repositories (like `repo.radeon.com`) is restricted. By building a source image first, all required driver sources are packaged into a container that can be transferred and used in isolated environments without internet access during module compilation. +``` #### Step 5a: Use Pre-Built Source Images (Recommended) @@ -770,7 +760,9 @@ spec: sourceImageRepo: "docker.io/amdpsdo/amdnic-drivers" # Pre-built source images ``` -> 💡 **Note**: If using a custom/internal source image registry, replace `docker.io/amdpsdo/amdnic-drivers` with your registry path. +```{tip} +Note: If using a custom/internal source image registry, replace `docker.io/amdpsdo/amdnic-drivers` with your registry path. +``` **How it works**: @@ -781,172 +773,7 @@ spec: --- -## Verifying Service Accounts - -The operator creates multiple service accounts for different components: - -```bash -kubectl get sa -n openshift-amd-network - -# Expected service accounts: -# - amd-network-operator-controller-manager -# - amd-network-operator-device-plugin -# - amd-network-operator-kmm-module-loader -# - amd-network-operator-node-labeller -# - amd-network-operator-metrics-exporter -# - amd-network-operator-config-manager -# - amd-network-operator-utils-container -``` - -## Deploying NetworkConfig CR - -### 1. Create NFD Rule for NIC Detection - -Create a `NodeFeatureRule` to instruct NFD to automatically label nodes that have AMD Pensando NICs: - -```bash -cat < **Note**: This requires the NFD operator to be installed and a `NodeFeatureDiscovery` CR to be created (see [Prerequisites](#required-operators-installation)). NFD will automatically apply the `feature.node.kubernetes.io/amd-nic: "true"` label to any node with AMD Pensando PCI devices (vendor `1dd8`). No node reboot is required. - -### 2. Create NetworkConfig - -Create the NetworkConfig CR to deploy drivers on your nodes. Choose the configuration based on which build method you used: - -**Option 1: RPM-based Build** (if you used Method 1): - -```bash -cat < -- chroot /host lsmod | grep -E '^(ionic|pds_core|tawk_ipc)' +oc debug node/ -- chroot /host lsmod | grep -E '^(ionic|pds_core|tawk_ipc)' # Expected: ionic, ionic_rdma, pds_core, tawk_ipc modules loaded # 7. RDMA Devices Available -kubectl debug node/ -- chroot /host ls /sys/class/infiniband/ +oc debug node/ -- chroot /host ls /sys/class/infiniband/ # Expected: ionic_0, ionic_1, ... (one per NIC) ``` -## Updating the Operator +## Workload Configuration + +### iommu=pt Kernel Parameter + +GPU and RDMA workloads require IOMMU passthrough mode. Create a MachineConfig: + +```yaml +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 99-iommu-passthrough +spec: + kernelArguments: + - iommu=pt +``` + +```{warning} +This triggers a rolling reboot of all nodes in the MachineConfigPool. +``` -### Update to New Version +Verify after reboot: ```bash -# Build new operator image -export NEW_VERSION=v1.0.1-netop-beta -export OPERATOR_IMG=${REGISTRY_URL}/amd-network-operator:${NEW_VERSION} -export BUNDLE_IMG=${REGISTRY_URL}/amd-network-operator-bundle:${NEW_VERSION} - -make docker-build IMG=${OPERATOR_IMG} -docker push ${OPERATOR_IMG} - -# Build new bundle -make bundle-build \ - IMG=${OPERATOR_IMG} \ - BUNDLE_IMG=${BUNDLE_IMG} \ - PROJECT_VERSION=${NEW_VERSION} - -make bundle-push BUNDLE_IMG=${BUNDLE_IMG} - -# Update via operator-sdk -./bin/operator-sdk run bundle-upgrade ${BUNDLE_IMG} \ - --use-http \ - --skip-tls \ - -n openshift-amd-network +cat /proc/cmdline | grep iommu +dmesg | grep "Default domain type" ``` -## Cleanup +### Host NIC IP Configuration + +On OpenShift (RHCOS), use `nmstate` for configuring static IPs on host NIC interfaces. +For testing, apply directly: `sudo nmstatectl apply .yaml`. For production, use the +NMState Operator with `NodeNetworkConfigurationPolicy` (NNCP) CRDs. + +For backend network configuration options, see the +[RoCE Cluster Network Configuration Guide](https://instinct.docs.amd.com/projects/cluster-documentation/latest/how-to/roce-network-config.html). + +## Uninstallation ### Remove Operator Completely @@ -1029,188 +866,3 @@ kubectl delete catalogsource amd-network-operator-catalog -n openshift-marketpla # Delete namespace (optional) kubectl delete namespace openshift-amd-network ``` - -## Troubleshooting - -### Common Issues and Solutions - -#### 1. operator-sdk: TLS Error with Insecure Registry - -**Problem**: - -```text -http: server gave HTTP response to HTTPS client -``` - -**Root Cause**: operator-sdk running locally doesn't know about cluster's insecure registry configuration. - -**Solution**: Use `--use-http` and `--skip-tls` flags: - -```bash -./bin/operator-sdk run bundle ${BUNDLE_IMG} --use-http --skip-tls -n -``` - ---- - -#### 2. OperatorGroup Conflict - -**Problem**: - -```text -csv failed: reason: "InterOperatorGroupOwnerConflict" -intersecting operatorgroups provide the same apis -``` - -**Root Cause**: Another operator instance already exists providing the same CRDs. - -**Solution**: Only one operator instance allowed per cluster: - -1. Check existing operators: `kubectl get csv -A | grep amd-network` -2. Either use existing namespace or cleanup old deployment first -3. Delete test namespaces if created during troubleshooting - ---- - -#### 3. Image Pull Failures - -**Problem**: Pods stuck in `ImagePullBackOff` or `ErrImagePull` - -**Root Cause**: Registry not accessible or missing credentials. - -**Solution**: - -1. **For insecure registries**: Verify configuration - - ```bash - kubectl get image.config.openshift.io/cluster -o yaml | grep insecureRegistries - ``` - -2. **For authenticated registries**: Check pull secrets - - ```bash - kubectl get secret -n openshift-amd-network | grep pull - ``` - -3. **Test registry access** from node: - - ```bash - kubectl debug node/ -- chroot /host podman pull --tls-verify=false - ``` - ---- - -#### 4. Driver Modules Not Loading - -**Problem**: `lsmod` shows no ionic modules on node - -**Diagnostic Steps**: - -```bash -# 1. Check KMM Module status -kubectl get module -n openshift-amd-network -o yaml - -# 2. Check if worker pods ran -kubectl get pods -n openshift-amd-network | grep worker - -# 3. Check worker pod logs -kubectl logs -n openshift-amd-network - -# 4. Verify node selector -kubectl get module -n openshift-amd-network -o jsonpath='{.spec.selector}' -kubectl get nodes --show-labels | grep