helm: atelet mounts /var/lib/kubelet/plugins with HostToContainer propagation

.agents/skills/update-against-main/SKILL.md

@@ -0,0 +1,29 @@
+---
+name: update-against-main
+description: Merge agent-substrate/substrate main into the kagent-dev/substrate fork's main branch, resolve conflicts, validate the result, and safely update the fork. Use only when explicitly synchronizing the fork's main branch with upstream main. Do not use for updating, rebasing, or resolving conflicts in feature branches or pull requests.
+---
+
+# Update Against Main
+
+This skill applies only to synchronizing the fork's `main` branch. Do not invoke it for a feature branch or PR merely because that branch is behind or conflicts with `main`.
+
+1. Confirm the worktree, current branch, tracking branch, and remotes. Do not disturb unrelated changes.
+2. Fetch `origin/main` and `upstream/main`, inspect their divergence, and create a dated backup branch from `origin/main`.
+3. Rebuild `main` from `upstream/main` by replaying only intentional fork feature commits in dependency order. Drop merge commits and fork commits superseded by upstream.
+4. Resolve conflicts in favor of current upstream APIs while preserving the remaining fork features. Inspect the resulting diff and linear history.
+5. Keep Helm charts synchronized with their corresponding manifests. When either changes, inspect and update the other while preserving intentional Helm templating and conditionals, then run `make verify-helm-template` and `make verify-crd-chart` and compare any relevant resources not covered by those checks.
+6. Run `make test` and `make verify`.
+7. Run the real Kind E2E matrix from `.github/workflows/pr-workflow.yaml`, but use agentgateway for all fork testing:
+   - Recreate the cluster with `hack/create-kind-cluster.sh`.
+   - Install the control plane with `hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=agentgateway`.
+   - Deploy the micro-VM demo with `hack/run-microvm-demo-kind.sh --skip-control-plane` so it does not reinstall the control plane.
+   - Deploy the gVisor counter demo and both standard egress demos.
+   - The full gVisor suite: `hack/run-e2e-kind.sh -v -args --no-color`
+   - The full micro-VM suite with the CI environment: `E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh -v -args --no-color`
+   - Switch egress to agentgateway sdsmint, then run the MITM trust and targeted networking lanes for both runtimes exactly as the workflow specifies.
+   - Verify the live router and egress workloads use agentgateway. Never use Envoy for fork validation.
+8. Treat `go test ./internal/e2e/...` without `-args --e2e` as compilation/package testing, not E2E coverage.
+9. Do not push when unit, verification, or E2E checks fail or cannot run. Report the exact blocker instead.
+10. After all checks pass, verify the worktree and rewritten commits, then update the fork with `git push --force-with-lease origin main`. Never use an unguarded force push.
+
+Use the current CI workflow as the source of truth for cluster setup, images, demos, runtime coverage, and environment variables, with the agentgateway-only override above. Never claim E2E passed unless workloads ran against the cluster.

.github/workflows/helm-e2e.yaml

@@ -0,0 +1,116 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: helm-e2e
+on:
+  pull_request:
+  push:
+    branches: [main]
+permissions:
+  contents: read
+jobs:
+  e2e-test:
+    runs-on: ubuntu-latest
+    env:
+      VERSION: helm-e2e
+    steps:
+    - name: Checkout
+      uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+    - name: Setup Go
+      uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+      with:
+        go-version-file: go.mod
+    - name: Setup Helm
+      uses: azure/setup-helm@v4
+    - name: Cache micro-VM assets
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: bin/microvm-assets/amd64
+        key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }}
+    - name: Enable KVM
+      run: |
+        echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+          | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+        sudo udevadm control --reload-rules
+        sudo udevadm trigger --name-match=kvm
+    - name: Create cluster
+      run: hack/create-kind-cluster.sh
+    - name: Label nodes with the installed version
+      run: kubectl label nodes --all ate.dev/substrate-version=${VERSION}
+    - name: Create install namespace
+      run: kubectl create namespace ate-system
+    - name: Install observability fixtures
+      run: |
+        kubectl apply -f manifests/ate-install/kind/otel-collector.yaml
+        kubectl apply -f manifests/ate-install/kind/prometheus.yaml
+    - name: Build chart images
+      run: |
+        for component in ateapi atecontroller atelet podcertcontroller atenet; do
+          KO_DOCKER_REPO="localhost:5001/${component}" \
+            ./hack/run-tool.sh ko build --bare --tags helm-e2e \
+              --platform linux/amd64 "./cmd/${component}"
+        done
+    - name: Install Agent Substrate with Helm
+      run: |
+        helm upgrade --install substrate-crds charts/substrate-crds
+        helm upgrade --install substrate charts/substrate \
+          --namespace ate-system \
+          --create-namespace \
+          --set image.registry=localhost:5001 \
+          --set image.tag=helm-e2e \
+          --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \
+          --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \
+          --set postgres.resources.requests.cpu=500m
+    - name: Bootstrap mTLS authorities
+      run: |
+        hack/install-ate-kind.sh --create-podcertificate-controller-cas
+        hack/install-ate-kind.sh --create-jwt-authority-pool-secret
+        hack/install-ate-kind.sh --create-actor-id-ca-pool-secret
+        hack/install-ate-kind.sh --create-actor-id-ca-certs-secret
+        hack/install-ate-kind.sh --create-api-authentication-config
+    - name: Wait for Helm install
+      run: |
+        helm upgrade substrate charts/substrate \
+          --namespace ate-system \
+          --reuse-values \
+          --wait --timeout=10m
+    - name: Enable NFS
+      run: |
+        sudo modprobe nfs || true
+        sudo modprobe nfsd || true
+    - name: Install CSI NFS driver
+      run: hack/install-ate-kind.sh --setup-csi=nfs
+    - name: Deploy micro-VM counter demo
+      # The deploy creates the substrate ActorTemplate and waits for its golden
+      # snapshot internally; the ActorTemplate CRD (and its Ready condition)
+      # no longer exists to wait on.
+      run: hack/run-microvm-demo-kind.sh --skip-control-plane
+    - name: Deploy gVisor counter demo
+      run: hack/install-ate-kind.sh --deploy-demo-counter
+    - name: Deploy egress demo
+      run: hack/install-ate-kind.sh --deploy-demo-egress
+    - name: Run E2E tests (gVisor)
+      run: hack/run-e2e-kind.sh -v -args --no-color
+    - name: Run E2E tests (micro-VM)
+      env:
+        E2E_SANDBOX_CLASS: microvm
+      run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color
+    - name: Dump diagnostics on failure
+      if: failure()
+      run: |
+        kubectl --context kind-kind get workerpool,pods -A -o wide || true
+        for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do
+          echo "=== logs: ate-system/${p} ==="
+          kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true
+        done

.github/workflows/pr-workflow.yaml

@@ -59,15 +59,11 @@ jobs:
   e2e-test-matrix:
     name: E2E (${{ matrix.dataplane }})
     runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.experimental }} # TODO: Make AgentGateway required once tests show stability
     strategy:
       fail-fast: false
       matrix:
         include:
-        - dataplane: envoy
-          experimental: false
         - dataplane: agentgateway
-          experimental: true
     env:
       E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }}
     steps:
@@ -101,7 +97,7 @@ jobs:
     - name: Install Agent Substrate (${{ matrix.dataplane }})
       # The dataplane selection applies to both the ingress router and egress
       # gateway.
-      run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }}
+      run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} --rollout-timeout=300s
     - name: Enable NFS
       # Load NFS kernel modules so in-cluster NFS server and CSI driver can run.
       run: |
@@ -111,11 +107,8 @@ jobs:
       run: hack/install-ate-kind.sh --setup-csi=nfs
     - name: Deploy micro-VM counter demo
       # Stages the (cached) assets into the cluster's rustfs and deploys the
-      # counter-microvm demo onto the control plane installed above. The demo
-      # redeploys the control plane, so retain the selected dataplane.
-      env:
-        ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }}
-      run: hack/run-microvm-demo-kind.sh
+      # counter-microvm demo onto the control plane installed above.
+      run: hack/run-microvm-demo-kind.sh --skip-control-plane
     - name: Deploy gVisor counter demo
       run: hack/install-ate-kind.sh --deploy-demo-counter
     - name: Deploy egress demos
@@ -195,8 +188,7 @@ jobs:
         kubectl --context kind-kind get pods -A -l ate.dev/worker-pool \
           -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \
           | while read -r ns name; do dump "$ns" "$name"; done
-  # Preserve the required-check name while the concrete Envoy and AgentGateway
-  # executions run as entries in the shared matrix above.
+  # Preserve the required-check name for the dataplane matrix above.
   e2e-test:
     name: e2e-test
     needs: e2e-test-matrix

.github/workflows/release.yaml

@@ -0,0 +1,154 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: release
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.'
+        required: false
+      create_release:
+        description: 'Create a GitHub release'
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+  packages: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Validate and resolve tag
+        id: tag
+        run: |
+          TAG="${{ inputs.tag }}"
+          if [[ -z "${TAG}" ]]; then
+            BRANCH="${GITHUB_REF_NAME//\//-}"
+            SHA="$(git rev-parse --short HEAD)"
+            TAG="${BRANCH}-${SHA}"
+          fi
+          if [[ "${{ inputs.create_release }}" == "true" ]]; then
+            if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then
+              echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)"
+              exit 1
+            fi
+          fi
+          echo "value=${TAG}" >> "$GITHUB_OUTPUT"
+          if [[ "${{ inputs.create_release }}" == "true" ]]; then
+            echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT"
+          else
+            echo "tags=${TAG}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Install ko
+        uses: ko-build/[email protected]
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU (multi-arch)
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build and push images
+        env:
+          # ghcr.io/<owner>/<repo> — resolves correctly in forks
+          IMAGE_REPOSITORY: ghcr.io/${{ github.repository }}
+          IMAGE_TAGS: ${{ steps.tag.outputs.tags }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet; do
+            KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \
+            ./hack/run-tool.sh ko build \
+              --tags "${IMAGE_TAGS}" \
+              --platform linux/amd64,linux/arm64 \
+              --bare \
+              "./cmd/${component}"
+          done
+
+      - name: Package and push Helm charts
+        if: inputs.create_release
+        env:
+          HELM_EXPERIMENTAL_OCI: "1"
+          CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          tag="${{ steps.tag.outputs.value }}"
+          chart_version="${tag#v}"
+          package_dir="${RUNNER_TEMP}/helm-packages"
+          mkdir -p "${package_dir}"
+
+          echo "${{ secrets.GITHUB_TOKEN }}" \
+            | helm registry login ghcr.io \
+              --username "${{ github.actor }}" \
+              --password-stdin
+
+          helm package charts/substrate-crds \
+            --destination "${package_dir}" \
+            --version "${chart_version}" \
+            --app-version "${tag}"
+          helm package charts/substrate \
+            --destination "${package_dir}" \
+            --version "${chart_version}" \
+            --app-version "${tag}"
+
+          helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}"
+          helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}"
+
+      - name: Build kubectl-ate release binaries
+        if: inputs.create_release
+        env:
+          VERSION: ${{ steps.tag.outputs.value }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          mkdir -p dist
+          for os in linux darwin; do
+            for arch in amd64 arm64; do
+              CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \
+                -trimpath \
+                -ldflags="-s -w -X=github.com/agent-substrate/substrate/internal/version.Version=${VERSION}" \
+                -o "dist/kubectl-ate-${os}-${arch}" \
+                ./cmd/kubectl-ate
+            done
+          done
+
+      - name: Create GitHub Release
+        if: inputs.create_release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.tag.outputs.value }}
+          generate_release_notes: true
+          files: dist/kubectl-ate-*

Makefile

@@ -70,7 +70,7 @@ build: build-images build-atectl build-ate-setup
 
 .PHONY: build-images
 build-images:
-	$(KO) build $(KO_FLAGS) \
+	$(KO) build --base-import-paths $(KO_FLAGS) \
 	    --ldflags="$(LDFLAGS)" \
 	    $(IMAGES)
 
@@ -131,3 +131,19 @@ verify: test
 .PHONY: clean
 clean:
 	rm -rf $(BINDIR)
+
+# Render the substrate Helm chart into manifests/ate-install/ (mTLS mode,
+# the historical default install). Run this whenever charts/substrate/ changes.
+.PHONY: helm-template
+helm-template:
+	@./hack/render-manifests.sh
+
+# Verify that manifests/ate-install/ matches the chart output. Used in CI.
+.PHONY: verify-helm-template
+verify-helm-template:
+	@./hack/render-manifests.sh --check
+
+# Verify that the CRD chart mirrors the generated CRDs.
+.PHONY: verify-crd-chart
+verify-crd-chart:
+	@./hack/verify/crd-chart.sh