From a778207ce66247bbdfc93165bbfc1eed425ddfd5 Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Mon, 24 Aug 2026 07:47:02 +0000 Subject: [PATCH 1/2] feat: implement opt-in cloud workload identity for Azure AKS and AWS EKS without static credentials Signed-off-by: Ankit Kr. Chowdhury --- .agents/skills/agentrax-context/SKILL.md | 3 +- charts/agentrax/templates/deployment.yaml | 3 ++ charts/agentrax/templates/serviceaccount.yaml | 11 ++++- charts/agentrax/values.yaml | 15 +++++++ .../irsa-serviceaccount.yaml | 32 ++++++++++++++ docs/ARCHITECTURE.md | 43 +++++++++++++++++++ 6 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 config/workload-identity/irsa-serviceaccount.yaml diff --git a/.agents/skills/agentrax-context/SKILL.md b/.agents/skills/agentrax-context/SKILL.md index 7285fe1..277f4ac 100644 --- a/.agents/skills/agentrax-context/SKILL.md +++ b/.agents/skills/agentrax-context/SKILL.md @@ -20,7 +20,8 @@ description: Project context and settled architecture decisions for the Agentrax - **Autoscaling**: native `HorizontalPodAutoscaler` pointed at Prometheus Adapter custom metrics (`queueDepth` or `gpuUtilization`). No custom scaling loop. During active canary, the stable HPA is paused (deleted) and no canary HPA is created — autoscaling resumes only after promotion or rollback. - **Traffic splitting**: Gateway API `HTTPRoute` weighted backends. Not Istio, not ingress annotations. -- **Network Isolation**: Two-tier Kubernetes `NetworkPolicy` (`allow-metrics-traffic` in `agentrax-system` allowing operator metrics on TCP 8443; `tenant-agent-isolation` rendered into every `tenant-*` namespace selecting agent pods with `agentrax.io/agent: "true"` for scraping on TCP 8080 and egress to API server/CoreDNS). No service mesh. +- **Network Isolation**: Two-tier Kubernetes `NetworkPolicy` (`allow-metrics-traffic` in `agentrax-system` allowing operator metrics on TCP 8443; `tenant-agent-isolation` rendered into every `tenant-*` namespace selecting agent pods with `agentrax.io/agent: "true"` for scraping on TCP 8080 and egress to API server/CoreDNS in `kube-system`). No service mesh. +- **Cloud Workload Identity**: No static cloud credentials ever. Azure deployments use AKS Workload Identity (`azure.workload.identity/client-id` + `/tenant-id` ServiceAccount annotations; `azure.workload.identity/use: "true"` pod label). AWS deployments use IRSA (`eks.amazonaws.com/role-arn` annotation). Both are opt-in via `workloadIdentity.enabled` in `charts/agentrax/values.yaml`; disabled by default for on-premises portability. - **MCP registry**: embedded HTTP handler inside the operator process, backed by a `ConfigMap`. Not a separate Deployment, not a new database — HA storage is a v2 item. - **Non-goals**: no model training/fine-tuning, no general-purpose workload management, no service mesh, no UI in v1. Flag any drift toward these rather than quietly implementing them. diff --git a/charts/agentrax/templates/deployment.yaml b/charts/agentrax/templates/deployment.yaml index e99bffb..a9eab27 100644 --- a/charts/agentrax/templates/deployment.yaml +++ b/charts/agentrax/templates/deployment.yaml @@ -19,6 +19,9 @@ spec: {{- end }} labels: {{- include "agentrax.selectorLabels" . | nindent 8 }} + {{- if and .Values.workloadIdentity.enabled (eq .Values.workloadIdentity.provider "azure") }} + azure.workload.identity/use: "true" + {{- end }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: diff --git a/charts/agentrax/templates/serviceaccount.yaml b/charts/agentrax/templates/serviceaccount.yaml index cb0618a..c71a283 100644 --- a/charts/agentrax/templates/serviceaccount.yaml +++ b/charts/agentrax/templates/serviceaccount.yaml @@ -6,8 +6,15 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "agentrax.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} annotations: + {{- with .Values.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} + {{- if and .Values.workloadIdentity.enabled (eq .Values.workloadIdentity.provider "azure") }} + azure.workload.identity/client-id: {{ .Values.workloadIdentity.azureClientId | quote }} + azure.workload.identity/tenant-id: {{ .Values.workloadIdentity.azureTenantId | quote }} + {{- end }} + {{- if and .Values.workloadIdentity.enabled (eq .Values.workloadIdentity.provider "aws") }} + eks.amazonaws.com/role-arn: {{ .Values.workloadIdentity.awsRoleArn | quote }} + {{- end }} {{- end }} diff --git a/charts/agentrax/values.yaml b/charts/agentrax/values.yaml index 1efa56d..2654ae9 100644 --- a/charts/agentrax/values.yaml +++ b/charts/agentrax/values.yaml @@ -104,6 +104,21 @@ mcp: # -- Periodic health check interval for registered agents (e.g. "30s") healthInterval: "30s" +# -- Cloud Workload Identity — secretless pod-level IAM. +# Supported providers: azure (AKS Workload Identity), aws (EKS IRSA). +# When disabled (the default), no annotations or labels are injected. +workloadIdentity: + # -- Set to true to enable workload identity for the controller-manager pod. + enabled: false + # -- Cloud provider. Accepted values: "azure" | "aws". + provider: azure + # -- Azure only: Application (client) ID of the managed identity. + azureClientId: "" + # -- Azure only: Azure AD tenant ID. + azureTenantId: "" + # -- AWS only: Full ARN of the IAM role to assume via IRSA. + awsRoleArn: "" + # -- Environment variables injected into the manager container. # These values are automatically populated from registry.ttl and mcp.healthInterval above. env: {} diff --git a/config/workload-identity/irsa-serviceaccount.yaml b/config/workload-identity/irsa-serviceaccount.yaml new file mode 100644 index 0000000..6d8117a --- /dev/null +++ b/config/workload-identity/irsa-serviceaccount.yaml @@ -0,0 +1,32 @@ +# Kustomize strategic-merge patch for AWS IRSA (IAM Roles for Service Accounts). +# Apply this patch in your cluster-specific overlay to bind the controller-manager +# ServiceAccount to an IAM role without static AWS credentials. +# +# Usage — in your overlay kustomization.yaml: +# +# resources: +# - ../../config/default +# patches: +# - path: ../../config/workload-identity/irsa-serviceaccount.yaml +# replacements: +# - source: +# kind: ConfigMap # or any source carrying your role ARN +# name: cluster-metadata +# fieldPath: data.roleArn +# targets: +# - select: +# kind: ServiceAccount +# name: agentrax-controller-manager +# fieldPaths: +# - metadata.annotations.[eks.amazonaws.com/role-arn] +# +# The role must have a trust policy allowing the OIDC provider of your EKS cluster +# to assume it on behalf of the agentrax-system/agentrax-controller-manager subject. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: agentrax-controller-manager # must match the name rendered by the Helm chart + namespace: agentrax-system + annotations: + # Replace this placeholder with the actual IAM role ARN before applying. + eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT_ID:role/ROLE_NAME" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 98a73e9..0bb285c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -338,6 +338,48 @@ Agentrax maintains a **two-tier network policy model**: --- +### 4.7 Keyless Cloud IAM — Workload Identity + +Agentrax requires cloud API access (e.g., Azure Container Registry pulls, AWS Secrets Manager reads) in production. Static credentials baked into `Secret` objects rotate manually, are visible in etcd, and create a long-lived blast radius if leaked. + +The operator is instead bound to a cloud-managed identity at the pod level: + +| Cloud | Mechanism | How it works | +| :------ | :------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------- | +| **Azure** (AKS) | Azure Workload Identity | An OIDC-projected service account token is exchanged for a short-lived Azure AD access token by the Azure Identity SDK. The AKS admission webhook injects the projected volume and `AZURE_*` env vars when the pod carries `azure.workload.identity/use: "true"`. | +| **AWS** (EKS) | IRSA (IAM Roles for SA) | EKS projects a signed OIDC token into the pod; the AWS SDK exchanges it for temporary STS credentials scoped to the bound IAM role via `eks.amazonaws.com/role-arn` annotation. | + +#### Helm Configuration + +Workload identity is off by default. Enable it via `values.yaml` or `--set`: + +```yaml +# Azure AKS +workloadIdentity: + enabled: true + provider: azure + azureClientId: "" + azureTenantId: "" + +# AWS EKS +workloadIdentity: + enabled: true + provider: aws + awsRoleArn: "arn:aws:iam:::role/" +``` + +When `workloadIdentity.enabled=true`: + +- **Azure**: The `ServiceAccount` gains `azure.workload.identity/client-id` and `azure.workload.identity/tenant-id` annotations; the manager pod gains the `azure.workload.identity/use: "true"` label required by the AKS mutating webhook. +- **AWS**: The `ServiceAccount` gains the `eks.amazonaws.com/role-arn` annotation consumed by the EKS pod identity webhook. For Kustomize-based cluster overlays, use `config/workload-identity/irsa-serviceaccount.yaml` as a strategic-merge patch. + +#### Invariants + +- No static cloud credentials (`client_secret`, `AWS_SECRET_ACCESS_KEY`) are ever stored in cluster `Secret` objects. +- `workloadIdentity.enabled=false` (the default) renders no identity annotations or pod labels — the chart remains fully portable to on-premises or non-cloud environments. + +--- + ## 5. Architectural Decision Records (ADRs) & Trade-Offs | Decision | Alternative Considered | Trade-Off & Rationale for Agentrax | @@ -347,6 +389,7 @@ Agentrax maintains a **two-tier network policy model**: | **Native HPA via Custom Metrics** | KEDA (`ScaledObject`) | KEDA is powerful but adds external CRD dependencies. Generating native Kubernetes `HorizontalPodAutoscaler` objects tied to the Prometheus Adapter custom metrics pipeline minimized dependencies while giving full control over stabilization windows. | | **Embedded Registry + ConfigMap Store** | Dedicated etcd / Redis / Database | Adding a dedicated database for service discovery increases operator operational complexity. The in-operator HTTP server with ConfigMap write-through store provides simple, robust storage for hundreds of agent services with cold-restart recovery. | | **Two-Tier NetworkPolicy** | Istio / Linkerd Service Mesh | Service mesh requires sidecar injection and significant control plane memory overhead. Native Kubernetes NetworkPolicy with label-selector binding (`agentrax.io/agent: "true"`) provides lightweight, CNI-enforced zero-trust tenant isolation with default-deny rules. | +| **Workload Identity (no static secrets)** | Kubernetes `Secret` with cloud credentials | Static credentials require manual rotation, are stored in etcd, and present a wide blast radius on leak. OIDC-projected pod tokens (Azure Workload Identity / AWS IRSA) are short-lived, auto-rotated, and scoped to a single identity. | | **Go (`controller-runtime`)** | Python (`Kopf`) | Go provides native compile-time safety, seamless alignment with Kubernetes upstream libraries, and access to `setup-envtest` for isolated in-process integration testing. | --- From b43de9618185cd1da9fa048104d72768d4cf299c Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Mon, 24 Aug 2026 08:43:04 +0000 Subject: [PATCH 2/2] coderabbit review fixes Signed-off-by: Ankit Kr. Chowdhury --- .agents/skills/agentrax-context/SKILL.md | 2 +- charts/agentrax/templates/serviceaccount.yaml | 18 ++++++++++++++++++ .../workload-identity/irsa-serviceaccount.yaml | 2 +- docs/ARCHITECTURE.md | 4 ++-- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.agents/skills/agentrax-context/SKILL.md b/.agents/skills/agentrax-context/SKILL.md index 277f4ac..b811b33 100644 --- a/.agents/skills/agentrax-context/SKILL.md +++ b/.agents/skills/agentrax-context/SKILL.md @@ -21,7 +21,7 @@ description: Project context and settled architecture decisions for the Agentrax - **Autoscaling**: native `HorizontalPodAutoscaler` pointed at Prometheus Adapter custom metrics (`queueDepth` or `gpuUtilization`). No custom scaling loop. During active canary, the stable HPA is paused (deleted) and no canary HPA is created — autoscaling resumes only after promotion or rollback. - **Traffic splitting**: Gateway API `HTTPRoute` weighted backends. Not Istio, not ingress annotations. - **Network Isolation**: Two-tier Kubernetes `NetworkPolicy` (`allow-metrics-traffic` in `agentrax-system` allowing operator metrics on TCP 8443; `tenant-agent-isolation` rendered into every `tenant-*` namespace selecting agent pods with `agentrax.io/agent: "true"` for scraping on TCP 8080 and egress to API server/CoreDNS in `kube-system`). No service mesh. -- **Cloud Workload Identity**: No static cloud credentials ever. Azure deployments use AKS Workload Identity (`azure.workload.identity/client-id` + `/tenant-id` ServiceAccount annotations; `azure.workload.identity/use: "true"` pod label). AWS deployments use IRSA (`eks.amazonaws.com/role-arn` annotation). Both are opt-in via `workloadIdentity.enabled` in `charts/agentrax/values.yaml`; disabled by default for on-premises portability. +- **Cloud Workload Identity**: No static cloud credentials ever. Azure deployments use AKS Workload Identity (`azure.workload.identity/client-id` + `/tenant-id` ServiceAccount annotations; `azure.workload.identity/use: "true"` pod label). AWS deployments use IRSA (`eks.amazonaws.com/role-arn` annotation). In Helm deployments, both are opt-in via `workloadIdentity.enabled` in `charts/agentrax/values.yaml` (disabled by default for portability); in Kustomize deployments, AWS IRSA is activated via the `config/workload-identity/irsa-serviceaccount.yaml` strategic-merge patch. - **MCP registry**: embedded HTTP handler inside the operator process, backed by a `ConfigMap`. Not a separate Deployment, not a new database — HA storage is a v2 item. - **Non-goals**: no model training/fine-tuning, no general-purpose workload management, no service mesh, no UI in v1. Flag any drift toward these rather than quietly implementing them. diff --git a/charts/agentrax/templates/serviceaccount.yaml b/charts/agentrax/templates/serviceaccount.yaml index c71a283..830f0fc 100644 --- a/charts/agentrax/templates/serviceaccount.yaml +++ b/charts/agentrax/templates/serviceaccount.yaml @@ -1,3 +1,21 @@ +{{- if .Values.workloadIdentity.enabled }} + {{- if not .Values.serviceAccount.create }} + {{- fail "workloadIdentity.enabled requires serviceAccount.create to be true so the required cloud identity annotations can be attached" }} + {{- end }} + {{- if not (has .Values.workloadIdentity.provider (list "azure" "aws")) }} + {{- fail (printf "workloadIdentity.provider must be either 'azure' or 'aws', got '%s'" .Values.workloadIdentity.provider) }} + {{- end }} + {{- if eq .Values.workloadIdentity.provider "azure" }} + {{- if or (not .Values.workloadIdentity.azureClientId) (not .Values.workloadIdentity.azureTenantId) }} + {{- fail "workloadIdentity.azureClientId and workloadIdentity.azureTenantId must be non-empty when provider is 'azure'" }} + {{- end }} + {{- end }} + {{- if eq .Values.workloadIdentity.provider "aws" }} + {{- if not .Values.workloadIdentity.awsRoleArn }} + {{- fail "workloadIdentity.awsRoleArn must be non-empty when provider is 'aws'" }} + {{- end }} + {{- end }} +{{- end }} {{- if .Values.serviceAccount.create -}} apiVersion: v1 kind: ServiceAccount diff --git a/config/workload-identity/irsa-serviceaccount.yaml b/config/workload-identity/irsa-serviceaccount.yaml index 6d8117a..3244d8c 100644 --- a/config/workload-identity/irsa-serviceaccount.yaml +++ b/config/workload-identity/irsa-serviceaccount.yaml @@ -25,7 +25,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: agentrax-controller-manager # must match the name rendered by the Helm chart + name: agentrax-controller-manager # Kustomize ServiceAccount name (Helm chart uses "agentrax" by default via workloadIdentity.awsRoleArn) namespace: agentrax-system annotations: # Replace this placeholder with the actual IAM role ARN before applying. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0bb285c..cb604bf 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -340,9 +340,9 @@ Agentrax maintains a **two-tier network policy model**: ### 4.7 Keyless Cloud IAM — Workload Identity -Agentrax requires cloud API access (e.g., Azure Container Registry pulls, AWS Secrets Manager reads) in production. Static credentials baked into `Secret` objects rotate manually, are visible in etcd, and create a long-lived blast radius if leaked. +Agentrax and managed agent pods often require cloud API access (e.g., Azure Key Vault, Azure OpenAI, AWS Secrets Manager) in production. Note that private container image pulls (such as ACR) are handled independently at the node layer via AKS Kubelet Managed Identity (with the `AcrPull` role) or Kubernetes `imagePullSecrets`, while Workload Identity specifically covers runtime cloud SDK calls originating from running pods. -The operator is instead bound to a cloud-managed identity at the pod level: +Static credentials baked into `Secret` objects rotate manually, are visible in etcd, and create a long-lived blast radius if leaked. The operator and tenant agents are instead bound to a cloud-managed identity at the pod level: | Cloud | Mechanism | How it works | | :------ | :------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------- |