diff --git a/.agents/skills/agentrax-context/SKILL.md b/.agents/skills/agentrax-context/SKILL.md index 88671d0..7285fe1 100644 --- a/.agents/skills/agentrax-context/SKILL.md +++ b/.agents/skills/agentrax-context/SKILL.md @@ -20,6 +20,7 @@ description: Project context and settled architecture decisions for the Agentrax - **Autoscaling**: native `HorizontalPodAutoscaler` pointed at Prometheus Adapter custom metrics (`queueDepth` or `gpuUtilization`). No custom scaling loop. During active canary, the stable HPA is paused (deleted) and no canary HPA is created — autoscaling resumes only after promotion or rollback. - **Traffic splitting**: Gateway API `HTTPRoute` weighted backends. Not Istio, not ingress annotations. +- **Network Isolation**: Two-tier Kubernetes `NetworkPolicy` (`allow-metrics-traffic` in `agentrax-system` allowing operator metrics on TCP 8443; `tenant-agent-isolation` rendered into every `tenant-*` namespace selecting agent pods with `agentrax.io/agent: "true"` for scraping on TCP 8080 and egress to API server/CoreDNS). No service mesh. - **MCP registry**: embedded HTTP handler inside the operator process, backed by a `ConfigMap`. Not a separate Deployment, not a new database — HA storage is a v2 item. - **Non-goals**: no model training/fine-tuning, no general-purpose workload management, no service mesh, no UI in v1. Flag any drift toward these rather than quietly implementing them. diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index db44c80..de04fec 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -31,7 +31,7 @@ resources: # Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. # Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will # be able to communicate with the Webhook Server. -#- ../network-policy +- ../network-policy # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml index ec0fb5e..9713cd9 100644 --- a/config/network-policy/kustomization.yaml +++ b/config/network-policy/kustomization.yaml @@ -1,2 +1,3 @@ resources: - allow-metrics-traffic.yaml +- tenant-agent-isolation.yaml diff --git a/config/network-policy/tenant-agent-isolation.yaml b/config/network-policy/tenant-agent-isolation.yaml new file mode 100644 index 0000000..bdc4740 --- /dev/null +++ b/config/network-policy/tenant-agent-isolation.yaml @@ -0,0 +1,76 @@ +--- +# Tenant Agent Isolation NetworkPolicy +# +# Purpose: Restricts agent pods across tenant namespaces, enforcing zero-trust +# isolation between tenants and preventing unauthorized outbound traffic. +# +# Selector: Matches all pods labelled `agentrax.io/agent: "true"`. +# The AgentDeployment reconciler sets this label on every pod template +# it manages, so this policy applies to all managed agent pods. +# +# Ingress rules: +# - Allow Prometheus to scrape metrics on port 8080 from namespaces +# labelled `monitoring: enabled` (kube-prometheus-stack namespace). +# +# Egress rules: +# - Allow egress to kube-apiserver on port 6443 (required for agent-to-API +# communication and tool-calling via the Kubernetes API). +# - Allow CoreDNS lookups on port 53 (UDP and TCP) for service discovery +# within the cluster. +# - All other egress (internet, cross-tenant) is denied by default. +# +# Usage: Apply this manifest to every tenant namespace: +# kubectl apply -n tenant- -f tenant-agent-isolation.yaml +# +# Note: This policy applies only to tenant-* namespaces (managed agent pods). +# The operator namespace (agentrax-system) is protected by allow-metrics-traffic.yaml (TCP 8443). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: tenant-agent-isolation + labels: + app.kubernetes.io/name: agentrax + app.kubernetes.io/managed-by: kustomize +spec: + # Select all pods carrying the agentrax.io/agent=true label. + # This label is set by agentLabels() in the AgentDeployment reconciler. + podSelector: + matchLabels: + agentrax.io/agent: "true" + policyTypes: + - Ingress + - Egress + ingress: + # Allow Prometheus to scrape agent /metrics on port 8080. + # Prometheus Operator runs in a namespace labelled `monitoring: enabled`. + - from: + - namespaceSelector: + matchLabels: + monitoring: enabled + ports: + - port: 8080 + protocol: TCP + egress: + # Allow outbound to kube-apiserver via ClusterIP (port 443) and direct endpoint (port 6443). + # Agents may call the Kubernetes API to discover services or use cluster tools. + - ports: + - port: 443 + protocol: TCP + - port: 6443 + protocol: TCP + # Allow CoreDNS resolution on UDP and TCP port 53. + # Matches cluster DNS pods in the kube-system namespace. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchExpressions: + - key: k8s-app + operator: In + values: ["kube-dns", "coredns"] + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0080652..98a73e9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -89,16 +89,16 @@ flowchart TB The repository enforces strict directional boundaries to prevent circular dependencies and isolate business logic from Kubernetes plumbing: -| Package | Scope & Responsibility | Key Invariants | -| ---------------------------- | ----------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| `api/v1alpha1/` | CRD type definitions, OpenAPI markers, schema validation rules, and status condition constants. | **Zero business logic**; only struct declarations and generated deep-copy methods. | -| `internal/controller/` | Controller-runtime reconcile loops (`AgentDeployment`, `TenantQuota`). | Only layer that executes write calls against the Kubernetes API for core-owned resources (Deployments, Services, HPAs, HTTPRoutes). Consumes subsystems via interfaces. | -| `internal/quota/` | Quota arithmetic and concurrency-safe in-flight reservation cache. | Pure arithmetic; mutex-guarded state map; zero direct API server network calls in calculation paths. | -| `internal/webhook/` | Validating and Mutating admission webhooks. | Shared with `internal/quota` to enforce admission rules before objects are persisted. | -| `internal/scaling/` | HPA synthesis, velocity rules, and dynamic quota ceiling headroom. | Calculates `QuotaHeadroom()` to cap HPA `maxReplicas` and applies stabilization windows. | -| `internal/rollout/` | Canary state machine, PromQL query construction, and threshold evaluation. | Re-entrant state machine; sample-size gating; fail-safe timeout evaluation. | -| `internal/registry/` | MCP registrar, JSON-RPC 2.0 handshake, TTL sweeper, and discovery REST API. | In-memory registry with ConfigMap write-through for persistence; background health probes and TTL sweep. Explicitly allowed to write the `agentrax-registry` ConfigMap for state recovery. | -| `internal/metrics/` | Bounded HTTP Prometheus query client. | Wraps all responses with `io.LimitReader` (1 MiB ceiling) to prevent memory exhaustion. | +| Package | Scope & Responsibility | Key Invariants | +| ---------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `api/v1alpha1/` | CRD type definitions, OpenAPI markers, schema validation rules, and status condition constants. | **Zero business logic**; only struct declarations and generated deep-copy methods. | +| `internal/controller/` | Controller-runtime reconcile loops (`AgentDeployment`, `TenantQuota`). | Only layer that executes write calls against the Kubernetes API for core-owned resources (Deployments, Services, HPAs, HTTPRoutes, ServiceMonitors). Consumes subsystems via interfaces. | +| `internal/quota/` | Quota arithmetic and concurrency-safe in-flight reservation cache. | Pure arithmetic; mutex-guarded state map; zero direct API server network calls in calculation paths. | +| `internal/webhook/` | Validating and Mutating admission webhooks. | Shared with `internal/quota` to enforce admission rules before objects are persisted. | +| `internal/scaling/` | HPA synthesis, velocity rules, and dynamic quota ceiling headroom. | Calculates `QuotaHeadroom()` to cap HPA `maxReplicas` and applies stabilization windows. | +| `internal/rollout/` | Canary state machine, PromQL query construction, and threshold evaluation. | Re-entrant state machine; sample-size gating; fail-safe timeout evaluation. | +| `internal/registry/` | MCP registrar, JSON-RPC 2.0 handshake, TTL sweeper, and discovery REST API. | In-memory registry with ConfigMap write-through for persistence; background health probes and TTL sweep. Explicitly allowed to write the `agentrax-registry` ConfigMap for state recovery. | +| `internal/metrics/` | Bounded HTTP Prometheus query client. | Wraps all responses with `io.LimitReader` (1 MiB ceiling) to prevent memory exhaustion. | --- @@ -319,19 +319,35 @@ When an `AgentDeployment` is deleted, Kubernetes sets `metadata.deletionTimestam 5. Kubernetes GC cascade deletes child resources (Deployment, Service, HPA, Route) ``` -**Invariant**: MCP deregistration MUST complete _before_ the child `Service` is garbage collected, ensuring external clients never encounter dead routing endpoints. +### 4.6 Zero-Trust Multi-Tenant Network Isolation + +Agentrax enforces a zero-trust network perimeter around all AI agent workloads running in `tenant-*` namespaces. Because autonomous agents dynamically execute tools via MCP and consume cluster resources, flat Kubernetes networking presents severe security risks (unauthorized inter-tenant access, data exfiltration, and lateral movement). + +Agentrax maintains a **two-tier network policy model**: + +| Policy Manifest | Target Namespace | Scope & Responsibility | +| :---------------------------- | :---------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `allow-metrics-traffic.yaml` | `agentrax-system` | Protects the operator process; allows Prometheus to scrape operator `/metrics` on port `:8443` (HTTPS). | +| `tenant-agent-isolation.yaml` | Every `tenant-*` | Isolates agent pods; enforces default-deny on ingress/egress, strictly whitelisting only metrics scraping (`:8080`), Kubernetes API server (`:443`/`:6443`), and CoreDNS (`:53`). | + +#### Ingress & Egress Invariants: + +- **Ingress**: Only TCP port `8080` from namespaces labeled `monitoring: enabled` (Prometheus scraping tenant agent metrics). +- **Egress**: Only to the Kubernetes API server (`kube-apiserver` on TCP ports `443`/`6443`) and cluster CoreDNS (`UDP/TCP :53` in `kube-system` DNS pods). All cross-tenant and arbitrary external internet egress destinations remain blocked at the CNI layer. +- **Label Selector Binding**: The `tenant-agent-isolation` policy selects pods dynamically via `agentrax.io/agent: "true"`. The `AgentDeploymentReconciler` automatically stamps this label into the `PodTemplateSpec` of every managed `Deployment` via `agentLabels()`. --- ## 5. Architectural Decision Records (ADRs) & Trade-Offs -| Decision | Alternative Considered | Trade-Off & Rationale for Agentrax | -| --------------------------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Gateway API (`HTTPRoute`)** | Istio `VirtualService` / Ingress Annotations | Istio requires a heavy service-mesh control plane and sidecar injection. Ingress annotations lack standardized multi-backend weighted traffic splits. Gateway API provides a lightweight, vendor-neutral standard for traffic shifting. | -| **Custom Canary Rollout Engine** | Argo Rollouts / Flagger | Generic rollout tools treat metric anomalies as pure percentages without low-traffic statistical gating (`minRequestSample`). Building an embedded, re-entrant state machine allowed us to guarantee sample-size gating and MCP tool re-registration upon promotion. | -| **Native HPA via Custom Metrics** | KEDA (`ScaledObject`) | KEDA is powerful but adds external CRD dependencies. Generating native Kubernetes `HorizontalPodAutoscaler` objects tied to the Prometheus Adapter custom metrics pipeline minimized dependencies while giving full control over stabilization windows. | -| **Embedded Registry + ConfigMap Store** | Dedicated etcd / Redis / Database | Adding a dedicated database for service discovery increases operator operational complexity. The in-operator HTTP server with ConfigMap write-through store provides simple, robust storage for hundreds of agent services with cold-restart recovery. | -| **Go (`controller-runtime`)** | Python (`Kopf`) | Go provides native compile-time safety, seamless alignment with Kubernetes upstream libraries, and access to `setup-envtest` for isolated in-process integration testing. | +| Decision | Alternative Considered | Trade-Off & Rationale for Agentrax | +| --------------------------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Gateway API (`HTTPRoute`)** | Istio `VirtualService` / Ingress Annotations | Istio requires a heavy service-mesh control plane and sidecar injection. Ingress annotations lack standardized multi-backend weighted traffic splits. Gateway API provides a lightweight, vendor-neutral standard for traffic shifting. | +| **Custom Canary Rollout Engine** | Argo Rollouts / Flagger | Generic rollout tools treat metric anomalies as pure percentages without low-traffic statistical gating (`minRequestSample`). Building an embedded, re-entrant state machine allowed us to guarantee sample-size gating and MCP tool re-registration upon promotion. | +| **Native HPA via Custom Metrics** | KEDA (`ScaledObject`) | KEDA is powerful but adds external CRD dependencies. Generating native Kubernetes `HorizontalPodAutoscaler` objects tied to the Prometheus Adapter custom metrics pipeline minimized dependencies while giving full control over stabilization windows. | +| **Embedded Registry + ConfigMap Store** | Dedicated etcd / Redis / Database | Adding a dedicated database for service discovery increases operator operational complexity. The in-operator HTTP server with ConfigMap write-through store provides simple, robust storage for hundreds of agent services with cold-restart recovery. | +| **Two-Tier NetworkPolicy** | Istio / Linkerd Service Mesh | Service mesh requires sidecar injection and significant control plane memory overhead. Native Kubernetes NetworkPolicy with label-selector binding (`agentrax.io/agent: "true"`) provides lightweight, CNI-enforced zero-trust tenant isolation with default-deny rules. | +| **Go (`controller-runtime`)** | Python (`Kopf`) | Go provides native compile-time safety, seamless alignment with Kubernetes upstream libraries, and access to `setup-envtest` for isolated in-process integration testing. | --- diff --git a/internal/controller/agentdeployment_builder_test.go b/internal/controller/agentdeployment_builder_test.go index 848f43c..9f8270f 100644 --- a/internal/controller/agentdeployment_builder_test.go +++ b/internal/controller/agentdeployment_builder_test.go @@ -240,7 +240,8 @@ func TestDesiredService_ClusterIPType(t *testing.T) { // ── agentLabels ─────────────────────────────────────────────────────────────── -// TestAgentLabels verifies standard label generation for an AgentDeployment. +// TestAgentLabels verifies standard label generation for an AgentDeployment, +// including the agentrax.io/agent selector key used by NetworkPolicies. func TestAgentLabels(t *testing.T) { ad := &agentraxv1alpha1.AgentDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, @@ -252,7 +253,14 @@ func TestAgentLabels(t *testing.T) { "app.kubernetes.io/name": "foo", "app.kubernetes.io/managed-by": "agentrax", "agentrax.io/tenant": "bar", + "agentrax.io/variant": "stable", + "agentrax.io/agent": "true", } + + if len(labels) != len(expected) { + t.Errorf("expected %d labels, got %d: %v", len(expected), len(labels), labels) + } + for k, v := range expected { if labels[k] != v { t.Errorf("agentLabels[%s] = %q, want %q", k, labels[k], v) diff --git a/internal/controller/agentdeployment_controller.go b/internal/controller/agentdeployment_controller.go index e44dfec..d64b90a 100644 --- a/internal/controller/agentdeployment_controller.go +++ b/internal/controller/agentdeployment_controller.go @@ -659,12 +659,15 @@ func (r *AgentDeploymentReconciler) reconcileMCPRegistration(ctx context.Context // agentLabels returns the canonical label set applied to all resources owned by ad. // For stable resources (Deployment, Service), this includes variant=stable. +// The agentrax.io/agent label is the NetworkPolicy selector key — all agent pod +// templates carry it so the tenant-agent-isolation policy applies automatically. func agentLabels(ad *agentraxv1alpha1.AgentDeployment) map[string]string { return map[string]string{ "app.kubernetes.io/name": ad.Name, "app.kubernetes.io/managed-by": "agentrax", "agentrax.io/tenant": ad.Spec.TenantRef, "agentrax.io/variant": "stable", + "agentrax.io/agent": "true", } } diff --git a/internal/controller/tenantquota_controller_test.go b/internal/controller/tenantquota_controller_test.go index 0455ab7..33db6e0 100644 --- a/internal/controller/tenantquota_controller_test.go +++ b/internal/controller/tenantquota_controller_test.go @@ -136,12 +136,18 @@ var _ = Describe("TenantQuota Controller", func() { g.Expect(fetched.Status.UsedAgents).To(BeNumerically("==", 1)) }, timeout, interval).Should(Succeed()) - // Remove finalizer so we can delete immediately. - fetched := &agentraxv1alpha1.AgentDeployment{} - Expect(k8sClient.Get(ctx, namespacedName("ad-del-1", tqNS), fetched)).To(Succeed()) - fetched.Finalizers = nil - Expect(k8sClient.Update(ctx, fetched)).To(Succeed()) - Expect(k8sClient.Delete(ctx, fetched)).To(Succeed()) + // Remove finalizer with conflict retry so deletion is not raced by the controller. + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &agentraxv1alpha1.AgentDeployment{} + if err := k8sClient.Get(ctx, namespacedName("ad-del-1", tqNS), latest); err != nil { + return err + } + latest.Finalizers = nil + return k8sClient.Update(ctx, latest) + })).To(Succeed()) + Expect(k8sClient.Delete(ctx, &agentraxv1alpha1.AgentDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "ad-del-1", Namespace: tqNS}, + })).To(Succeed()) Eventually(func(g Gomega) { tqFetched := &agentraxv1alpha1.TenantQuota{} @@ -217,11 +223,17 @@ var _ = Describe("TenantQuota Controller", func() { }, timeout, interval).Should(Succeed()) // Delete one AD → usage falls back to 1 == maxAgents; OverQuota clears. - ad1 := &agentraxv1alpha1.AgentDeployment{} - Expect(k8sClient.Get(ctx, namespacedName("ad-clearoq-1", tqNS), ad1)).To(Succeed()) - ad1.Finalizers = nil - Expect(k8sClient.Update(ctx, ad1)).To(Succeed()) - Expect(k8sClient.Delete(ctx, ad1)).To(Succeed()) + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &agentraxv1alpha1.AgentDeployment{} + if err := k8sClient.Get(ctx, namespacedName("ad-clearoq-1", tqNS), latest); err != nil { + return err + } + latest.Finalizers = nil + return k8sClient.Update(ctx, latest) + })).To(Succeed()) + Expect(k8sClient.Delete(ctx, &agentraxv1alpha1.AgentDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "ad-clearoq-1", Namespace: tqNS}, + })).To(Succeed()) Eventually(func(g Gomega) { f := &agentraxv1alpha1.TenantQuota{}