feat: implement prometheus-based autoscaling logic and custom metrics… #6
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,6 +26,8 @@ import ( | |
| // to ensure that exec-entrypoint and run can make use of them. | ||
| _ "k8s.io/client-go/plugin/pkg/client/auth" | ||
|
|
||
| autoscalingv2 "k8s.io/api/autoscaling/v2" | ||
| apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| utilruntime "k8s.io/apimachinery/pkg/util/runtime" | ||
| clientgoscheme "k8s.io/client-go/kubernetes/scheme" | ||
|
|
@@ -36,6 +38,8 @@ import ( | |
| metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" | ||
| "sigs.k8s.io/controller-runtime/pkg/webhook" | ||
|
|
||
| monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" | ||
|
|
||
| agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1" | ||
| "github.com/gitcommitankit/agentrax/internal/controller" | ||
| "github.com/gitcommitankit/agentrax/internal/quota" | ||
|
|
@@ -48,13 +52,18 @@ var ( | |
| setupLog = ctrl.Log.WithName("setup") | ||
| ) | ||
|
|
||
| // init registers all Kubernetes core, CRD, and monitoring schemes. | ||
| func init() { | ||
| utilruntime.Must(clientgoscheme.AddToScheme(scheme)) | ||
| utilruntime.Must(autoscalingv2.AddToScheme(scheme)) | ||
| utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) | ||
| utilruntime.Must(monitoringv1.AddToScheme(scheme)) | ||
|
|
||
| utilruntime.Must(agentraxv1alpha1.AddToScheme(scheme)) | ||
| // +kubebuilder:scaffold:scheme | ||
| } | ||
|
|
||
| // main is the entrypoint for the Agentrax controller manager binary. | ||
| func main() { | ||
| var metricsAddr string | ||
| var enableLeaderElection bool | ||
|
|
@@ -98,9 +107,18 @@ func main() { | |
| tlsOpts = append(tlsOpts, disableHTTP2) | ||
| } | ||
|
|
||
| // Resolve the webhook-enabled flag once so both the server creation and | ||
| // handler registration use the same value. Log it explicitly so operators | ||
| // can confirm the resolved state at startup. | ||
| enableWebhooks := os.Getenv("ENABLE_WEBHOOKS") != "false" | ||
| setupLog.Info("webhook state resolved", "enabled", enableWebhooks) | ||
|
|
||
| var webhookServer webhook.Server | ||
| if enableWebhooks { | ||
| webhookServer = webhook.NewServer(webhook.Options{ | ||
| TLSOpts: tlsOpts, | ||
| }) | ||
| } | ||
|
Comment on lines
+110
to
+121
Contributor
There was a problem hiding this comment. 🩺 Stability & Availability | 🟠 Major | 🏗️ Heavy lift 🧩 Analysis chain🌐 Web query:
💡 Result: In sigs.k8s.io/controller-runtime v0.19.0, if you do not explicitly provide a WebhookServer in the Manager Options, the manager automatically initializes and adds a default webhook server [1]. When the manager's options (ctrl.Options) have a nil WebhookServer, the manager performs the following behavior during initialization: 1. Automatic Creation: It invokes webhook.NewServer(webhook.Options{}) to create a server instance using default configurations [1]. 2. Default Settings: The default webhook server is configured with the following defaults [2]: - Port: 9443 [2][3]. - Host: "" (listens on all addresses) [2]. - CertDir: /k8s-webhook-server/serving-certs [2]. - CertName: tls.crt [2]. - KeyName: tls.key [2]. - WebhookMux: A new http.ServeMux is initialized [2]. 3. Registration: The manager automatically adds this default server to its internal list of runnables [1][4]. Consequently, the server is started when the manager is started [4]. There is no built-in "nil" or "disabled" state for the webhook server in the v0.19.0 manager options; it is always instantiated if not provided [1][5]. If you need to avoid running a webhook server, users have historically used workarounds, such as setting the port to an invalid value or a non-reachable port, as the server will attempt to bind and start regardless [5][6]. Citations:
🏁 Script executed: #!/bin/bash
set -eu
printf '%s\n' '--- candidate files ---'
git ls-files | grep -E '(^|/)(cmd/main\.go|config/webhook/|go\.mod)$' || true
printf '%s\n' '--- cmd/main.go relevant sections ---'
if [ -f cmd/main.go ]; then
nl -ba cmd/main.go | sed -n '1,240p'
fi
printf '%s\n' '--- webhook references ---'
rg -n -C 3 'ENABLE_WEBHOOKS|enableWebhooks|WebhookServer|webhook\.NewServer|RegisterWebhook|SetupWebhook|Webhook' --glob '*.go' --glob '*.yaml' --glob '*.yml' . || true
printf '%s\n' '--- controller-runtime version ---'
if [ -f go.mod ]; then
rg -n 'sigs\.k8s\.io/controller-runtime' go.mod go.sum 2>/dev/null || true
fi
printf '%s\n' '--- webhook manifests ---'
for f in config/webhook/manifests.yaml config/webhook/*.yaml config/webhook/*.yml; do
if [ -f "$f" ]; then
printf '\n### %s\n' "$f"
nl -ba "$f"
fi
doneRepository: gitcommitankit/agentrax Length of output: 12100 🏁 Script executed: #!/bin/bash
set -eu
printf '%s\n' '--- cmd/main.go 100-205 ---'
sed -n '100,205p' cmd/main.go
printf '%s\n' '--- config/webhook/manifests.yaml ---'
cat -n config/webhook/manifests.yaml 2>/dev/null || cat -n config/webhook/manifests.yaml
printf '%s\n' '--- config/default/kustomization.yaml relevant entries ---'
sed -n '1,125p' config/default/kustomization.yaml
printf '%s\n' '--- webhook setup implementation ---'
sed -n '45,90p' internal/webhook/agentdeployment_webhook.go
printf '%s\n' '--- dependency source locations ---'
find . -path '*controller-runtime*' -o -path './vendor/*' | head -80
find "${GOMODCACHE:-/nonexistent}" -path '*[email protected]*' -type f 2>/dev/null | head -40 || trueRepository: gitcommitankit/agentrax Length of output: 12927 🌐 Web query:
💡 Result: In controller-runtime v0.19.0, the Manager is responsible for initializing the WebhookServer if one is not externally provided [1]. Within the Manager's initialization or options-processing logic, it checks if the WebhookServer field in the manager.Options is nil [1][2]. If o.WebhookServer == nil, the Manager automatically creates a server using webhook.NewServer(webhook.Options{...}) [1]. This ensures that the Manager has a functional webhook server ready for use, populated with default settings derived from the manager's configuration (such as Port, Host, and CertDir) if they are provided [1]. Regarding the specific functions mentioned: 1. NewServer: Defined in the webhook package, this function constructs a new webhook.Server based on the provided options [3]. The Manager utilizes this function to instantiate the default server when no server is explicitly injected [1]. 2. NewWebhookManagedBy: This is a helper function typically found in the builder package (often aliased for convenience) [4][5]. It returns a new webhook builder that allows you to register and configure webhooks (such as Defaulters or Validators) for specific types, ensuring they are properly integrated into the Manager's lifecycle [6]. In summary, when you do not provide a WebhookServer, controller-runtime v0.19.0 gracefully handles this by calling webhook.NewServer to initialize a default instance, and you use NewWebhookManagedBy to register your custom webhook logic with the manager [1][4]. Citations:
🏁 Script executed: #!/bin/bash
set -eu
printf '%s\n' '--- webhook configuration files ---'
git ls-files config | grep -E 'webhook|manager_webhook|service' || true
printf '%s\n' '--- webhook configuration references ---'
rg -n -C 4 'webhook-service|manager_webhook|manifests\.yaml|mutate-agentrax|validate-agentrax|NewWebhookManagedBy|WebhookServer' config cmd internal --glob '*.yaml' --glob '*.yml' --glob '*.go' || trueRepository: gitcommitankit/agentrax Length of output: 6558 🏁 Script executed: #!/bin/bash
set -eu
python3 - <<'PY'
from urllib.request import urlopen
from urllib.error import HTTPError
import re
base = "https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.19.0/"
files = [
"pkg/manager/manager.go",
"pkg/webhook/webhook.go",
"pkg/webhook/webhook_builder.go",
"pkg/webhook/server.go",
]
patterns = [
r"WebhookServer",
r"NewServer",
r"NewWebhookManagedBy",
r"mutate-",
r"validate-",
r"Handle",
r"Register",
]
for path in files:
try:
text = urlopen(base + path, timeout=10).read().decode()
except HTTPError as e:
print(f"\n### {path}: HTTP {e.code}")
continue
except Exception as e:
print(f"\n### {path}: {e}")
continue
lines = text.splitlines()
hits = [i for i, line in enumerate(lines) if any(re.search(p, line) for p in patterns)]
print(f"\n### {path}")
for i in hits:
lo, hi = max(0, i - 3), min(len(lines), i + 5)
print(f"-- lines {lo+1}-{hi} --")
for n in range(lo, hi):
print(f"{n+1}: {lines[n]}")
PYRepository: gitcommitankit/agentrax Length of output: 824 🏁 Script executed: #!/bin/bash
set -eu
base='https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.19.0'
for path in pkg/manager/manager.go pkg/webhook/webhook.go pkg/webhook/webhook_builder.go pkg/webhook/server.go; do
tmp=$(mktemp)
curl -fsSLk "$base/$path" -o "$tmp"
printf '\n### %s\n' "$path"
grep -n -E -C 5 'WebhookServer|NewServer|NewWebhookManagedBy|mutate-|validate-|Register\(' "$tmp" | head -240 || true
rm -f "$tmp"
doneRepository: gitcommitankit/agentrax Length of output: 2786 Keep webhook deployment and handler registration synchronized. When 🤖 Prompt for AI Agents |
||
|
|
||
| // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. | ||
| // More info: | ||
|
|
@@ -154,8 +172,9 @@ func main() { | |
| quotaEnforcer := quota.NewEnforcer(gpuResourceName) | ||
|
|
||
| if err = (&controller.AgentDeploymentReconciler{ | ||
| Client: mgr.GetClient(), | ||
| Scheme: mgr.GetScheme(), | ||
| GPUResourceName: gpuResourceName, | ||
| }).SetupWithManager(mgr); err != nil { | ||
| setupLog.Error(err, "unable to create controller", "controller", "AgentDeployment") | ||
| os.Exit(1) | ||
|
|
@@ -168,9 +187,11 @@ func main() { | |
| setupLog.Error(err, "unable to create controller", "controller", "TenantQuota") | ||
| os.Exit(1) | ||
| } | ||
| if enableWebhooks { | ||
| if err = agentraxwebhook.SetupAgentDeploymentWebhookWithManager(mgr, quotaEnforcer); err != nil { | ||
| setupLog.Error(err, "unable to register webhook", "webhook", "AgentDeployment") | ||
| os.Exit(1) | ||
| } | ||
| } | ||
| // +kubebuilder:scaffold:builder | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| # Prometheus Adapter custom metrics configuration for Agentrax. | ||
| # | ||
| # This ConfigMap is consumed by the prometheus-adapter deployment (typically in | ||
| # the monitoring namespace). It maps PromQL queries to named custom metrics that | ||
| # the HorizontalPodAutoscaler can target via the custom.metrics.k8s.io API. | ||
| # | ||
| # Deploy with: | ||
| # kubectl apply -f config/prometheus-adapter/custom-metrics-config.yaml | ||
| # kubectl rollout restart deployment/prometheus-adapter -n monitoring | ||
| # | ||
| # Verify metrics are registered: | ||
| # kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 | jq . | ||
| # | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| # Use a distinct name so this ConfigMap does not collide with or overwrite | ||
| # the upstream prometheus-adapter ConfigMap (commonly named adapter-config). | ||
| # Reference this name in your prometheus-adapter Deployment via | ||
| # --config=/etc/adapter/config.yaml mounted from this ConfigMap. | ||
| name: agentrax-custom-metrics | ||
| namespace: monitoring | ||
| labels: | ||
| app.kubernetes.io/name: prometheus-adapter | ||
| app.kubernetes.io/managed-by: agentrax | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
| data: | ||
| config.yaml: | | ||
| externalRules: | ||
| # ── queueDepth ──────────────────────────────────────────────────────────── | ||
| # Exposes agentrax_queue_depth via external.metrics.k8s.io, which is the | ||
| # API group queried by HPAs using ExternalMetricSourceType (the type | ||
| # BuildHPA produces). The <<.LabelMatchers>> template is populated by the | ||
| # Prometheus Adapter from the HPA metric selector. Prometheus sanitizes | ||
| # label names (app.kubernetes.io/name → app_kubernetes_io_name), so the | ||
| # query references the sanitized forms that actually exist in storage. | ||
| - seriesQuery: 'agentrax_queue_depth{namespace!=""}' | ||
| name: | ||
| matches: "^agentrax_queue_depth$" | ||
| as: "agentrax_queue_depth" | ||
| metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)' | ||
|
|
||
| # ── gpuUtilization ──────────────────────────────────────────────────────── | ||
| # Exposes agentrax_gpu_utilization via external.metrics.k8s.io. | ||
| # If your GPU device plugin exposes a different metric name (e.g., from DCGM), | ||
| # update the seriesQuery and the as: name here; the HPA target in the | ||
| # AgentDeployment spec references the as: name, which stays stable. | ||
| - seriesQuery: 'agentrax_gpu_utilization{namespace!=""}' | ||
| name: | ||
| matches: "^agentrax_gpu_utilization$" | ||
| as: "agentrax_gpu_utilization" | ||
| metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)' | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| apiVersion: kustomize.config.k8s.io/v1beta1 | ||
| kind: Kustomization | ||
|
|
||
| # Prometheus Adapter custom metrics configuration for Agentrax. | ||
| # | ||
| # Apply this overlay after the Prometheus Adapter base is installed: | ||
| # kubectl apply -k config/prometheus-adapter/ | ||
| # | ||
| # Note: This kustomization targets the monitoring namespace where the | ||
| # prometheus-adapter deployment is expected to run. If your cluster uses | ||
| # a different namespace, update the namespace field below. | ||
| namespace: monitoring | ||
|
|
||
| resources: | ||
| - custom-metrics-config.yaml | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
|
|
||
| # To apply this configuration, either: | ||
| # 1. Include the prometheus-adapter Deployment base in resources above, then | ||
| # uncomment the patches section below, OR | ||
| # 2. Manually configure your existing prometheus-adapter Deployment to mount | ||
| # this ConfigMap at /etc/adapter/config.yaml and pass --config=/etc/adapter/config.yaml | ||
| # | ||
| # patches: | ||
| # - target: | ||
| # kind: Deployment | ||
| # name: prometheus-adapter | ||
| # patch: |- | ||
| # - op: add | ||
| # path: /spec/template/spec/volumes/- | ||
| # value: | ||
| # name: adapter-config | ||
| # configMap: | ||
| # name: agentrax-custom-metrics | ||
| # - op: add | ||
| # path: /spec/template/spec/containers/0/volumeMounts/- | ||
| # value: | ||
| # name: adapter-config | ||
| # mountPath: /etc/adapter | ||
| # - op: add | ||
| # path: /spec/template/spec/containers/0/args/- | ||
| # value: --config=/etc/adapter/config.yaml | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -50,3 +50,4 @@ webhooks: | |
| resources: | ||
| - agentdeployments | ||
| sideEffects: None | ||
| timeoutSeconds: 10 | ||
Uh oh!
There was an error while loading. Please reload this page.