feat: add a health-aware ingestion load balancer

.github/workflows/ci-cd.yml

@@ -49,3 +49,65 @@ jobs:
             coverage.xml
             pytest-results.xml
           if-no-files-found: warn
+
+
+  ingestion-load-balancer:
+    name: Ingestion load-balancer smoke test
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Validate NGINX configuration
+        run: |
+          docker run --rm \
+            -v "$PWD/nginx/ingestion-load-balancer.conf:/etc/nginx/nginx.conf:ro" \
+            nginx:1.27.5-alpine nginx -t
+
+      - name: Start three ingestion replicas behind NGINX
+        env:
+          EXPOSE_INSTANCE_ID: "true"
+        run: |
+          docker compose up --build --detach --scale ingestion-service=3 ingestion-load-balancer
+
+      - name: Verify readiness through the public gateway
+        run: |
+          for attempt in $(seq 1 45); do
+            if curl --fail --silent --show-error http://localhost:8080/ready > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Gateway never became ready" >&2
+          docker compose ps
+          docker compose logs --no-color ingestion-service ingestion-load-balancer
+          exit 1
+
+      - name: Verify requests reach multiple replicas
+        run: |
+          for request in $(seq 1 18); do
+            curl --fail --silent --show-error --dump-header - --output /dev/null \
+              http://localhost:8080/health \
+              | tr -d '\r' | awk -F': ' 'tolower($1) == "x-instance-id" {print $2}'
+          done | sort -u > /tmp/ingestion-replicas.txt
+          cat /tmp/ingestion-replicas.txt
+          test "$(wc -l < /tmp/ingestion-replicas.txt)" -ge 2
+
+      - name: Verify one backend loss does not take down readiness
+        run: |
+          docker ps --filter label=com.docker.compose.service=ingestion-service --format '{{.ID}}' \
+            | head -n 1 | xargs --no-run-if-empty docker stop
+          for attempt in $(seq 1 15); do
+            if curl --fail --silent --show-error http://localhost:8080/ready > /dev/null; then
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Gateway did not remain ready after one replica stopped" >&2
+          exit 1
+
+      - name: Stop Compose services
+        if: always()
+        run: docker compose down --volumes

Dailylog.md

@@ -0,0 +1,9 @@
+# Daily Log
+
+## 2026-08-12 — Load-balancer gap recorded
+
+An infrastructure audit found that the Compose stack exposes the ingestion service, drift engine, and LLM guard directly on host ports. Existing service health checks do not provide a stable proxy endpoint, multi-replica routing, or unhealthy-upstream handling.
+
+Tracking issue: [#21 — add a health-aware load-balancing layer for API workers](https://github.com/CoreyLeath-code/SentinelAI/issues/21).
+
+This record does **not** claim a load balancer, horizontal scaling, resilience test, or benchmark result has been implemented. The issue defines the required implementation and validation scope.

README.md

@@ -107,7 +107,7 @@ Once running, open:
 | Streamlit Dashboard | http://localhost:8501 |
 | Prometheus | http://localhost:9090 |
 | Grafana (admin / admin) | http://localhost:3000 |
-| Ingestion API | http://localhost:8080 |
+| Ingestion API (NGINX gateway) | http://localhost:8080 |
 | Drift Engine API | http://localhost:7070 |
 | LLM Guard API | http://localhost:8000 |
 
@@ -156,10 +156,14 @@ All configuration is via environment variables.  Copy `.env.example` to `.env` a
 
 ---
 
+## Load-balanced ingestion path
+
+The host-facing ingestion endpoint is an NGINX gateway at port `8080`; Go ingestion replicas are internal-only and can be started with `docker compose up --build --scale ingestion-service=3`. `/health` is liveness and `/ready` includes the Postgres dependency; the CI smoke test checks gateway readiness, multi-replica routing, and continued readiness after one replica stops. The optional `EXPOSE_INSTANCE_ID=true` setting exists only for that test and is disabled by default.
+
 ## 🏗️ Architecture
 
 ```
-User → Go Ingestion API (8080) → Postgres (local) / Snowflake (optional)
+User → NGINX Ingestion Gateway (8080) → Go Ingestion Replicas → Postgres (local) / Snowflake (optional)
                                         ↓
                               Drift Engine C++ (7070)
                                         ↓

docker-compose.yml

@@ -27,17 +27,41 @@ services:
       WAREHOUSE_MODE: ${WAREHOUSE_MODE:-postgres}
       DATABASE_URL:   ${DATABASE_URL:-postgres://sentinel:sentinel@postgres:5432/sentinel?sslmode=disable}
       PORT:           "8080"
-    ports:
-      - "8080:8080"
+      # Disabled by default; CI enables this to verify multiple replicas route through NGINX.
+      EXPOSE_INSTANCE_ID: ${EXPOSE_INSTANCE_ID:-false}
+    expose:
+      - "8080"
     depends_on:
       postgres:
         condition: service_healthy
     healthcheck:
-      test: ["CMD-SHELL", "wget -qO- http://localhost:8080/health || exit 1"]
+      test: ["CMD-SHELL", "wget -qO- http://localhost:8080/ready || exit 1"]
       interval: 15s
       timeout: 5s
       retries: 5
 
+  # ── Public ingress for scaled ingestion replicas ──────────────────────────
+  ingestion-load-balancer:
+    image: nginx:1.27.5-alpine
+    restart: unless-stopped
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./nginx/ingestion-load-balancer.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      ingestion-service:
+        condition: service_healthy
+
+  # NGINX exposes status only on the internal Compose network; Prometheus
+  # receives gateway metrics from this exporter rather than a public endpoint.
+  nginx-prometheus-exporter:
+    image: nginx/nginx-prometheus-exporter:1.4.0
+    restart: unless-stopped
+    command:
+      - "--nginx.scrape-uri=http://ingestion-load-balancer:8080/nginx_status"
+    depends_on:
+      - ingestion-load-balancer
+
   # ── Drift engine (C++ + Python HTTP wrapper) ─────────────────────────────
   drift-engine:
     build: ./drift-engine
@@ -83,7 +107,7 @@ services:
     environment:
       WAREHOUSE_MODE:   ${WAREHOUSE_MODE:-postgres}
       DATABASE_URL:     ${DATABASE_URL:-postgres://sentinel:sentinel@postgres:5432/sentinel?sslmode=disable}
-      INGESTION_URL:    ${INGESTION_URL:-http://ingestion-service:8080}
+      INGESTION_URL:    ${INGESTION_URL:-http://ingestion-load-balancer:8080}
       DRIFT_ENGINE_URL: ${DRIFT_ENGINE_URL:-http://drift-engine:7070}
       LLM_GUARD_URL:    ${LLM_GUARD_URL:-http://llm-guard:8000}
     ports:

ingestion-service/main.go

@@ -67,7 +67,15 @@ var db *sql.DB
 // Handlers
 // ---------------------------------------------------------------------------
 
+// healthHandler reports process liveness. It intentionally does not depend on
+// Postgres so an orchestrator can distinguish a failed dependency from a dead process.
 func healthHandler(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(map[string]string{"status": "ok", "service": "ingestion-service"})
+}
+
+// readyHandler reports whether this replica can accept durable ingestion writes.
+func readyHandler(w http.ResponseWriter, _ *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	if db != nil {
 		if err := db.Ping(); err != nil {
@@ -79,6 +87,32 @@ func healthHandler(w http.ResponseWriter, _ *http.Request) {
 	json.NewEncoder(w).Encode(map[string]string{"status": "ok", "service": "ingestion-service"})
 }
 
+func withOptionalInstanceID(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// This header is disabled in normal operation. It is a deterministic
+		// test aid for the Compose scaling smoke test, not a public identifier.
+		if os.Getenv("EXPOSE_INSTANCE_ID") == "true" {
+			instanceID := os.Getenv("INSTANCE_ID")
+			if instanceID == "" {
+				instanceID, _ = os.Hostname()
+			}
+			if instanceID != "" {
+				w.Header().Set("X-Instance-ID", instanceID)
+			}
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func newMux() *http.ServeMux {
+	mux := http.NewServeMux()
+	mux.Handle("/log", withOptionalInstanceID(http.HandlerFunc(logHandler)))
+	mux.Handle("/health", withOptionalInstanceID(http.HandlerFunc(healthHandler)))
+	mux.Handle("/ready", withOptionalInstanceID(http.HandlerFunc(readyHandler)))
+	mux.Handle("/metrics", withOptionalInstanceID(promhttp.Handler()))
+	return mux
+}
+
 func logHandler(w http.ResponseWriter, r *http.Request) {
 	timer := prometheus.NewTimer(ingestLatency)
 	defer timer.ObserveDuration()
@@ -172,10 +206,6 @@ func main() {
 		port = "8080"
 	}
 
-	http.HandleFunc("/log", logHandler)
-	http.HandleFunc("/health", healthHandler)
-	http.Handle("/metrics", promhttp.Handler())
-
 	log.Printf("SentinelAI Ingestion Service running on :%s (warehouse=%s)", port, warehouseMode)
-	log.Fatal(http.ListenAndServe(":"+port, nil))
+	log.Fatal(http.ListenAndServe(":"+port, newMux()))
 }

ingestion-service/main_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestMuxExposesInstanceHeaderOnlyWhenEnabled(t *testing.T) {
+	t.Setenv("EXPOSE_INSTANCE_ID", "true")
+	t.Setenv("INSTANCE_ID", "test-replica")
+
+	response := httptest.NewRecorder()
+	newMux().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/health", nil))
+
+	if response.Code != http.StatusOK {
+		t.Fatalf("health status = %d, want %d", response.Code, http.StatusOK)
+	}
+	if got := response.Header().Get("X-Instance-ID"); got != "test-replica" {
+		t.Fatalf("X-Instance-ID = %q, want test-replica", got)
+	}
+}
+
+func TestMuxDoesNotExposeInstanceHeaderByDefault(t *testing.T) {
+	t.Setenv("EXPOSE_INSTANCE_ID", "false")
+	t.Setenv("INSTANCE_ID", "test-replica")
+
+	response := httptest.NewRecorder()
+	newMux().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/health", nil))
+
+	if got := response.Header().Get("X-Instance-ID"); got != "" {
+		t.Fatalf("X-Instance-ID = %q, want no header", got)
+	}
+}

monitoring/prometheus.yml

@@ -7,6 +7,10 @@ scrape_configs:
     static_configs:
       - targets: ['ingestion-service:8080']
 
+  - job_name: 'ingestion-load-balancer'
+    static_configs:
+      - targets: ['nginx-prometheus-exporter:9113']
+
   - job_name: 'drift-engine'
     static_configs:
       - targets: ['drift-engine:7070']

nginx/ingestion-load-balancer.conf

@@ -0,0 +1,73 @@
+worker_processes auto;
+
+events {
+  worker_connections 1024;
+}
+
+http {
+  log_format upstream '$remote_addr "$request" $status '
+                      'upstream=$upstream_addr upstream_status=$upstream_status '
+                      'request_time=$request_time upstream_time=$upstream_response_time';
+  access_log /var/log/nginx/access.log upstream;
+
+  # Docker's embedded DNS resolves all scaled ingestion-service replicas.
+  resolver 127.0.0.11 valid=10s ipv6=off;
+
+  upstream ingestion_backends {
+    zone ingestion_backends 64k;
+    least_conn;
+    server ingestion-service:8080 resolve;
+    keepalive 32;
+  }
+
+  server {
+    listen 8080;
+    server_name _;
+
+    client_max_body_size 1m;
+
+    location = /nginx_status {
+      stub_status;
+      access_log off;
+    }
+
+    # These GET endpoints are safe to retry against another healthy replica.
+    location = /health {
+      proxy_pass http://ingestion_backends;
+      proxy_http_version 1.1;
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_next_upstream error timeout http_502 http_503 http_504;
+      proxy_next_upstream_tries 2;
+      proxy_connect_timeout 2s;
+      proxy_read_timeout 5s;
+    }
+
+    location = /ready {
+      proxy_pass http://ingestion_backends;
+      proxy_http_version 1.1;
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_next_upstream error timeout http_502 http_503 http_504;
+      proxy_next_upstream_tries 2;
+      proxy_connect_timeout 2s;
+      proxy_read_timeout 5s;
+    }
+
+    # Do not retry writes: the ingestion endpoint has no idempotency key contract.
+    location / {
+      proxy_pass http://ingestion_backends;
+      proxy_http_version 1.1;
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_connect_timeout 2s;
+      proxy_read_timeout 15s;
+    }
+  }
+}