diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 5637db0..c26e6a6 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -49,3 +49,65 @@ jobs: coverage.xml pytest-results.xml if-no-files-found: warn + + + ingestion-load-balancer: + name: Ingestion load-balancer smoke test + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Validate NGINX configuration + run: | + docker run --rm \ + -v "$PWD/nginx/ingestion-load-balancer.conf:/etc/nginx/nginx.conf:ro" \ + nginx:1.27.5-alpine nginx -t + + - name: Start three ingestion replicas behind NGINX + env: + EXPOSE_INSTANCE_ID: "true" + run: | + docker compose up --build --detach --scale ingestion-service=3 ingestion-load-balancer + + - name: Verify readiness through the public gateway + run: | + for attempt in $(seq 1 45); do + if curl --fail --silent --show-error http://localhost:8080/ready > /dev/null; then + exit 0 + fi + sleep 2 + done + echo "Gateway never became ready" >&2 + docker compose ps + docker compose logs --no-color ingestion-service ingestion-load-balancer + exit 1 + + - name: Verify requests reach multiple replicas + run: | + for request in $(seq 1 18); do + curl --fail --silent --show-error --dump-header - --output /dev/null \ + http://localhost:8080/health \ + | tr -d '\r' | awk -F': ' 'tolower($1) == "x-instance-id" {print $2}' + done | sort -u > /tmp/ingestion-replicas.txt + cat /tmp/ingestion-replicas.txt + test "$(wc -l < /tmp/ingestion-replicas.txt)" -ge 2 + + - name: Verify one backend loss does not take down readiness + run: | + docker ps --filter label=com.docker.compose.service=ingestion-service --format '{{.ID}}' \ + | head -n 1 | xargs --no-run-if-empty docker stop + for attempt in $(seq 1 15); do + if curl --fail --silent --show-error http://localhost:8080/ready > /dev/null; then + exit 0 + fi + sleep 1 + done + echo "Gateway did not remain ready after one replica stopped" >&2 + exit 1 + + - name: Stop Compose services + if: always() + run: docker compose down --volumes diff --git a/Dailylog.md b/Dailylog.md new file mode 100644 index 0000000..c698076 --- /dev/null +++ b/Dailylog.md @@ -0,0 +1,9 @@ +# Daily Log + +## 2026-08-12 — Load-balancer gap recorded + +An infrastructure audit found that the Compose stack exposes the ingestion service, drift engine, and LLM guard directly on host ports. Existing service health checks do not provide a stable proxy endpoint, multi-replica routing, or unhealthy-upstream handling. + +Tracking issue: [#21 — add a health-aware load-balancing layer for API workers](https://github.com/CoreyLeath-code/SentinelAI/issues/21). + +This record does **not** claim a load balancer, horizontal scaling, resilience test, or benchmark result has been implemented. The issue defines the required implementation and validation scope. diff --git a/README.md b/README.md index a789257..8e49683 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ Once running, open: | Streamlit Dashboard | http://localhost:8501 | | Prometheus | http://localhost:9090 | | Grafana (admin / admin) | http://localhost:3000 | -| Ingestion API | http://localhost:8080 | +| Ingestion API (NGINX gateway) | http://localhost:8080 | | Drift Engine API | http://localhost:7070 | | LLM Guard API | http://localhost:8000 | @@ -156,10 +156,14 @@ All configuration is via environment variables. Copy `.env.example` to `.env` a --- +## Load-balanced ingestion path + +The host-facing ingestion endpoint is an NGINX gateway at port `8080`; Go ingestion replicas are internal-only and can be started with `docker compose up --build --scale ingestion-service=3`. `/health` is liveness and `/ready` includes the Postgres dependency; the CI smoke test checks gateway readiness, multi-replica routing, and continued readiness after one replica stops. The optional `EXPOSE_INSTANCE_ID=true` setting exists only for that test and is disabled by default. + ## 🏗️ Architecture ``` -User → Go Ingestion API (8080) → Postgres (local) / Snowflake (optional) +User → NGINX Ingestion Gateway (8080) → Go Ingestion Replicas → Postgres (local) / Snowflake (optional) ↓ Drift Engine C++ (7070) ↓ diff --git a/docker-compose.yml b/docker-compose.yml index 619dcb9..4aa6554 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,17 +27,41 @@ services: WAREHOUSE_MODE: ${WAREHOUSE_MODE:-postgres} DATABASE_URL: ${DATABASE_URL:-postgres://sentinel:sentinel@postgres:5432/sentinel?sslmode=disable} PORT: "8080" - ports: - - "8080:8080" + # Disabled by default; CI enables this to verify multiple replicas route through NGINX. + EXPOSE_INSTANCE_ID: ${EXPOSE_INSTANCE_ID:-false} + expose: + - "8080" depends_on: postgres: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "wget -qO- http://localhost:8080/health || exit 1"] + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/ready || exit 1"] interval: 15s timeout: 5s retries: 5 + # ── Public ingress for scaled ingestion replicas ────────────────────────── + ingestion-load-balancer: + image: nginx:1.27.5-alpine + restart: unless-stopped + ports: + - "8080:8080" + volumes: + - ./nginx/ingestion-load-balancer.conf:/etc/nginx/nginx.conf:ro + depends_on: + ingestion-service: + condition: service_healthy + + # NGINX exposes status only on the internal Compose network; Prometheus + # receives gateway metrics from this exporter rather than a public endpoint. + nginx-prometheus-exporter: + image: nginx/nginx-prometheus-exporter:1.4.0 + restart: unless-stopped + command: + - "--nginx.scrape-uri=http://ingestion-load-balancer:8080/nginx_status" + depends_on: + - ingestion-load-balancer + # ── Drift engine (C++ + Python HTTP wrapper) ───────────────────────────── drift-engine: build: ./drift-engine @@ -83,7 +107,7 @@ services: environment: WAREHOUSE_MODE: ${WAREHOUSE_MODE:-postgres} DATABASE_URL: ${DATABASE_URL:-postgres://sentinel:sentinel@postgres:5432/sentinel?sslmode=disable} - INGESTION_URL: ${INGESTION_URL:-http://ingestion-service:8080} + INGESTION_URL: ${INGESTION_URL:-http://ingestion-load-balancer:8080} DRIFT_ENGINE_URL: ${DRIFT_ENGINE_URL:-http://drift-engine:7070} LLM_GUARD_URL: ${LLM_GUARD_URL:-http://llm-guard:8000} ports: diff --git a/ingestion-service/main.go b/ingestion-service/main.go index d7b6de2..2e301d8 100644 --- a/ingestion-service/main.go +++ b/ingestion-service/main.go @@ -67,7 +67,15 @@ var db *sql.DB // Handlers // --------------------------------------------------------------------------- +// healthHandler reports process liveness. It intentionally does not depend on +// Postgres so an orchestrator can distinguish a failed dependency from a dead process. func healthHandler(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]string{"status": "ok", "service": "ingestion-service"}) +} + +// readyHandler reports whether this replica can accept durable ingestion writes. +func readyHandler(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") if db != nil { if err := db.Ping(); err != nil { @@ -79,6 +87,32 @@ func healthHandler(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok", "service": "ingestion-service"}) } +func withOptionalInstanceID(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // This header is disabled in normal operation. It is a deterministic + // test aid for the Compose scaling smoke test, not a public identifier. + if os.Getenv("EXPOSE_INSTANCE_ID") == "true" { + instanceID := os.Getenv("INSTANCE_ID") + if instanceID == "" { + instanceID, _ = os.Hostname() + } + if instanceID != "" { + w.Header().Set("X-Instance-ID", instanceID) + } + } + next.ServeHTTP(w, r) + }) +} + +func newMux() *http.ServeMux { + mux := http.NewServeMux() + mux.Handle("/log", withOptionalInstanceID(http.HandlerFunc(logHandler))) + mux.Handle("/health", withOptionalInstanceID(http.HandlerFunc(healthHandler))) + mux.Handle("/ready", withOptionalInstanceID(http.HandlerFunc(readyHandler))) + mux.Handle("/metrics", withOptionalInstanceID(promhttp.Handler())) + return mux +} + func logHandler(w http.ResponseWriter, r *http.Request) { timer := prometheus.NewTimer(ingestLatency) defer timer.ObserveDuration() @@ -172,10 +206,6 @@ func main() { port = "8080" } - http.HandleFunc("/log", logHandler) - http.HandleFunc("/health", healthHandler) - http.Handle("/metrics", promhttp.Handler()) - log.Printf("SentinelAI Ingestion Service running on :%s (warehouse=%s)", port, warehouseMode) - log.Fatal(http.ListenAndServe(":"+port, nil)) + log.Fatal(http.ListenAndServe(":"+port, newMux())) } diff --git a/ingestion-service/main_test.go b/ingestion-service/main_test.go new file mode 100644 index 0000000..98c726e --- /dev/null +++ b/ingestion-service/main_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestMuxExposesInstanceHeaderOnlyWhenEnabled(t *testing.T) { + t.Setenv("EXPOSE_INSTANCE_ID", "true") + t.Setenv("INSTANCE_ID", "test-replica") + + response := httptest.NewRecorder() + newMux().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/health", nil)) + + if response.Code != http.StatusOK { + t.Fatalf("health status = %d, want %d", response.Code, http.StatusOK) + } + if got := response.Header().Get("X-Instance-ID"); got != "test-replica" { + t.Fatalf("X-Instance-ID = %q, want test-replica", got) + } +} + +func TestMuxDoesNotExposeInstanceHeaderByDefault(t *testing.T) { + t.Setenv("EXPOSE_INSTANCE_ID", "false") + t.Setenv("INSTANCE_ID", "test-replica") + + response := httptest.NewRecorder() + newMux().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/health", nil)) + + if got := response.Header().Get("X-Instance-ID"); got != "" { + t.Fatalf("X-Instance-ID = %q, want no header", got) + } +} diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index ce81656..9f1df4a 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -7,6 +7,10 @@ scrape_configs: static_configs: - targets: ['ingestion-service:8080'] + - job_name: 'ingestion-load-balancer' + static_configs: + - targets: ['nginx-prometheus-exporter:9113'] + - job_name: 'drift-engine' static_configs: - targets: ['drift-engine:7070'] diff --git a/nginx/ingestion-load-balancer.conf b/nginx/ingestion-load-balancer.conf new file mode 100644 index 0000000..37448e6 --- /dev/null +++ b/nginx/ingestion-load-balancer.conf @@ -0,0 +1,73 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + log_format upstream '$remote_addr "$request" $status ' + 'upstream=$upstream_addr upstream_status=$upstream_status ' + 'request_time=$request_time upstream_time=$upstream_response_time'; + access_log /var/log/nginx/access.log upstream; + + # Docker's embedded DNS resolves all scaled ingestion-service replicas. + resolver 127.0.0.11 valid=10s ipv6=off; + + upstream ingestion_backends { + zone ingestion_backends 64k; + least_conn; + server ingestion-service:8080 resolve; + keepalive 32; + } + + server { + listen 8080; + server_name _; + + client_max_body_size 1m; + + location = /nginx_status { + stub_status; + access_log off; + } + + # These GET endpoints are safe to retry against another healthy replica. + location = /health { + proxy_pass http://ingestion_backends; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_connect_timeout 2s; + proxy_read_timeout 5s; + } + + location = /ready { + proxy_pass http://ingestion_backends; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_connect_timeout 2s; + proxy_read_timeout 5s; + } + + # Do not retry writes: the ingestion endpoint has no idempotency key contract. + location / { + proxy_pass http://ingestion_backends; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 2s; + proxy_read_timeout 15s; + } + } +}