From 04c0f3f48182a0d1d37385b9349062afd7e4d027 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sat, 1 Aug 2026 13:03:31 +0600 Subject: [PATCH 01/56] feat: Zeek SNI detection and domain reputation layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a second trust signal alongside Pi-hole's IP history. Zeek reads the TLS SNI off the wire, giving the domain a connection actually claims rather than just the IP it dialed — closing the gap where a shared CDN edge IP is trusted for one domain and abused for another. - internal/zeek: new package. Tails ssl.log into a short-lived per-connection SNI cache keyed on local port, and watches notice.log for DNS-bypass alerts. Holds partial trailing lines across polls so records flushed mid-write are not silently dropped. - internal/pihole: IsBlockedByPolicy checks Pi-hole's blocklist directly, independent of query history. - internal/deghost: CheckDomain + ShouldBlockDomain for domain-shaped reputation lookups. Unlike the IP endpoint, 400/403 carry a real body and are decoded rather than treated as "safe". - internal/store: domain_checks cache (6h), plus zeek_alerts and sni_observations audit tables. - internal/monitor: SNI-bypass connections escalate through blocklist, then cache, then the domain API — never the IP reputation check, which reads clean on shared CDN addresses regardless of what is fronted through them. - main.go: --zeek-log, --zeek-notice-log, --enable-zeek flags; flag parsing extracted into parseFlags/runConfig. Zeek support defaults to off. Watch mode remains the default. --- .gitignore | 2 + internal/deghost/deghost.go | 76 +++++++ internal/deghost/deghost_test.go | 282 ++++++++++++++++++++++++ internal/monitor/monitor.go | 116 +++++++++- internal/pihole/pihole.go | 35 +++ internal/pihole/policy_test.go | 11 + internal/store/store.go | 176 +++++++++++++++ internal/store/store_test.go | 99 +++++++++ internal/zeek/zeek.go | 356 +++++++++++++++++++++++++++++++ internal/zeek/zeek_test.go | 168 +++++++++++++++ main.go | 124 +++++++++-- 11 files changed, 1421 insertions(+), 24 deletions(-) create mode 100644 internal/deghost/deghost_test.go create mode 100644 internal/pihole/policy_test.go create mode 100644 internal/zeek/zeek.go create mode 100644 internal/zeek/zeek_test.go diff --git a/.gitignore b/.gitignore index 66a86d9..a53ed68 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ tmp .vscode coverage.txt .build +pihole-guard.db* +inventory.ndjson diff --git a/internal/deghost/deghost.go b/internal/deghost/deghost.go index f7e6788..6ec2d6b 100644 --- a/internal/deghost/deghost.go +++ b/internal/deghost/deghost.go @@ -119,3 +119,79 @@ func ShouldKill(report *IPReport) bool { return report.Security.IsAbuser || report.Security.IsAttacker || report.Security.IsThreat } + +// DomainReport matches the API payload for domain reputation checks. +type DomainReport struct { + Status string `json:"status"` + HasMX bool `json:"has_mx"` + Disposable bool `json:"disposable"` + Spam bool `json:"spam"` + PublicDomain bool `json:"public_domain"` + RelayDomain bool `json:"relay_domain"` + Blacklisted bool `json:"blacklisted"` + DomainAgeInDays int `json:"domain_age_in_days"` +} + +// CheckDomain fetches a reputation report for a single domain. +// Unlike CheckIP's 403 handling (nil report for private/reserved IPs), this endpoint +// returns a real JSON body on HTTP 200, 400, and 403 — all are decoded into a DomainReport. +// Only 499/504/500/other status codes are treated as hard errors. +func (c *Client) CheckDomain(ctx context.Context, domain string) (*DomainReport, error) { + if c == nil { + return nil, errors.New("nil deghost client") + } + if c.httpClient == nil { + return nil, errors.New("nil deghost http client") + } + + baseURL := strings.TrimSpace(c.baseURL) + if baseURL == "" { + return nil, errors.New("deghost base URL is required") + } + + domain = strings.ToLower(strings.TrimSpace(domain)) + if domain == "" { + return nil, errors.New("domain is required") + } + + endpoint, err := url.JoinPath(baseURL, "domain", domain) + if err != nil { + return nil, fmt.Errorf("build endpoint: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusBadRequest && resp.StatusCode != http.StatusForbidden { + return nil, fmt.Errorf("deghost returned status %d %s", resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + var report DomainReport + if err := json.NewDecoder(resp.Body).Decode(&report); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + + return &report, nil +} + +// ShouldBlockDomain reports whether the domain report matches the current block policy. +// It triggers only on explicit negative signals (status "not_allowed" or blacklisted), +// ignoring softer signals like disposable, spam, or domain age which are too noisy +// to act on alone — consistent with how ShouldKill only fires on explicit threat fields. +func ShouldBlockDomain(report *DomainReport) bool { + if report == nil { + return false + } + + return report.Status == "not_allowed" || report.Blacklisted +} diff --git a/internal/deghost/deghost_test.go b/internal/deghost/deghost_test.go new file mode 100644 index 0000000..8529a3c --- /dev/null +++ b/internal/deghost/deghost_test.go @@ -0,0 +1,282 @@ +package deghost + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +func TestCheckDomainOK(t *testing.T) { + t.Parallel() + + want := DomainReport{ + Status: "allowed", + HasMX: true, + Disposable: false, + Spam: false, + PublicDomain: true, + RelayDomain: false, + Blacklisted: false, + DomainAgeInDays: 11117, + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("method = %s, want GET", r.Method) + } + if r.URL.Path != "/domain/example.com" { + t.Errorf("path = %s, want /domain/example.com", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(want) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") + if err != nil { + t.Fatalf("CheckDomain() error = %v", err) + } + if got == nil { + t.Fatal("CheckDomain() = nil, want report") + } + if got.Status != want.Status { + t.Errorf("Status = %q, want %q", got.Status, want.Status) + } + if got.HasMX != want.HasMX { + t.Errorf("HasMX = %v, want %v", got.HasMX, want.HasMX) + } + if got.DomainAgeInDays != want.DomainAgeInDays { + t.Errorf("DomainAgeInDays = %d, want %d", got.DomainAgeInDays, want.DomainAgeInDays) + } +} + +func TestCheckDomainStatusForbiddenDecodesBody(t *testing.T) { + t.Parallel() + + want := DomainReport{ + Status: "not_allowed", + HasMX: false, + Disposable: false, + Spam: false, + PublicDomain: false, + RelayDomain: false, + Blacklisted: false, + DomainAgeInDays: 0, + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(want) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "blocked.example.com") + if err != nil { + t.Fatalf("CheckDomain() on 403 error = %v", err) + } + if got == nil { + t.Fatal("CheckDomain() on 403 = nil, want decoded report (not nil like IP endpoint)") + } + if got.Status != "not_allowed" { + t.Errorf("Status = %q, want %q", got.Status, "not_allowed") + } +} + +func TestCheckDomainStatusBadRequestDecodesBody(t *testing.T) { + t.Parallel() + + want := DomainReport{ + Status: "not_allowed", + HasMX: false, + Blacklisted: true, + DomainAgeInDays: 0, + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(want) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "bad.example.com") + if err != nil { + t.Fatalf("CheckDomain() on 400 error = %v", err) + } + if got == nil { + t.Fatal("CheckDomain() on 400 = nil, want decoded report") + } + if !got.Blacklisted { + t.Error("Blacklisted = false, want true") + } +} + +func TestCheckDomainStatusInternalServerError(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on 500 = %v, want error", got) + } +} + +func TestCheckDomainStatusGatewayTimeout(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusGatewayTimeout) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on 504 = %v, want error", got) + } +} + +func TestCheckDomainStatusClientClosedRequest(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(499) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on 499 = %v, want error", got) + } +} + +func TestCheckDomainMalformedJSON(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"status": "allowed" bad json`) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on bad JSON = %v, want error", got) + } +} + +func TestCheckDomainNilClient(t *testing.T) { + t.Parallel() + + var c *Client + got, err := c.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on nil client = %v, want error", got) + } +} + +func TestCheckDomainEmptyBaseURL(t *testing.T) { + t.Parallel() + + client := &Client{baseURL: "", httpClient: &http.Client{}} + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on empty baseURL = %v, want error", got) + } +} + +func TestCheckDomainEmptyDomain(t *testing.T) { + t.Parallel() + + client := NewClient("https://example.com", 5) + got, err := client.CheckDomain(context.Background(), " ") + if err == nil { + t.Fatalf("CheckDomain() on empty domain = %v, want error", got) + } +} + +func TestCheckDomainNormalized(t *testing.T) { + t.Parallel() + + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(DomainReport{Status: "allowed"}) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + _, err := client.CheckDomain(context.Background(), " Example.COM ") + if err != nil { + t.Fatalf("CheckDomain() error = %v", err) + } + want := "/domain/example.com" + if gotPath != want { + t.Errorf("request path = %q, want %q (domain should be trimmed and lowercased)", gotPath, want) + } +} + +func TestShouldBlockDomainNilReport(t *testing.T) { + t.Parallel() + if ShouldBlockDomain(nil) { + t.Error("ShouldBlockDomain(nil) = true, want false") + } +} + +func TestShouldBlockDomainNotAllowed(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "not_allowed"} + if !ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(not_allowed) = false, want true") + } +} + +func TestShouldBlockDomainBlacklisted(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "allowed", Blacklisted: true} + if !ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(blacklisted) = false, want true") + } +} + +func TestShouldBlockDomainBothFalse(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "allowed", Blacklisted: false} + if ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(allowed, not blacklisted) = true, want false") + } +} + +func TestShouldBlockDomainDisposableAloneDoesNotBlock(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "allowed", Disposable: true} + if ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(disposable alone) = true, want false") + } +} + +func TestShouldBlockDomainSpamAloneDoesNotBlock(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "allowed", Spam: true} + if ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(spam alone) = true, want false") + } +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 74265e9..3d87832 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -12,8 +12,10 @@ import ( "github.com/hexbytedev/hexwall/internal/allowlist" "github.com/hexbytedev/hexwall/internal/deghost" + "github.com/hexbytedev/hexwall/internal/pihole" "github.com/hexbytedev/hexwall/internal/somo" "github.com/hexbytedev/hexwall/internal/store" + "github.com/hexbytedev/hexwall/internal/zeek" ) const ( @@ -66,8 +68,42 @@ func logScanConnection(debug bool, ip, program, status string) { slog.Info("scan connection", "ip", ip, "program", program, "status", status) } +// HandleZeekEvent evaluates a Zeek notice event against the current policy and persists it. +func HandleZeekEvent(checker *pihole.Checker, hexwallStore *store.Store, mode string, event zeek.Event) { + if checker == nil || hexwallStore == nil { + return + } + if event.SNI == "" { + return + } + + selectedMode := normalizeMode(mode) + blocked, err := checker.IsBlockedByPolicy(event.SNI) + if err != nil { + slog.Warn("zeek policy lookup failed", "sni", event.SNI, "err", err) + return + } + + confidence := "medium" + actionTaken := "logged" + if blocked { + confidence = "high" + slog.Warn("zeek bypass alert", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI, "mode", selectedMode) + } else { + slog.Info("zeek notice was not policy-blocked", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI) + } + + if err := hexwallStore.LogZeekAlert(event.SrcIP, event.DstIP, event.DstPort, event.SNI, blocked, confidence, actionTaken); err != nil { + slog.Error("failed to persist zeek alert", "src_ip", event.SrcIP, "sni", event.SNI, "err", err) + } + + if selectedMode == ModeEnforce && blocked { + slog.Info("enforce mode: zeek alert recorded without direct connection kill", "src_ip", event.SrcIP, "sni", event.SNI) + } +} + // RunScan inspects established connections and applies the selected trust and kill policy. -func RunScan(ctx context.Context, hexwallStore *store.Store, deghostClient *deghost.Client, mode string, debug bool) { +func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.Store, deghostClient *deghost.Client, zeekClient *zeek.Client, mode string, debug bool) { if hexwallStore == nil { slog.Error("scan aborted: nil hexwall store") return @@ -110,6 +146,84 @@ func RunScan(ctx context.Context, hexwallStore *store.Store, deghostClient *degh continue } + sniBypass := false + var zeekDomain string + if zeekClient != nil { + if domain, ok := zeekClient.Lookup(conn.LPort, ipStr, conn.RPort); ok { + zeekDomain = domain + known, err := checker.IsDomainKnown(domain) + if err != nil { + slog.Debug("sni domain check failed", "domain", domain, "ip", ipStr, "err", err) + } else if !known { + sniBypass = true + slog.Debug("sni domain not in pihole history, bypassing IP trust", "domain", domain, "ip", ipStr) + } + if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, known); logErr != nil { + slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) + } + } else { + slog.Debug("zeek: no SNI seen for connection", "local_port", conn.LPort, "remote_ip", ipStr, "remote_port", conn.RPort) + } + } + + if sniBypass { + shouldBlock, reason := false, "" + domain := strings.ToLower(strings.TrimSpace(zeekDomain)) + + if blocked, err := checker.IsBlockedByPolicy(domain); err != nil { + slog.Debug("domain policy check failed", "domain", domain, "err", err) + } else if blocked { + shouldBlock, reason = true, "policy-blocked" + } + + if !shouldBlock { + if cached, err := hexwallStore.GetRecentDomainCheck(domain); err != nil { + slog.Error("domain cache lookup failed", "domain", domain, "err", err) + continue + } else if cached != nil { + shouldBlock = cached.ShouldBlock + reason = "cached-domain-check" + } + } + + if !shouldBlock && reason == "" { + report, err := deghostClient.CheckDomain(ctx, domain) + if err != nil { + slog.Error("deghost domain check failed", "domain", domain, "program", conn.Program, "err", err) + continue + } + + shouldBlock = deghost.ShouldBlockDomain(report) + reason = "live-domain-check" + if err := hexwallStore.UpsertDomainCheck(domain, shouldBlock); err != nil { + slog.Error("failed to cache domain check", "domain", domain, "program", conn.Program, "err", err) + } + } + + if !shouldBlock { + slog.Info("unrecognized but clean domain", "domain", domain, "ip", ipStr, "program", conn.Program, "reason", reason) + logScanConnection(debug, ipStr, conn.Program, "unrecognized-clean") + continue + } + + logScanConnection(debug, ipStr, conn.Program, "vulnerable") + slog.Warn("vulnerable connection detected", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", reason) + if selectedMode == ModeWatch { + slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + continue + } + + if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { + slog.Error("failed to log kill", "address", conn.RAddress, "err", err) + } + if err := somo.KillConnection(conn.PID); err != nil { + slog.Error("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) + } else { + slog.Info("killed connection", "address", conn.RAddress) + } + continue + } + allowed, err := hexwallStore.IsAllowed(ipStr) if err != nil { slog.Error("store lookup failed", "address", conn.RAddress, "err", err) diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index 028e834..683f449 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -74,6 +74,9 @@ func NewChecker(config *Config) (*Checker, error) { // Close closes the Checker database connection. func (c *Checker) Close() error { + if c == nil || c.db == nil { + return nil + } return c.db.Close() } @@ -106,6 +109,38 @@ func (c *Checker) IsDomainKnown(domain string) (bool, error) { return true, nil } +// IsBlockedByPolicy reports whether the domain currently matches a Pi-hole policy entry. +func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { + domain = normalizeDomain(domain) + if domain == "" { + return false, nil + } + if c == nil || c.db == nil { + return false, errors.New("pihole checker is not initialized") + } + + var found int + err := c.db.QueryRow(` + SELECT 1 + FROM domainlist + WHERE type IN (1, 2) + AND TRIM(domain) <> '' + AND domain = ? COLLATE NOCASE + LIMIT 1 + `, domain).Scan(&found) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + if strings.Contains(err.Error(), "no such table") { + return false, nil + } + return false, fmt.Errorf("policy lookup failed: %w", err) + } + + return true, nil +} + // DomainsSeenSince returns the distinct domains Pi-hole recorded since the given Unix timestamp. func (c *Checker) DomainsSeenSince(since int64) ([]string, error) { rows, err := c.db.Query(` diff --git a/internal/pihole/policy_test.go b/internal/pihole/policy_test.go new file mode 100644 index 0000000..ae9f2cc --- /dev/null +++ b/internal/pihole/policy_test.go @@ -0,0 +1,11 @@ +package pihole + +import ( + "testing" +) + +func TestNormalizeDomain(t *testing.T) { + if got := normalizeDomain(" Example.COM "); got != "example.com" { + t.Fatalf("normalizeDomain() = %q, want %q", got, "example.com") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 5b03123..e234747 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net/url" + "strings" "time" // Register the pure-Go SQLite driver used for the local hexwall database. @@ -43,6 +44,33 @@ CREATE TABLE IF NOT EXISTS fraud_checks ( should_kill INTEGER NOT NULL, checked_at INTEGER NOT NULL ); + +CREATE TABLE IF NOT EXISTS sni_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL, + domain TEXT NOT NULL, + local_port TEXT NOT NULL, + known INTEGER NOT NULL, + observed_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS zeek_alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + src_ip TEXT NOT NULL, + dst_ip TEXT NOT NULL, + dst_port TEXT NOT NULL, + sni TEXT NOT NULL, + blocked INTEGER NOT NULL, + confidence TEXT NOT NULL, + detected_at INTEGER NOT NULL, + action_taken TEXT +); + +CREATE TABLE IF NOT EXISTS domain_checks ( + domain TEXT PRIMARY KEY, + should_block INTEGER NOT NULL, + checked_at INTEGER NOT NULL +); ` // FraudCheckCacheEntry stores the cached kill decision for a prior fraud API lookup. @@ -51,6 +79,25 @@ type FraudCheckCacheEntry struct { CheckedAt int64 } +// DomainCheckCacheEntry stores the cached block decision for a prior domain reputation lookup. +type DomainCheckCacheEntry struct { + ShouldBlock bool + CheckedAt int64 +} + +// ZeekAlertEntry stores a Zeek-derived alert that was persisted for audit. +type ZeekAlertEntry struct { + ID int64 + SrcIP string + DstIP string + DstPort string + SNI string + Blocked bool + Confidence string + DetectedAt int64 + ActionTaken string +} + // Store wraps the local hexwall database. type Store struct { readWrite *sql.DB @@ -254,6 +301,58 @@ func (s *Store) UpsertFraudCheck(ip string, shouldKill bool) error { return nil } +// GetRecentDomainCheck returns the cached domain-check decision when it was recorded within the cache window. +func (s *Store) GetRecentDomainCheck(domain string) (*DomainCheckCacheEntry, error) { + domain = strings.ToLower(strings.TrimSpace(domain)) + cutoff := time.Now().Add(-fraudCheckCacheWindow).Unix() + + var shouldBlock int + var checkedAt int64 + err := s.readOnly.QueryRow(` + SELECT should_block, checked_at + FROM domain_checks + WHERE domain = ? + AND checked_at >= ? + LIMIT 1 + `, domain, cutoff).Scan(&shouldBlock, &checkedAt) + + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + + if err != nil { + return nil, fmt.Errorf("get recent domain check for %s: %w", domain, err) + } + + return &DomainCheckCacheEntry{ + ShouldBlock: shouldBlock != 0, + CheckedAt: checkedAt, + }, nil +} + +// UpsertDomainCheck stores the current domain-check decision. +func (s *Store) UpsertDomainCheck(domain string, shouldBlock bool) error { + domain = strings.ToLower(strings.TrimSpace(domain)) + shouldBlockInt := 0 + if shouldBlock { + shouldBlockInt = 1 + } + + _, err := s.readWrite.Exec(` + INSERT INTO domain_checks (domain, should_block, checked_at) + VALUES (?, ?, ?) + ON CONFLICT(domain) DO UPDATE SET + should_block = excluded.should_block, + checked_at = excluded.checked_at + `, domain, shouldBlockInt, time.Now().Unix()) + + if err != nil { + return fmt.Errorf("upsert domain check %s: %w", domain, err) + } + + return nil +} + // LogKill records a killed connection in the audit log. func (s *Store) LogKill(ip, pid, program string) error { _, err := s.readWrite.Exec(` @@ -267,3 +366,80 @@ func (s *Store) LogKill(ip, pid, program string) error { return nil } + +// LogZeekAlert persists a Zeek-derived event for later review. +func (s *Store) LogZeekAlert(srcIP, dstIP, dstPort, sni string, blocked bool, confidence, actionTaken string) error { + blockedInt := 0 + if blocked { + blockedInt = 1 + } + + _, err := s.readWrite.Exec(` + INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, srcIP, dstIP, dstPort, sni, blockedInt, confidence, time.Now().Unix(), actionTaken) + if err != nil { + return fmt.Errorf("log zeek alert %s: %w", srcIP, err) + } + + return nil +} + +// RecentZeekAlert returns the most recent Zeek alert for the given source IP and SNI. +func (s *Store) RecentZeekAlert(srcIP, sni string) (*ZeekAlertEntry, error) { + cutoff := time.Now().Add(-time.Hour).Unix() + + var id int64 + var dstIP string + var dstPort string + var blockedInt int + var confidence string + var detectedAt int64 + var actionTaken string + err := s.readOnly.QueryRow(` + SELECT id, dst_ip, dst_port, blocked, confidence, detected_at, action_taken + FROM zeek_alerts + WHERE src_ip = ? + AND sni = ? + AND detected_at >= ? + ORDER BY id DESC + LIMIT 1 + `, srcIP, sni, cutoff).Scan(&id, &dstIP, &dstPort, &blockedInt, &confidence, &detectedAt, &actionTaken) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get recent zeek alert for %s: %w", srcIP, err) + } + + return &ZeekAlertEntry{ + ID: id, + SrcIP: srcIP, + DstIP: dstIP, + DstPort: dstPort, + SNI: sni, + Blocked: blockedInt != 0, + Confidence: confidence, + DetectedAt: detectedAt, + ActionTaken: actionTaken, + }, nil +} + +// LogSNIObservation records an SNI domain decision for auditability. +func (s *Store) LogSNIObservation(ip, domain, localPort string, known bool) error { + knownInt := 0 + if known { + knownInt = 1 + } + + _, err := s.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, known, observed_at) + VALUES (?, ?, ?, ?, ?) + `, ip, domain, localPort, knownInt, time.Now().Unix()) + + if err != nil { + return fmt.Errorf("log sni observation %s: %w", ip, err) + } + + return nil +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 15acf98..f6e0a04 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -95,3 +95,102 @@ func TestFraudCheckCacheExpiresAfterWindow(t *testing.T) { t.Fatalf("GetRecentFraudCheck() = %#v, want nil for stale entry", entry) } } + +func TestDomainCheckCacheRoundTrip(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + entry, err := hexwallStore.GetRecentDomainCheck("evil.example.com") + if err != nil { + t.Fatalf("GetRecentDomainCheck() error = %v", err) + } + if entry != nil { + t.Fatalf("GetRecentDomainCheck() = %#v, want nil before insert", entry) + } + + if err := hexwallStore.UpsertDomainCheck("evil.example.com", true); err != nil { + t.Fatalf("UpsertDomainCheck() error = %v", err) + } + + entry, err = hexwallStore.GetRecentDomainCheck("evil.example.com") + if err != nil { + t.Fatalf("GetRecentDomainCheck() after insert error = %v", err) + } + if entry == nil { + t.Fatal("GetRecentDomainCheck() = nil, want cached entry") + } + if !entry.ShouldBlock { + t.Fatal("GetRecentDomainCheck().ShouldBlock = false, want true") + } + if entry.CheckedAt <= 0 { + t.Fatalf("GetRecentDomainCheck().CheckedAt = %d, want positive unix timestamp", entry.CheckedAt) + } +} + +func TestDomainCheckCacheNormalizedKey(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + if err := hexwallStore.UpsertDomainCheck("EXAMPLE.COM", false); err != nil { + t.Fatalf("UpsertDomainCheck() error = %v", err) + } + + entry, err := hexwallStore.GetRecentDomainCheck(" example.com ") + if err != nil { + t.Fatalf("GetRecentDomainCheck() error = %v", err) + } + if entry == nil { + t.Fatal("GetRecentDomainCheck() = nil, normalized key lookup should match") + } + if entry.ShouldBlock { + t.Fatal("GetRecentDomainCheck().ShouldBlock = true, want false") + } +} + +func TestDomainCheckCacheExpiresAfterWindow(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + staleCheckedAt := time.Now().Add(-fraudCheckCacheWindow - time.Minute).Unix() + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO domain_checks (domain, should_block, checked_at) + VALUES (?, ?, ?) + `, "stale.example.com", 1, staleCheckedAt); err != nil { + t.Fatalf("insert stale domain check error = %v", err) + } + + entry, err := hexwallStore.GetRecentDomainCheck("stale.example.com") + if err != nil { + t.Fatalf("GetRecentDomainCheck() error = %v", err) + } + if entry != nil { + t.Fatalf("GetRecentDomainCheck() = %#v, want nil for stale entry", entry) + } +} diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go new file mode 100644 index 0000000..01cd31e --- /dev/null +++ b/internal/zeek/zeek.go @@ -0,0 +1,356 @@ +// Package zeek tails Zeek logs to extract SNI/domain data and DNS-bypass notices. +package zeek + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "strings" + "sync" + "time" +) + +const ( + tickInterval = 5 * time.Second + entryExpiry = 2 * time.Minute +) + +// Config holds the configuration for the Zeek ssl.log tailer. +type Config struct { + LogPath string +} + +// Event is a parsed DNS bypass notice from Zeek. +type Event struct { + Timestamp time.Time + SrcIP string + DstIP string + DstPort string + SNI string +} + +type sniEntry struct { + domain string + observedAt time.Time +} + +// sslRecord is the subset of fields we extract from a Zeek SSL JSON log line. +type sslRecord struct { + OrigH string `json:"id.orig_h"` + OrigP any `json:"id.orig_p"` + RespH string `json:"id.resp_h"` + RespP any `json:"id.resp_p"` + ServerName string `json:"server_name"` +} + +// Client tails a Zeek ssl.log and maintains a short-lived in-memory cache +// mapping connection tuples to their observed SNI domain. +type Client struct { + logPath string + + mu sync.RWMutex + cache map[string]sniEntry + + offset int64 + pending []byte // bytes read past the last newline; held until the rest of the line arrives + stopCh chan struct{} + done chan struct{} +} + +// NewClient opens the ssl.log at the configured path and starts a +// background goroutine that tails new JSON lines. +// If cfg is nil or LogPath is empty, it returns nil without error. +func NewClient(cfg *Config) (*Client, error) { + if cfg == nil || strings.TrimSpace(cfg.LogPath) == "" { + return nil, nil + } + + logPath := strings.TrimSpace(cfg.LogPath) + + // Verify the file exists and is readable before starting the tailer. + f, err := os.Open(logPath) + if err != nil { + return nil, fmt.Errorf("failed to open zeek ssl.log %s: %w", logPath, err) + } + fi, err := f.Stat() + _ = f.Close() + if err != nil { + return nil, fmt.Errorf("failed to stat zeek ssl.log %s: %w", logPath, err) + } + + c := &Client{ + logPath: logPath, + cache: make(map[string]sniEntry), + offset: fi.Size(), + stopCh: make(chan struct{}), + done: make(chan struct{}), + } + + go c.tailLoop(context.Background()) + + slog.Info("zeek log watcher started", "path", logPath, "offset", c.offset) + + return c, nil +} + +// Close stops the background tailer and waits for it to finish. +func (c *Client) Close() { + if c == nil { + return + } + close(c.stopCh) + <-c.done +} + +// Lookup returns the SNI domain observed for the given connection tuple, +// or ("", false) if no matching record exists in the cache. +func (c *Client) Lookup(localPort, remoteIP, remotePort string) (string, bool) { + key := localPort + "|" + remoteIP + "|" + remotePort + + c.mu.RLock() + entry, ok := c.cache[key] + c.mu.RUnlock() + + if !ok { + return "", false + } + + if time.Since(entry.observedAt) > entryExpiry { + return "", false + } + + return entry.domain, true +} + +// ParseNoticeLine parses a single Zeek notice JSON line into an Event. +func ParseNoticeLine(line string) (Event, error) { + line = strings.TrimSpace(line) + if line == "" { + return Event{}, errors.New("empty notice line") + } + + var payload map[string]any + if err := json.Unmarshal([]byte(line), &payload); err != nil { + return Event{}, fmt.Errorf("parse notice line: %w", err) + } + + event := Event{Timestamp: time.Now()} + if ts, ok := payload["ts"].(string); ok { + if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil { + event.Timestamp = parsed + } + } + if ts, ok := payload["timestamp"].(string); ok { + if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil { + event.Timestamp = parsed + } + } + + event.SrcIP = stringValue(payload["src_ip"], payload["src"], payload["id.orig_h"]) + event.DstIP = stringValue(payload["dst_ip"], payload["dst"], payload["id.resp_h"]) + event.DstPort = stringValue(payload["dst_port"], payload["p"], payload["id.resp_p"]) + event.SNI = stringValue(payload["sni"], payload["server_name"]) + + if event.SrcIP == "" && event.DstIP == "" && event.SNI == "" { + return Event{}, errors.New("notice line did not contain recognizable fields") + } + + return event, nil +} + +// WatchNoticeLog tails a Zeek notice log and emits parsed Events. +func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error { + logPath := strings.TrimSpace(path) + if ctx == nil || logPath == "" { + return nil + } + + var offset int64 + poller := time.NewTicker(2 * time.Second) + defer poller.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-poller.C: + file, err := os.Open(logPath) + if err != nil { + slog.Debug("zeek: cannot open notice log", "path", logPath, "err", err) + continue + } + + fi, err := file.Stat() + if err != nil { + _ = file.Close() + slog.Debug("zeek: cannot stat notice log", "path", logPath, "err", err) + continue + } + + if fi.Size() < offset { + offset = 0 + } + if fi.Size() > offset { + if _, err := file.Seek(offset, io.SeekStart); err != nil { + _ = file.Close() + slog.Debug("zeek: seek notice log failed", "path", logPath, "err", err) + continue + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + event, err := ParseNoticeLine(line) + if err != nil { + continue + } + + select { + case events <- event: + default: + } + } + if err := scanner.Err(); err != nil { + slog.Debug("zeek: notice log scan failed", "path", logPath, "err", err) + } + } + _ = file.Close() + offset = fi.Size() + } + } +} + +func (c *Client) tailLoop(ctx context.Context) { + defer close(c.done) + + ticker := time.NewTicker(tickInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-c.stopCh: + return + case <-ticker.C: + c.processNewLines() + } + } +} + +func (c *Client) processNewLines() { + file, err := os.Open(c.logPath) + if err != nil { + slog.Debug("zeek: cannot open ssl.log", "path", c.logPath, "err", err) + return + } + defer file.Close() + + fi, err := file.Stat() + if err != nil { + slog.Debug("zeek: cannot stat ssl.log", "path", c.logPath, "err", err) + return + } + + // File was likely rotated — reset to the beginning and drop any + // partial line held over from the old file. + if fi.Size() < c.offset { + c.offset = 0 + c.pending = nil + } + + if fi.Size() == c.offset { + return + } + + if _, err := file.Seek(c.offset, io.SeekStart); err != nil { + slog.Debug("zeek: seek failed", "path", c.logPath, "err", err) + return + } + + data, err := io.ReadAll(file) + if err != nil { + slog.Debug("zeek: read failed", "path", c.logPath, "err", err) + return + } + + // We've now consumed everything up to fi.Size() from the file, even + // if the trailing bytes don't yet form a complete line — hold them in + // pending rather than losing them, since Zeek may still be mid-write + // on the last line when this tick fires. + c.offset = fi.Size() + + buf := append(c.pending, data...) + lastNewline := bytes.LastIndexByte(buf, '\n') + if lastNewline == -1 { + c.pending = buf + return + } + + complete := buf[:lastNewline] + c.pending = append([]byte(nil), buf[lastNewline+1:]...) + + lines := strings.Split(string(complete), "\n") + + c.mu.Lock() + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + var rec sslRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + continue + } + + if rec.ServerName == "" || rec.OrigH == "" || rec.RespH == "" { + continue + } + + origP := fmt.Sprint(rec.OrigP) + respP := fmt.Sprint(rec.RespP) + now := time.Now() + entry := sniEntry{domain: rec.ServerName, observedAt: now} + + // Store both orientations so Lookup works regardless of which + // side of the connection is the local endpoint. + c.cache[origP+"|"+rec.RespH+"|"+respP] = entry + c.cache[respP+"|"+rec.OrigH+"|"+origP] = entry + } + + // Prune expired entries. + cutoff := time.Now().Add(-entryExpiry) + for k, v := range c.cache { + if v.observedAt.Before(cutoff) { + delete(c.cache, k) + } + } + c.mu.Unlock() +} + +func stringValue(values ...any) string { + for _, value := range values { + switch v := value.(type) { + case string: + return strings.TrimSpace(v) + case fmt.Stringer: + return strings.TrimSpace(v.String()) + case nil: + continue + default: + return fmt.Sprint(v) + } + } + return "" +} diff --git a/internal/zeek/zeek_test.go b/internal/zeek/zeek_test.go new file mode 100644 index 0000000..2991a4b --- /dev/null +++ b/internal/zeek/zeek_test.go @@ -0,0 +1,168 @@ +package zeek + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// newTestClient builds a Client pointed at path without starting the +// background tailLoop goroutine, so tests can call processNewLines directly +// at controlled points instead of racing a real ticker. +func newTestClient(t *testing.T, path string) *Client { + t.Helper() + return &Client{ + logPath: path, + cache: make(map[string]sniEntry), + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } +} + +func TestParseNoticeLine(t *testing.T) { + line := `{"note":"DNSBypass::PossibleBypass","src_ip":"192.0.2.10","dst_ip":"203.0.113.20","dst_port":"443","sni":"evil.example","msg":"SNI evil.example from 192.0.2.10"}` + + event, err := ParseNoticeLine(line) + if err != nil { + t.Fatalf("ParseNoticeLine() error = %v", err) + } + + if event.SrcIP != "192.0.2.10" { + t.Fatalf("ParseNoticeLine().SrcIP = %q, want %q", event.SrcIP, "192.0.2.10") + } + if event.DstIP != "203.0.113.20" { + t.Fatalf("ParseNoticeLine().DstIP = %q, want %q", event.DstIP, "203.0.113.20") + } + if event.DstPort != "443" { + t.Fatalf("ParseNoticeLine().DstPort = %q, want %q", event.DstPort, "443") + } + if event.SNI != "evil.example" { + t.Fatalf("ParseNoticeLine().SNI = %q, want %q", event.SNI, "evil.example") + } + if event.Timestamp.IsZero() { + t.Fatal("ParseNoticeLine().Timestamp = zero time, want non-zero") + } + if event.Timestamp.After(time.Now().Add(time.Second)) { + t.Fatalf("ParseNoticeLine().Timestamp too far in the future: %v", event.Timestamp) + } +} + +func TestParseNoticeLineRejectsInvalidJSON(t *testing.T) { + _, err := ParseNoticeLine(`{"note":`) + if err == nil { + t.Fatal("ParseNoticeLine() error = nil, want error") + } +} + +func TestClientProcessNewLines_BasicParse(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}`+"\n") + + c := newTestClient(t, path) + c.processNewLines() + + domain, ok := c.Lookup("51820", "203.0.113.20", "443") + if !ok { + t.Fatal("Lookup() ok = false, want true") + } + if domain != "example.com" { + t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") + } + + // The reverse orientation must also resolve, since Lookup doesn't know + // in advance which side of the connection is the local endpoint. + domain, ok = c.Lookup("443", "10.0.0.5", "51820") + if !ok { + t.Fatal("Lookup() reverse orientation ok = false, want true") + } + if domain != "example.com" { + t.Fatalf("Lookup() reverse orientation domain = %q, want %q", domain, "example.com") + } +} + +func TestClientProcessNewLines_SkipsIncompleteRecords(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443}`+"\n"+ + `not json at all`+"\n"+ + `{"id.orig_h":"","id.orig_p":1,"id.resp_h":"203.0.113.21","id.resp_p":443,"server_name":"missing-orig-host.example"}`+"\n") + + c := newTestClient(t, path) + c.processNewLines() + + if len(c.cache) != 0 { + t.Fatalf("cache = %v, want empty (no valid records in input)", c.cache) + } +} + +func TestClientProcessNewLines_PartialLineHeldAcrossTicks(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + full := `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}` + "\n" + + // Simulate Zeek flushing mid-record: only the first half of the line + // lands before the poll fires. + split := len(full) / 2 + writeFile(t, path, full[:split]) + + c := newTestClient(t, path) + c.processNewLines() + + if _, ok := c.Lookup("51820", "203.0.113.20", "443"); ok { + t.Fatal("Lookup() found a record from a partial line, want none yet") + } + if len(c.pending) == 0 { + t.Fatal("pending is empty, want the partial line to be held over") + } + + // The rest of the line arrives on the next tick. + writeFile(t, path, full) + c.processNewLines() + + domain, ok := c.Lookup("51820", "203.0.113.20", "443") + if !ok { + t.Fatal("Lookup() ok = false after completing the line, want true") + } + if domain != "example.com" { + t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") + } +} + +func TestClientProcessNewLines_Rotation(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}`+"\n") + + c := newTestClient(t, path) + c.processNewLines() + + if _, ok := c.Lookup("51820", "203.0.113.20", "443"); !ok { + t.Fatal("Lookup() ok = false before rotation, want true") + } + + // Rotation: zeekctl closes the current log and starts a fresh, empty + // file at the same path — simulate the two ticks that straddle that. + writeFile(t, path, "") + c.processNewLines() + writeFile(t, path, `{"id.orig_h":"10.0.0.9","id.orig_p":9000,"id.resp_h":"203.0.113.30","id.resp_p":443,"server_name":"post-rotation.example"}`+"\n") + c.processNewLines() + + if domain, ok := c.Lookup("9000", "203.0.113.30", "443"); !ok || domain != "post-rotation.example" { + t.Fatalf("Lookup() after rotation = (%q, %v), want (%q, true)", domain, ok, "post-rotation.example") + } +} + +func TestClientLookup_Expiry(t *testing.T) { + c := newTestClient(t, "") + c.cache["51820|203.0.113.20|443"] = sniEntry{ + domain: "example.com", + observedAt: time.Now().Add(-entryExpiry - time.Second), + } + + if _, ok := c.Lookup("51820", "203.0.113.20", "443"); ok { + t.Fatal("Lookup() ok = true for an expired entry, want false") + } +} diff --git a/main.go b/main.go index 7199497..2d7aebd 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ import ( "github.com/hexbytedev/hexwall/internal/pihole" "github.com/hexbytedev/hexwall/internal/somo" "github.com/hexbytedev/hexwall/internal/store" + "github.com/hexbytedev/hexwall/internal/zeek" ) const ( @@ -28,39 +29,81 @@ const ( connectionScanInterval = 10 * time.Second ) -var version = "dev" -var platform = runtime.GOOS + "/" + runtime.GOARCH +var ( + version = "dev" + platform = runtime.GOOS + "/" + runtime.GOARCH +) + +type runConfig struct { + dbPath string + hexwallDBPath string + zeekLogPath string + zeekNoticeLog string + enableZeek bool + mode string + debug bool + showVersion bool +} func main() { os.Exit(run()) } -func run() int { +func parseFlags() (*runConfig, error) { dbPath := flag.String("db", "", "path to pihole-FTL.db (auto-detected if not set)") hexwallDB := flag.String("hexwall-db", "./hexwall.db", "path to local hexwall database") + zeekLogPath := flag.String("zeek-log", "", "path to zeek ssl.log for SNI-based domain verification (empty to skip)") + zeekNoticeLog := flag.String("zeek-notice-log", "/opt/zeek/logs/current/notice.log", "path to Zeek notice.log for DNS-bypass alerts") + enableZeek := flag.Bool("enable-zeek", false, "enable Zeek-based DNS-bypass detection") mode := flag.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") debug := flag.Bool("debug", false, "enable verbose per-connection scan logging") showVersion := flag.Bool("version", false, "print version and exit") flag.Parse() - if *showVersion { - fmt.Printf("%s (%s)\n", version, platform) - return 0 + cfg := &runConfig{ + dbPath: strings.TrimSpace(*dbPath), + hexwallDBPath: strings.TrimSpace(*hexwallDB), + zeekLogPath: strings.TrimSpace(*zeekLogPath), + zeekNoticeLog: strings.TrimSpace(*zeekNoticeLog), + enableZeek: *enableZeek, + mode: strings.TrimSpace(*mode), + debug: *debug, + showVersion: *showVersion, + } + + if cfg.showVersion { + return cfg, nil } - selectedMode := strings.ToLower(strings.TrimSpace(*mode)) + selectedMode := strings.ToLower(cfg.mode) if selectedMode != monitor.ModeWatch && selectedMode != monitor.ModeEnforce { - slog.Error("invalid --mode value", "mode", *mode, "allowed", []string{monitor.ModeWatch, monitor.ModeEnforce}) + return nil, fmt.Errorf("invalid --mode value %q: allowed %q or %q", cfg.mode, monitor.ModeWatch, monitor.ModeEnforce) + } + cfg.mode = selectedMode + + if cfg.hexwallDBPath == "" { + return nil, fmt.Errorf("invalid --hexwall-db value %q", *hexwallDB) + } + + return cfg, nil +} + +func run() int { + cfg, err := parseFlags() + if err != nil { + slog.Error(err.Error()) return 1 } - resolvedDBPath := strings.TrimSpace(*dbPath) - hexwallDBPath := strings.TrimSpace(*hexwallDB) - if hexwallDBPath == "" { - slog.Error("invalid --hexwall-db value", "path", *hexwallDB) + if cfg == nil { return 1 } + if cfg.showVersion { + fmt.Printf("%s (%s)\n", version, platform) + return 0 + } + // Cancel background work cleanly on Ctrl+C. ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -72,21 +115,21 @@ func run() int { } // 2. Resolve the Pi-hole database path from --db or auto-detection. - if resolvedDBPath == "" { + if cfg.dbPath == "" { detected, err := detector.FindDBPath() if err != nil { slog.Error("could not find pi-hole installation", "err", err) return 1 } - resolvedDBPath = detected - slog.Info("pi-hole database auto-detected", "path", resolvedDBPath) + cfg.dbPath = detected + slog.Info("pi-hole database auto-detected", "path", cfg.dbPath) } // 3. Open the Pi-hole database in read-only mode. - checker, err := pihole.NewChecker(&pihole.Config{DBPath: resolvedDBPath}) + checker, err := pihole.NewChecker(&pihole.Config{DBPath: cfg.dbPath}) if err != nil { - slog.Error("failed to open pi-hole database", "path", resolvedDBPath, "err", err) + slog.Error("failed to open pi-hole database", "path", cfg.dbPath, "err", err) return 1 } defer func() { @@ -96,9 +139,9 @@ func run() int { }() // 4. Open the local hexwall database, creating it if needed. - hexwallStore, err := store.NewStore(hexwallDBPath) + hexwallStore, err := store.NewStore(cfg.hexwallDBPath) if err != nil { - slog.Error("failed to open hexwall database", "path", hexwallDBPath, "err", err) + slog.Error("failed to open hexwall database", "path", cfg.hexwallDBPath, "err", err) return 1 } defer func() { @@ -107,7 +150,42 @@ func run() int { } }() - slog.Info("hexwall database ready", "path", hexwallDBPath) + slog.Info("hexwall database ready", "path", cfg.hexwallDBPath) + + // 4b. Start the Zeek ssl.log tailer when a path is provided. + var zeekClient *zeek.Client + if cfg.zeekLogPath != "" { + zc, err := zeek.NewClient(&zeek.Config{ + LogPath: cfg.zeekLogPath, + }) + + if err != nil { + slog.Error("failed to start zeek log watcher", "path", cfg.zeekLogPath, "err", err) + return 1 + } + + defer zc.Close() + zeekClient = zc + } + + var zeekEvents = make(chan zeek.Event, 100) + if cfg.enableZeek && cfg.zeekNoticeLog != "" { + go func() { + if err := zeek.WatchNoticeLog(ctx, cfg.zeekNoticeLog, zeekEvents); err != nil { + slog.Error("zeek notice watcher stopped", "path", cfg.zeekNoticeLog, "err", err) + } + }() + go func() { + for { + select { + case <-ctx.Done(): + return + case event := <-zeekEvents: + monitor.HandleZeekEvent(checker, hexwallStore, cfg.mode, event) + } + } + }() + } deghostClient := deghost.NewClient(deghostBaseURL, deghostTimeout) @@ -131,8 +209,8 @@ func run() int { }() fmt.Println("Starting network monitor...") - fmt.Printf("> Connections will be checked every %s in %s mode.\n", connectionScanInterval, selectedMode) - if *debug { + fmt.Printf("> Connections will be checked every %s in %s mode.\n", connectionScanInterval, cfg.mode) + if cfg.debug { fmt.Println("> Debug logging is enabled for every scanned connection.") } @@ -146,7 +224,7 @@ func run() int { slog.Info("shutting down") return 0 case <-ticker.C: - monitor.RunScan(ctx, hexwallStore, deghostClient, selectedMode, *debug) + monitor.RunScan(ctx, checker, hexwallStore, deghostClient, zeekClient, cfg.mode, cfg.debug) } } } From 493b940db920c65a8b6a2fbfec6557d05746703b Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 3 Aug 2026 17:29:28 +0600 Subject: [PATCH 02/56] add: golangci --- .golangci.yml | 10 + internal/monitor/monitor.go | 346 ++++++++++++-------- internal/pihole/cache.go | 56 ++-- internal/pihole/domain_match_test.go | 42 +++ internal/pihole/pihole.go | 174 ++++++++-- internal/pihole/policy_test.go | 468 +++++++++++++++++++++++++++ internal/store/store.go | 86 ++++- internal/store/store_test.go | 178 ++++++++++ main.go | 38 ++- 9 files changed, 1215 insertions(+), 183 deletions(-) create mode 100644 .golangci.yml create mode 100644 internal/pihole/domain_match_test.go diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..667a405 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,10 @@ +version: "2" +linters: + enable: + - staticcheck + - errcheck + - revive + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 3d87832..c32d42b 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -25,6 +25,17 @@ const ( ModeEnforce = "enforce" ) +// scanSummary tracks per-scan outcome counts for the summary line. +type scanSummary struct { + connections int + withSNI int + withoutSNI int + trusted int + blocked int + unknown int + errored int +} + func normalizeMode(mode string) string { switch strings.TrimSpace(strings.ToLower(mode)) { case ModeEnforce: @@ -60,14 +71,6 @@ func remoteIP(address string) (net.IP, error) { return ip, nil } -func logScanConnection(debug bool, ip, program, status string) { - if !debug { - return - } - - slog.Info("scan connection", "ip", ip, "program", program, "status", status) -} - // HandleZeekEvent evaluates a Zeek notice event against the current policy and persists it. func HandleZeekEvent(checker *pihole.Checker, hexwallStore *store.Store, mode string, event zeek.Event) { if checker == nil || hexwallStore == nil { @@ -102,24 +105,29 @@ func HandleZeekEvent(checker *pihole.Checker, hexwallStore *store.Store, mode st } } -// RunScan inspects established connections and applies the selected trust and kill policy. +// RunScan inspects established connections and applies the decision ladder. +// +// The ladder is ordered from cheapest/most-authoritative to most-expensive/least-authoritative: +// 1. Static CIDR allowlist (loopback, RFC1918, cloud metadata) +// 2. Zeek SNI presence — if no SNI, fall back to IP-only evaluation (step 7) +// 3. Pi-hole explicit allowlist (user intent, highest authority) +// 4. Pi-hole explicit denylist / gravity (user intent) +// 5. Inferred trust from DNS history (hexwall's own cache) +// 6. Unknown domain → deghost escalation if enabled; no action if disabled +// 7. IP-only fallback → deghost escalation if enabled; no action if disabled +// +// Safety property: with deghost disabled, only rung 4 (Pi-hole blocklist match) can kill. func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.Store, deghostClient *deghost.Client, zeekClient *zeek.Client, mode string, debug bool) { if hexwallStore == nil { slog.Error("scan aborted: nil hexwall store") return } - if deghostClient == nil { - slog.Error("scan aborted: nil deghost client") - return - } selectedMode := normalizeMode(mode) if selectedMode != mode { slog.Warn("invalid scan mode; defaulting to watch", "mode", mode, "fallback", selectedMode) } - fmt.Printf("[%s] Scanning connections (%s mode)...\n", time.Now().Format("15:04:05"), selectedMode) - connections, err := somo.GetEstablishedConnections() if err != nil { slog.Error("error fetching connections", "err", err) @@ -131,186 +139,250 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S return } + var summary scanSummary + summary.connections = len(connections) + for _, conn := range connections { ip, err := remoteIP(conn.RAddress) if err != nil { slog.Warn("invalid IP address", "address", conn.RAddress, "err", err) + summary.errored++ continue } ipStr := ip.String() - // Trust allowlisted IPs immediately. + // Rung 1: static CIDR allowlist. if allowlist.Contains(ip) { - logScanConnection(debug, ipStr, conn.Program, "allowed") + summary.trusted++ continue } - sniBypass := false - var zeekDomain string + // Rung 2: did Zeek see an SNI for this connection? + var sniDomain string + sniFound := false if zeekClient != nil { if domain, ok := zeekClient.Lookup(conn.LPort, ipStr, conn.RPort); ok { - zeekDomain = domain - known, err := checker.IsDomainKnown(domain) - if err != nil { - slog.Debug("sni domain check failed", "domain", domain, "ip", ipStr, "err", err) - } else if !known { - sniBypass = true - slog.Debug("sni domain not in pihole history, bypassing IP trust", "domain", domain, "ip", ipStr) - } - if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, known); logErr != nil { - slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) - } + sniDomain = domain + sniFound = true + summary.withSNI++ } else { - slog.Debug("zeek: no SNI seen for connection", "local_port", conn.LPort, "remote_ip", ipStr, "remote_port", conn.RPort) + summary.withoutSNI++ } + } else { + summary.withoutSNI++ } - if sniBypass { - shouldBlock, reason := false, "" - domain := strings.ToLower(strings.TrimSpace(zeekDomain)) - - if blocked, err := checker.IsBlockedByPolicy(domain); err != nil { - slog.Debug("domain policy check failed", "domain", domain, "err", err) - } else if blocked { - shouldBlock, reason = true, "policy-blocked" - } - - if !shouldBlock { - if cached, err := hexwallStore.GetRecentDomainCheck(domain); err != nil { - slog.Error("domain cache lookup failed", "domain", domain, "err", err) - continue - } else if cached != nil { - shouldBlock = cached.ShouldBlock - reason = "cached-domain-check" - } - } - - if !shouldBlock && reason == "" { - report, err := deghostClient.CheckDomain(ctx, domain) - if err != nil { - slog.Error("deghost domain check failed", "domain", domain, "program", conn.Program, "err", err) - continue - } - - shouldBlock = deghost.ShouldBlockDomain(report) - reason = "live-domain-check" - if err := hexwallStore.UpsertDomainCheck(domain, shouldBlock); err != nil { - slog.Error("failed to cache domain check", "domain", domain, "program", conn.Program, "err", err) - } - } + if !sniFound { + // Rung 7: IP-only fallback — no SNI means non-TLS, pre-existing connection, or ECH. + action := evaluateIPOnly(ctx, hexwallStore, deghostClient, checker, ipStr, conn, selectedMode, debug) + applyOutcome(&summary, action) + continue + } - if !shouldBlock { - slog.Info("unrecognized but clean domain", "domain", domain, "ip", ipStr, "program", conn.Program, "reason", reason) - logScanConnection(debug, ipStr, conn.Program, "unrecognized-clean") - continue - } + domain := strings.ToLower(strings.TrimSpace(sniDomain)) - logScanConnection(debug, ipStr, conn.Program, "vulnerable") - slog.Warn("vulnerable connection detected", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", reason) - if selectedMode == ModeWatch { - slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - continue + // Rung 3: Pi-hole explicit allowlist — user intent outranks everything inferred. + if allowed, err := checker.IsAllowedByPolicy(domain); err != nil { + slog.Debug("domain allowlist lookup failed", "domain", domain, "err", err) + } else if allowed { + if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, true); logErr != nil { + slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } + summary.trusted++ + continue + } - if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { - slog.Error("failed to log kill", "address", conn.RAddress, "err", err) - } - if err := somo.KillConnection(conn.PID); err != nil { - slog.Error("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) - } else { - slog.Info("killed connection", "address", conn.RAddress) + // Rung 4: Pi-hole explicit denylist / gravity — only rung that can kill. + if blocked, err := checker.IsBlockedByPolicy(domain); err != nil { + slog.Debug("domain denylist lookup failed", "domain", domain, "err", err) + } else if blocked { + if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, false); logErr != nil { + slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } + slog.Warn("policy-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + applyBlockedOutcome(hexwallStore, selectedMode, ipStr, conn) + summary.blocked++ continue } - allowed, err := hexwallStore.IsAllowed(ipStr) + // Rung 5: inferred trust from DNS history — scoped to this IP. + known := false + knownDomains, err := hexwallStore.DomainsForIP(ipStr) if err != nil { - slog.Error("store lookup failed", "address", conn.RAddress, "err", err) + slog.Debug("sni domain check failed", "domain", domain, "ip", ipStr, "err", err) + } else { + for _, knownDomain := range knownDomains { + if pihole.DomainMatches(domain, knownDomain) { + known = true + break + } + } + } + if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, known); logErr != nil { + slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) + } + + if known { + summary.trusted++ continue } - if allowed { - logScanConnection(debug, ipStr, conn.Program, "allowed") - // Keep long-running connections trusted after their Pi-hole refresh window expires. - if err := hexwallStore.UpdateEstablished(ipStr); err != nil { - slog.Error("failed to update established", "address", conn.RAddress, "err", err) + // Rung 6: unknown domain — deghost escalation if enabled. + if deghostClient == nil { + if debug { + slog.Info("unknown domain, deghost disabled; no action", "domain", domain, "ip", ipStr, "program", conn.Program) } + summary.unknown++ continue } - cachedFraudCheck, err := hexwallStore.GetRecentFraudCheck(ipStr) - if err != nil { - slog.Error("fraud cache lookup failed", "ip", ipStr, "program", conn.Program, "err", err) + shouldBlock, reason := false, "" + + if cached, err := hexwallStore.GetRecentDomainCheck(domain); err != nil { + slog.Error("domain cache lookup failed", "domain", domain, "err", err) + summary.errored++ continue + } else if cached != nil { + shouldBlock = cached.ShouldBlock + reason = "cached-domain-check" } - if cachedFraudCheck != nil { - if !cachedFraudCheck.ShouldKill { - slog.Info("unrecognized but clean ip", "ip", ipStr, "program", conn.Program, "reason", "cached-fraud-check") - logScanConnection(debug, ipStr, conn.Program, "unrecognized-clean") + if !shouldBlock && reason == "" { + report, err := deghostClient.CheckDomain(ctx, domain) + if err != nil { + slog.Error("deghost domain check failed", "domain", domain, "program", conn.Program, "err", err) + summary.errored++ continue } - logScanConnection(debug, ipStr, conn.Program, "vulnerable") - slog.Warn("vulnerable connection detected", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", "cached-fraud-check") - if selectedMode == ModeWatch { - slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - continue + shouldBlock = deghost.ShouldBlockDomain(report) + reason = "live-domain-check" + if err := hexwallStore.UpsertDomainCheck(domain, shouldBlock); err != nil { + slog.Error("failed to cache domain check", "domain", domain, "program", conn.Program, "err", err) } + } - if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { - slog.Error("failed to log kill", "address", conn.RAddress, "err", err) - } - if err := somo.KillConnection(conn.PID); err != nil { - slog.Error("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) - } else { - slog.Info("killed connection", "address", conn.RAddress) + if !shouldBlock { + if debug { + slog.Info("unknown but clean domain", "domain", domain, "ip", ipStr, "program", conn.Program, "reason", reason) } + summary.unknown++ continue } - report, err := deghostClient.CheckIP(ctx, ipStr) - if err != nil { - slog.Error("deghost check failed", "ip", ipStr, "program", conn.Program, "err", err) - continue + slog.Warn("deghost-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", reason) + applyBlockedOutcome(hexwallStore, selectedMode, ipStr, conn) + summary.blocked++ + } + + fmt.Printf("[%s] Scan complete: %d connections, %d trusted, %d blocked, %d unknown (sni=%d no-sni=%d errored=%d)\n", + time.Now().Format("15:04:05"), summary.connections, summary.trusted, summary.blocked, summary.unknown, summary.withSNI, summary.withoutSNI, summary.errored) +} + +// evaluateIPOnly handles the IP-only fallback path (rung 7) when no SNI is available. +// It returns "trusted", "blocked", "unknown", or "error". +func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClient *deghost.Client, checker *pihole.Checker, ipStr string, conn somo.Connection, mode string, debug bool) string { + allowed, err := hexwallStore.IsAllowed(ipStr) + if err != nil { + slog.Error("store lookup failed", "address", conn.RAddress, "err", err) + return "error" + } + + if allowed { + if err := hexwallStore.UpdateEstablished(ipStr); err != nil { + slog.Error("failed to update established", "address", conn.RAddress, "err", err) + } + return "trusted" + } + + if deghostClient == nil { + if debug { + slog.Info("unknown IP, deghost disabled; no action", "ip", ipStr, "program", conn.Program) } + return "unknown" + } - if report == nil { - if err := hexwallStore.UpsertFraudCheck(ipStr, false); err != nil { - slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) + cachedFraudCheck, err := hexwallStore.GetRecentFraudCheck(ipStr) + if err != nil { + slog.Error("fraud cache lookup failed", "ip", ipStr, "program", conn.Program, "err", err) + return "error" + } + + if cachedFraudCheck != nil { + if !cachedFraudCheck.ShouldKill { + if debug { + slog.Info("unknown but clean IP (cached)", "ip", ipStr, "program", conn.Program) } - slog.Info("unrecognized but clean ip", "ip", ipStr, "program", conn.Program, "reason", "403/private-or-reserved") - logScanConnection(debug, ipStr, conn.Program, "unrecognized-clean") - continue + return "unknown" } - shouldKill := deghost.ShouldKill(report) - if err := hexwallStore.UpsertFraudCheck(ipStr, shouldKill); err != nil { + slog.Warn("deghost-blocked IP (cached)", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + applyBlockedOutcome(hexwallStore, mode, ipStr, conn) + return "blocked" + } + + report, err := deghostClient.CheckIP(ctx, ipStr) + if err != nil { + slog.Error("deghost check failed", "ip", ipStr, "program", conn.Program, "err", err) + return "error" + } + + if report == nil { + if err := hexwallStore.UpsertFraudCheck(ipStr, false); err != nil { slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) } - - if !shouldKill { - slog.Info("unrecognized but clean ip", "ip", ipStr, "program", conn.Program) - logScanConnection(debug, ipStr, conn.Program, "unrecognized-clean") - continue + if debug { + slog.Info("unknown but clean IP (private/reserved)", "ip", ipStr, "program", conn.Program) } + return "unknown" + } - logScanConnection(debug, ipStr, conn.Program, "vulnerable") + shouldKill := deghost.ShouldKill(report) + if err := hexwallStore.UpsertFraudCheck(ipStr, shouldKill); err != nil { + slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) + } - slog.Warn("vulnerable connection detected", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - if selectedMode == ModeWatch { - slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - continue + if !shouldKill { + if debug { + slog.Info("unknown but clean IP", "ip", ipStr, "program", conn.Program) } + return "unknown" + } - if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { - slog.Error("failed to log kill", "address", conn.RAddress, "err", err) - } - if err := somo.KillConnection(conn.PID); err != nil { - slog.Error("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) - } else { - slog.Info("killed connection", "address", conn.RAddress) - } + slog.Warn("deghost-blocked IP", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + applyBlockedOutcome(hexwallStore, mode, ipStr, conn) + return "blocked" +} + +// applyBlockedOutcome enforces the kill/watch policy for a blocked connection. +func applyBlockedOutcome(hexwallStore *store.Store, mode, ipStr string, conn somo.Connection) { + selectedMode := normalizeMode(mode) + if selectedMode == ModeWatch { + slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + return + } + + if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { + slog.Error("failed to log kill", "address", conn.RAddress, "err", err) + } + if err := somo.KillConnection(conn.PID); err != nil { + slog.Error("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) + } else { + slog.Info("killed connection", "address", conn.RAddress) + } +} + +// applyOutcome increments the appropriate summary counter for a given outcome string. +func applyOutcome(summary *scanSummary, outcome string) { + switch outcome { + case "trusted": + summary.trusted++ + case "blocked": + summary.blocked++ + case "unknown": + summary.unknown++ + case "error": + summary.errored++ } } diff --git a/internal/pihole/cache.go b/internal/pihole/cache.go index 12df0ff..5979c4b 100644 --- a/internal/pihole/cache.go +++ b/internal/pihole/cache.go @@ -53,13 +53,14 @@ func (c *IPCache) Refresh(ctx context.Context) { sort.Strings(domains) - type result struct { - domain string - ips []string - } - results := make(chan result, maxConcurrentLookups) sem := make(chan struct{}, maxConcurrentLookups) + // Workers write straight into resolved under mu. Using a shared map instead of a + // results channel keeps a worker from ever blocking on a consumer that has not + // started yet, which would otherwise pin its slot in sem and stall the spawn loop. + var mu sync.Mutex + resolved := make(map[string][]string, len(domains)) + var wg sync.WaitGroup spawnLoop: for _, domain := range domains { @@ -82,26 +83,24 @@ spawnLoop: // DNS failure is normal for blocked domains, expired records, and similar cases. return } - results <- result{domain: d, ips: ips} + + mu.Lock() + resolved[d] = ips + mu.Unlock() }(domain) } - go func() { - wg.Wait() - close(results) - }() - - resolved := make(map[string][]string, len(domains)) - for r := range results { - resolved[r.domain] = r.ips - } + wg.Wait() if ctx.Err() != nil { return } - // The store keeps one domain per IP, so keep the first domain in a stable order. - uniqueIPs := make(map[string]string, len(resolved)) + // A shared CDN edge IP serves many unrelated domains, so keep the full set per IP. + // domains is sorted, so each IP's slice stays in a stable order and its first entry + // is the one recorded as the representative domain in allowed_ips. + ipDomains := make(map[string][]string, len(resolved)) + seen := make(map[string]struct{}, len(resolved)) for _, domain := range domains { ips, ok := resolved[domain] if !ok { @@ -109,23 +108,34 @@ spawnLoop: } for _, ip := range ips { - if _, exists := uniqueIPs[ip]; exists { + key := ip + "\x00" + domain + if _, exists := seen[key]; exists { continue } - uniqueIPs[ip] = domain + seen[key] = struct{}{} + ipDomains[ip] = append(ipDomains[ip], domain) } } var totalIPs int - for ip, domain := range uniqueIPs { - if err := c.store.UpsertAllowedIP(ip, domain); err != nil { - slog.Error("cache refresh: failed to upsert IP", "ip", ip, "domain", domain, "err", err) + var totalPairs int + for ip, ipDomainSet := range ipDomains { + if err := c.store.UpsertAllowedIP(ip, ipDomainSet[0]); err != nil { + slog.Error("cache refresh: failed to upsert IP", "ip", ip, "domain", ipDomainSet[0], "err", err) } else { totalIPs++ } + + for _, domain := range ipDomainSet { + if err := c.store.UpsertAllowedIPDomain(ip, domain); err != nil { + slog.Error("cache refresh: failed to upsert IP domain", "ip", ip, "domain", domain, "err", err) + } else { + totalPairs++ + } + } } - slog.Info("cache refreshed", "domains", len(domains), "ips", totalIPs) + slog.Info("cache refreshed", "domains", len(domains), "ips", totalIPs, "pairs", totalPairs) } // RunRefresh calls Refresh on the given interval until ctx is cancelled. diff --git a/internal/pihole/domain_match_test.go b/internal/pihole/domain_match_test.go new file mode 100644 index 0000000..902e216 --- /dev/null +++ b/internal/pihole/domain_match_test.go @@ -0,0 +1,42 @@ +package pihole + +import ( + "testing" +) + +func TestDomainMatches(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + observed string + known string + want bool + }{ + {name: "exact match", observed: "github.com", known: "github.com", want: true}, + {name: "subdomain observed", observed: "www.github.com", known: "github.com", want: true}, + {name: "subdomain known", observed: "github.com", known: "www.github.com", want: true}, + {name: "deeper subdomain", observed: "a.b.github.com", known: "github.com", want: true}, + {name: "deeper subdomain reversed", observed: "github.com", known: "a.b.github.com", want: true}, + {name: "lookalike suffix is not a match", observed: "github.com.evil.com", known: "github.com", want: false}, + {name: "lookalike suffix reversed is not a match", observed: "github.com", known: "github.com.evil.com", want: false}, + {name: "partial label is not a match", observed: "notgithub.com", known: "github.com", want: false}, + {name: "unrelated domains", observed: "example.org", known: "github.com", want: false}, + {name: "empty observed", observed: "", known: "github.com", want: false}, + {name: "empty known", observed: "github.com", known: "", want: false}, + {name: "both empty", observed: "", known: "", want: false}, + {name: "whitespace only", observed: " ", known: "github.com", want: false}, + {name: "case and whitespace normalized", observed: " GitHub.COM ", known: "github.com", want: true}, + {name: "case and whitespace normalized subdomain", observed: " WWW.GitHub.com ", known: " GITHUB.com ", want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := DomainMatches(tt.observed, tt.known); got != tt.want { + t.Fatalf("DomainMatches(%q, %q) = %v, want %v", tt.observed, tt.known, got, tt.want) + } + }) + } +} diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index 683f449..043c897 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -6,8 +6,10 @@ import ( "database/sql" "errors" "fmt" + "log/slog" "net/url" "os" + "path/filepath" "strings" "time" @@ -17,12 +19,14 @@ import ( // Config holds the Pi-hole database configuration. type Config struct { - DBPath string + DBPath string + GravityDBPath string // optional; derived from DBPath when empty } -// Checker reads Pi-hole query history from the FTL database. +// Checker reads Pi-hole query history from the FTL database and policy lists from gravity. type Checker struct { - db *sql.DB + db *sql.DB + gravityDB *sql.DB } const domainLookback = time.Hour @@ -67,17 +71,64 @@ func NewChecker(config *Config) (*Checker, error) { return nil, fmt.Errorf("failed to connect to pihole-db: %w", err) } + // Open gravity.db for policy list lookups. + gravityPath := config.GravityDBPath + if gravityPath == "" { + gravityPath = filepath.Join(filepath.Dir(config.DBPath), "gravity.db") + } + + var gravityDB *sql.DB + if err := openGravityDB(gravityPath, &gravityDB); err != nil { + slog.Warn("pi-hole gravity database unavailable; policy checks disabled", + "path", gravityPath, "err", err) + } + return &Checker{ - db: db, + db: db, + gravityDB: gravityDB, }, nil } -// Close closes the Checker database connection. +// openGravityDB opens the Pi-hole gravity database in read-only mode. +// On success it sets *out; on failure it returns an error but does not touch *out. +func openGravityDB(path string, out **sql.DB) error { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("cannot access %s: %w", path, err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close access check for %s: %w", path, err) + } + + dsn := "file:" + (&url.URL{Path: path, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return fmt.Errorf("failed to open gravity-db: %w", err) + } + + if err := db.Ping(); err != nil { + _ = db.Close() + return fmt.Errorf("failed to connect to gravity-db: %w", err) + } + + *out = db + return nil +} + +// Close closes both database connections. func (c *Checker) Close() error { - if c == nil || c.db == nil { + if c == nil { return nil } - return c.db.Close() + var errFTL error + var errGravity error + if c.db != nil { + errFTL = c.db.Close() + } + if c.gravityDB != nil { + errGravity = c.gravityDB.Close() + } + return errors.Join(errFTL, errGravity) } // IsDomainKnown reports whether domain appeared in Pi-hole query history within the last hour. @@ -109,38 +160,93 @@ func (c *Checker) IsDomainKnown(domain string) (bool, error) { return true, nil } -// IsBlockedByPolicy reports whether the domain currently matches a Pi-hole policy entry. +// IsAllowedByPolicy reports whether the domain currently appears in Pi-hole's allowlist. +func (c *Checker) IsAllowedByPolicy(domain string) (bool, error) { + domain = normalizeDomain(domain) + if domain == "" { + return false, nil + } + if c == nil || c.gravityDB == nil { + return false, nil + } + + var found int + err := c.gravityDB.QueryRow(` + SELECT 1 FROM vw_allowlist + WHERE domain = ? COLLATE NOCASE + LIMIT 1 + `, domain).Scan(&found) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("allowlist lookup failed: %w", err) + } + + return true, nil +} + +// IsBlockedByPolicy reports whether the domain currently matches a Pi-hole denylist or gravity entry. func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { domain = normalizeDomain(domain) if domain == "" { return false, nil } - if c == nil || c.db == nil { - return false, errors.New("pihole checker is not initialized") + if c == nil || c.gravityDB == nil { + return false, nil } var found int - err := c.db.QueryRow(` - SELECT 1 - FROM domainlist - WHERE type IN (1, 2) - AND TRIM(domain) <> '' - AND domain = ? COLLATE NOCASE + err := c.gravityDB.QueryRow(` + SELECT 1 FROM vw_denylist + WHERE domain = ? COLLATE NOCASE + LIMIT 1 + `, domain).Scan(&found) + if errors.Is(err, sql.ErrNoRows) { + // Not in denylist; check gravity. + } else if err != nil { + return false, fmt.Errorf("denylist lookup failed: %w", err) + } else { + return true, nil + } + + err = c.gravityDB.QueryRow(` + SELECT 1 FROM vw_gravity + WHERE domain = ? COLLATE NOCASE LIMIT 1 `, domain).Scan(&found) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { - if strings.Contains(err.Error(), "no such table") { - return false, nil - } - return false, fmt.Errorf("policy lookup failed: %w", err) + return false, fmt.Errorf("gravity lookup failed: %w", err) } return true, nil } +// PolicyCounts returns the number of entries in each Pi-hole policy view. +// The returned map is keyed by view name. Returns nil if gravity DB is unavailable. +func (c *Checker) PolicyCounts() (map[string]int64, error) { + if c == nil || c.gravityDB == nil { + return nil, nil + } + + views := []string{"vw_allowlist", "vw_denylist", "vw_gravity", "vw_regex_allowlist", "vw_regex_denylist"} + counts := make(map[string]int64, len(views)) + + for _, view := range views { + var count int64 + err := c.gravityDB.QueryRow(fmt.Sprintf(`SELECT COUNT(*) FROM %s`, view)).Scan(&count) + if err != nil { + return nil, fmt.Errorf("count %s: %w", view, err) + } + counts[view] = count + } + + return counts, nil +} + // DomainsSeenSince returns the distinct domains Pi-hole recorded since the given Unix timestamp. func (c *Checker) DomainsSeenSince(since int64) ([]string, error) { rows, err := c.db.Query(` @@ -178,6 +284,34 @@ func (c *Checker) DomainsSeenSince(since int64) ([]string, error) { return domains, nil } +// DomainMatches reports whether an observed SNI domain corresponds to a known domain, +// treating a subdomain as a match for its parent in either direction. +// +// The direction is intentionally symmetric: Zeek reports the SNI exactly as sent on the wire +// (www.github.com) while the store holds whatever Pi-hole logged and the cache resolved +// (github.com), and either can be the more specific name. +// +// The match requires a label boundary, so github.com.evil.com does not match github.com: +// it does not end in ".github.com". A plain suffix test would have accepted it. +// +// A registrable-domain (eTLD+1) comparison via golang.org/x/net/publicsuffix would handle more +// edge cases -- notably unrelated names under a shared public suffix -- but it adds a third-party +// dependency for a marginal gain, so the label-boundary rule is the deliberate v1 tradeoff. +func DomainMatches(observed, known string) bool { + observed = normalizeDomain(observed) + known = normalizeDomain(known) + + if observed == "" || known == "" { + return false + } + + if observed == known { + return true + } + + return strings.HasSuffix(observed, "."+known) || strings.HasSuffix(known, "."+observed) +} + func normalizeDomain(domain string) string { return strings.ToLower(strings.TrimSpace(domain)) } diff --git a/internal/pihole/policy_test.go b/internal/pihole/policy_test.go index ae9f2cc..2dffbef 100644 --- a/internal/pihole/policy_test.go +++ b/internal/pihole/policy_test.go @@ -1,7 +1,12 @@ package pihole import ( + "database/sql" + "net/url" + "path/filepath" "testing" + + _ "modernc.org/sqlite" ) func TestNormalizeDomain(t *testing.T) { @@ -9,3 +14,466 @@ func TestNormalizeDomain(t *testing.T) { t.Fatalf("normalizeDomain() = %q, want %q", got, "example.com") } } + +// openTestGravityDB creates a temporary SQLite database with Pi-hole-like views +// and returns the opened DB along with its path. The caller must close it. +func openTestGravityDB(t *testing.T) (*sql.DB, string) { + t.Helper() + dir := t.TempDir() + dbPath := filepath.Join(dir, "gravity.db") + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: dbPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open test gravity db: %v", err) + } + + // Create backing tables and views matching Pi-hole v6 shapes. + schema := ` + CREATE TABLE adlist (id INTEGER PRIMARY KEY, address TEXT, enabled INTEGER, date_added INTEGER, date_modified INTEGER, comment TEXT, type INTEGER, number INTEGER); + CREATE TABLE domainlist (id INTEGER PRIMARY KEY, type INTEGER, domain TEXT, enabled INTEGER, date_added INTEGER, date_modified INTEGER, comment TEXT, groups TEXT); + CREATE TABLE gravity (id INTEGER PRIMARY KEY, domain TEXT, adlist_id INTEGER); + CREATE INDEX idx_gravity ON gravity (domain, adlist_id); + + CREATE VIEW vw_allowlist AS SELECT domain FROM domainlist WHERE type = 0 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_denylist AS SELECT domain FROM domainlist WHERE type = 1 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_gravity AS SELECT domain FROM gravity; + CREATE VIEW vw_regex_allowlist AS SELECT domain FROM domainlist WHERE type = 2 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_regex_denylist AS SELECT domain FROM domainlist WHERE type = 3 AND enabled = 1 AND TRIM(domain) <> ''; + ` + if _, err := db.Exec(schema); err != nil { + _ = db.Close() + t.Fatalf("create schema: %v", err) + } + + return db, dbPath +} + +// insertDomain inserts a domain into the domainlist table with the given type. +// type 0 = allowlist, type 1 = denylist, type 2 = regex allowlist, type 3 = regex denylist. +func insertDomain(t *testing.T, db *sql.DB, domain string, listType int) { + t.Helper() + _, err := db.Exec(`INSERT INTO domainlist (type, domain, enabled) VALUES (?, ?, 1)`, listType, domain) + if err != nil { + t.Fatalf("insert domain %q type %d: %v", domain, listType, err) + } +} + +// insertGravity inserts a domain into the gravity table. +func insertGravity(t *testing.T, db *sql.DB, domain string) { + t.Helper() + _, err := db.Exec(`INSERT INTO gravity (domain, adlist_id) VALUES (?, 1)`, domain) + if err != nil { + t.Fatalf("insert gravity domain %q: %v", domain, err) + } +} + +func newTestChecker(t *testing.T, gravityPath string) *Checker { + t.Helper() + ftlPath := filepath.Join(t.TempDir(), "pihole-FTL.db") + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open test FTL db: %v", err) + } + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY, timestamp INTEGER, type INTEGER, domain TEXT, status INTEGER, ip TEXT)`); err != nil { + _ = db.Close() + t.Fatalf("create FTL schema: %v", err) + } + + var gravityDB *sql.DB + if gravityPath != "" { + gravityDB, err = sql.Open("sqlite", "file:"+(&url.URL{Path: gravityPath, RawQuery: "mode=ro"}).String()) + if err != nil { + _ = db.Close() + t.Fatalf("open gravity db: %v", err) + } + } + + return &Checker{db: db, gravityDB: gravityDB} +} + +func TestIsBlockedByPolicy_Denylist(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "blocked.example.com", 1) // type 1 = denylist + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + blocked, err := checker.IsBlockedByPolicy("blocked.example.com") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true for denylist domain") + } +} + +func TestIsBlockedByPolicy_Gravity(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertGravity(t, gravDB, "malware.example.net") + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + blocked, err := checker.IsBlockedByPolicy("malware.example.net") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true for gravity domain") + } +} + +func TestIsBlockedByPolicy_NoMatch(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertGravity(t, gravDB, "other.example.com") + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + blocked, err := checker.IsBlockedByPolicy("safe.example.org") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + if blocked { + t.Fatal("IsBlockedByPolicy() = true, want false for unknown domain") + } +} + +func TestIsAllowedByPolicy_Allowlist(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "trusted.example.com", 0) // type 0 = allowlist + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + allowed, err := checker.IsAllowedByPolicy("trusted.example.com") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v", err) + } + if !allowed { + t.Fatal("IsAllowedByPolicy() = false, want true for allowlist domain") + } +} + +func TestIsAllowedByPolicy_NoMatch(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "trusted.example.com", 0) + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + allowed, err := checker.IsAllowedByPolicy("unknown.example.org") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v", err) + } + if allowed { + t.Fatal("IsAllowedByPolicy() = true, want false for unknown domain") + } +} + +func TestPolicyLookup_CaseNormalization(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "CaseSensitive.COM", 1) + insertGravity(t, gravDB, "GravityDomain.ORG") + insertDomain(t, gravDB, "AllowCase.COM", 0) + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + // Denylist: case-insensitive match + blocked, err := checker.IsBlockedByPolicy(" casesensitive.com ") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true for case-insensitive denylist match") + } + + // Gravity: case-insensitive match + blocked, err = checker.IsBlockedByPolicy("GRAVITYDOMAIN.ORG") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true for case-insensitive gravity match") + } + + // Allowlist: case-insensitive match + allowed, err := checker.IsAllowedByPolicy("allowcase.com") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v", err) + } + if !allowed { + t.Fatal("IsAllowedByPolicy() = false, want true for case-insensitive allowlist match") + } +} + +func TestPolicyLookup_EmptyDomain(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "blocked.example.com", 1) + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + blocked, err := checker.IsBlockedByPolicy("") + if err != nil { + t.Fatalf("IsBlockedByPolicy(\"\") error = %v", err) + } + if blocked { + t.Fatal("IsBlockedByPolicy(\"\") = true, want false") + } + + allowed, err := checker.IsAllowedByPolicy("") + if err != nil { + t.Fatalf("IsAllowedByPolicy(\"\") error = %v", err) + } + if allowed { + t.Fatal("IsAllowedByPolicy(\"\") = true, want false") + } +} + +func TestPolicyLookup_NilGravityDB(t *testing.T) { + t.Parallel() + + // Create a checker with nil gravityDB (simulates gravity.db unavailable). + checker := &Checker{ + db: nil, // not needed for these methods + gravityDB: nil, + } + + blocked, err := checker.IsBlockedByPolicy("anything.example.com") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v, want nil", err) + } + if blocked { + t.Fatal("IsBlockedByPolicy() = true, want false for nil gravity DB") + } + + allowed, err := checker.IsAllowedByPolicy("anything.example.com") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v, want nil", err) + } + if allowed { + t.Fatal("IsAllowedByPolicy() = true, want false for nil gravity DB") + } +} + +func TestPolicyCounts(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "allow1.com", 0) + insertDomain(t, gravDB, "allow2.com", 0) + insertDomain(t, gravDB, "deny1.com", 1) + insertGravity(t, gravDB, "g1.com") + insertGravity(t, gravDB, "g2.com") + insertGravity(t, gravDB, "g3.com") + insertDomain(t, gravDB, "regex_allow", 2) + insertDomain(t, gravDB, "regex_deny1", 3) + insertDomain(t, gravDB, "regex_deny2", 3) + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + counts, err := checker.PolicyCounts() + if err != nil { + t.Fatalf("PolicyCounts() error = %v", err) + } + if counts == nil { + t.Fatal("PolicyCounts() = nil, want non-nil map") + } + + want := map[string]int64{ + "vw_allowlist": 2, + "vw_denylist": 1, + "vw_gravity": 3, + "vw_regex_allowlist": 1, + "vw_regex_denylist": 2, + } + for view, wantCount := range want { + if got := counts[view]; got != wantCount { + t.Errorf("PolicyCounts()[%q] = %d, want %d", view, got, wantCount) + } + } +} + +func TestPolicyCounts_NilGravityDB(t *testing.T) { + t.Parallel() + checker := &Checker{gravityDB: nil} + + counts, err := checker.PolicyCounts() + if err != nil { + t.Fatalf("PolicyCounts() error = %v, want nil", err) + } + if counts != nil { + t.Fatalf("PolicyCounts() = %v, want nil for nil gravity DB", counts) + } +} + +func TestGravityDBPath_Derived(t *testing.T) { + t.Parallel() + + // Create the FTL db in a known directory. + dir := t.TempDir() + ftlPath := filepath.Join(dir, "pihole-FTL.db") + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open FTL db: %v", err) + } + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY)`); err != nil { + _ = db.Close() + t.Fatalf("init FTL db: %v", err) + } + _ = db.Close() + + // Create gravity.db next to it. + gravPath := filepath.Join(dir, "gravity.db") + gravDB, err := sql.Open("sqlite", "file:"+(&url.URL{Path: gravPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open gravity db: %v", err) + } + // Create minimal schema so gravity opens cleanly. + if _, err := gravDB.Exec(`CREATE TABLE adlist (id INTEGER PRIMARY KEY); CREATE VIEW vw_gravity AS SELECT 1 WHERE 0;`); err != nil { + _ = gravDB.Close() + t.Fatalf("create gravity schema: %v", err) + } + _ = gravDB.Close() + + // NewChecker should auto-discover gravity.db next to pihole-FTL.db. + checker, err := NewChecker(&Config{DBPath: ftlPath}) + if err != nil { + t.Fatalf("NewChecker() error = %v", err) + } + defer checker.Close() + + if checker.gravityDB == nil { + t.Fatal("gravityDB is nil, want it to be auto-discovered") + } +} + +func TestGravityDBPath_Missing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + ftlPath := filepath.Join(dir, "pihole-FTL.db") + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open FTL db: %v", err) + } + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY)`); err != nil { + _ = db.Close() + t.Fatalf("init FTL db: %v", err) + } + _ = db.Close() + + // No gravity.db in dir — NewChecker should succeed with nil gravityDB. + checker, err := NewChecker(&Config{DBPath: ftlPath}) + if err != nil { + t.Fatalf("NewChecker() error = %v, want nil when gravity.db is missing", err) + } + defer checker.Close() + + if checker.gravityDB != nil { + t.Fatal("gravityDB is non-nil, want nil when gravity.db is missing") + } + + // Policy methods should gracefully return false. + blocked, err := checker.IsBlockedByPolicy("anything.com") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + if blocked { + t.Fatal("IsBlockedByPolicy() = true, want false when gravity unavailable") + } +} + +func TestCloseJoinsErrors(t *testing.T) { + t.Parallel() + + // Close on a checker with both handles should not panic. + gravDB, gravPath := openTestGravityDB(t) + _ = gravDB.Close() // close it first so Close() returns an error + + checker := newTestChecker(t, gravPath) + // Force gravity DB to nil after construction to test the nil path. + _ = checker.gravityDB.Close() + checker.gravityDB = nil + + err := checker.Close() + if err != nil { + // We expect an error from closing the already-closed FTL db or nil gravity. + // Just verify it doesn't panic. + t.Logf("Close() returned (expected for double-close): %v", err) + } +} + +func TestIsBlockedByPolicy_DenylistBeatsGravity(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "both.example.com", 1) // denylist + insertGravity(t, gravDB, "both.example.com") // also in gravity + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + blocked, err := checker.IsBlockedByPolicy("both.example.com") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true when domain is in both denylist and gravity") + } +} + +func TestIsAllowedByPolicy_TakesPrecedenceOverGravity(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "special.example.com", 0) // allowlist + insertGravity(t, gravDB, "special.example.com") // also in gravity + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + // Allowlist match should be independent of gravity presence. + allowed, err := checker.IsAllowedByPolicy("special.example.com") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v", err) + } + if !allowed { + t.Fatal("IsAllowedByPolicy() = false, want true for allowlisted domain") + } +} + +func TestIsBlockedByPolicy_WhitespaceNormalization(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertGravity(t, gravDB, "trim.example.com") + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer checker.Close() + + blocked, err := checker.IsBlockedByPolicy(" Trim.Example.COM ") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true with whitespace around domain") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index e234747..4f6945c 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -31,6 +31,14 @@ CREATE TABLE IF NOT EXISTS allowed_ips ( last_established INTEGER ); +CREATE TABLE IF NOT EXISTS allowed_ip_domains ( + ip TEXT NOT NULL, + domain TEXT NOT NULL, + first_seen INTEGER NOT NULL, + last_refreshed INTEGER NOT NULL, + PRIMARY KEY (ip, domain) +); + CREATE TABLE IF NOT EXISTS killed_connections ( id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, @@ -73,6 +81,14 @@ CREATE TABLE IF NOT EXISTS domain_checks ( ); ` +// backfillAllowedIPDomains seeds the per-IP domain set from the older IP-level trust table. +// It runs on every startup: INSERT OR IGNORE makes it idempotent, and without it an upgraded +// install would treat every Zeek-observed SNI as a mismatch until the first cache refresh. +const backfillAllowedIPDomains = ` +INSERT OR IGNORE INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) +SELECT ip, domain, first_approved, last_refreshed FROM allowed_ips WHERE TRIM(domain) <> ''; +` + // FraudCheckCacheEntry stores the cached kill decision for a prior fraud API lookup. type FraudCheckCacheEntry struct { ShouldKill bool @@ -121,6 +137,11 @@ func NewStore(dbPath string) (*Store, error) { return nil, fmt.Errorf("failed to apply schema: %w", err) } + if _, err := readWrite.Exec(backfillAllowedIPDomains); err != nil { + _ = readWrite.Close() + return nil, fmt.Errorf("failed to backfill allowed ip domains: %w", err) + } + readOnly, err := sql.Open("sqlite", sqliteDSN(dbPath, "ro")) if err != nil { _ = readWrite.Close() @@ -140,7 +161,10 @@ func sqliteDSN(dbPath, mode string) string { query := url.Values{} query.Set("mode", mode) - return "file:" + (&url.URL{Path: dbPath, RawQuery: query.Encode()}).String() + return "file:" + (&url.URL{ + Path: dbPath, + RawQuery: query.Encode(), + }).String() } func configureConnection(db *sql.DB, readOnly bool) error { @@ -206,6 +230,64 @@ func (s *Store) UpsertAllowedIP(ip, domain string) error { return nil } +// UpsertAllowedIPDomain inserts or refreshes one domain in the set of domains known for an IP. +// Unlike allowed_ips, which keeps a single representative domain per IP, this table records +// every domain observed for the IP so an SNI can be matched against that IP specifically. +// On conflict, it updates last_refreshed while preserving first_seen. +func (s *Store) UpsertAllowedIPDomain(ip, domain string) error { + domain = strings.ToLower(strings.TrimSpace(domain)) + now := time.Now().Unix() + + _, err := s.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + ON CONFLICT(ip, domain) DO UPDATE SET + last_refreshed = excluded.last_refreshed + `, ip, domain, now, now) + + if err != nil { + return fmt.Errorf("upsert allowed ip domain %s/%s: %w", ip, domain, err) + } + + return nil +} + +// DomainsForIP returns every domain on file for the IP that was refreshed within the trust window. +// It returns an empty slice when the IP has no fresh domains on record. +func (s *Store) DomainsForIP(ip string) ([]string, error) { + cutoff := time.Now().Add(-refreshTrustWindow).Unix() + + rows, err := s.readOnly.Query(` + SELECT domain + FROM allowed_ip_domains + WHERE ip = ? + AND last_refreshed >= ? + ORDER BY domain + `, ip, cutoff) + if err != nil { + return nil, fmt.Errorf("domains for ip %s: %w", ip, err) + } + defer func() { + _ = rows.Close() + }() + + domains := []string{} + for rows.Next() { + var domain string + if err := rows.Scan(&domain); err != nil { + return nil, fmt.Errorf("scan domain for ip %s: %w", ip, err) + } + + domains = append(domains, domain) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate domains for ip %s: %w", ip, err) + } + + return domains, nil +} + // UpdateEstablished stamps the current time as last_established for an IP. // The monitor calls it when somo confirms the connection is still active. func (s *Store) UpdateEstablished(ip string) error { @@ -334,6 +416,7 @@ func (s *Store) GetRecentDomainCheck(domain string) (*DomainCheckCacheEntry, err func (s *Store) UpsertDomainCheck(domain string, shouldBlock bool) error { domain = strings.ToLower(strings.TrimSpace(domain)) shouldBlockInt := 0 + if shouldBlock { shouldBlockInt = 1 } @@ -396,6 +479,7 @@ func (s *Store) RecentZeekAlert(srcIP, sni string) (*ZeekAlertEntry, error) { var confidence string var detectedAt int64 var actionTaken string + err := s.readOnly.QueryRow(` SELECT id, dst_ip, dst_port, blocked, confidence, detected_at, action_taken FROM zeek_alerts diff --git a/internal/store/store_test.go b/internal/store/store_test.go index f6e0a04..c0c3fd3 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -96,6 +96,184 @@ func TestFraudCheckCacheExpiresAfterWindow(t *testing.T) { } } +func TestAllowedIPDomainsMultiplePerIP(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + for _, domain := range []string{"github.com", "example.com", "EXAMPLE.NET"} { + if err := hexwallStore.UpsertAllowedIPDomain("203.0.113.30", domain); err != nil { + t.Fatalf("UpsertAllowedIPDomain(%q) error = %v", domain, err) + } + } + + domains, err := hexwallStore.DomainsForIP("203.0.113.30") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + + want := []string{"example.com", "example.net", "github.com"} + if len(domains) != len(want) { + t.Fatalf("DomainsForIP() = %v, want %v", domains, want) + } + for i, domain := range want { + if domains[i] != domain { + t.Fatalf("DomainsForIP()[%d] = %q, want %q", i, domains[i], domain) + } + } +} + +func TestDomainsForIPUnknownIP(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + if err := hexwallStore.UpsertAllowedIPDomain("203.0.113.30", "github.com"); err != nil { + t.Fatalf("UpsertAllowedIPDomain() error = %v", err) + } + + domains, err := hexwallStore.DomainsForIP("203.0.113.99") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + if len(domains) != 0 { + t.Fatalf("DomainsForIP() = %v, want empty for unknown ip", domains) + } +} + +func TestDomainsForIPExcludesStaleEntries(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + staleRefreshed := time.Now().Add(-refreshTrustWindow - time.Minute).Unix() + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.40", "stale.example.com", staleRefreshed, staleRefreshed); err != nil { + t.Fatalf("insert stale allowed ip domain error = %v", err) + } + + if err := hexwallStore.UpsertAllowedIPDomain("203.0.113.40", "fresh.example.com"); err != nil { + t.Fatalf("UpsertAllowedIPDomain() error = %v", err) + } + + domains, err := hexwallStore.DomainsForIP("203.0.113.40") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + if len(domains) != 1 || domains[0] != "fresh.example.com" { + t.Fatalf("DomainsForIP() = %v, want only the fresh domain", domains) + } +} + +func TestUpsertAllowedIPDomainRefreshesWithoutDuplicating(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + originalSeen := time.Now().Add(-30 * time.Minute).Unix() + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.50", "example.com", originalSeen, originalSeen); err != nil { + t.Fatalf("insert allowed ip domain error = %v", err) + } + + if err := hexwallStore.UpsertAllowedIPDomain("203.0.113.50", "example.com"); err != nil { + t.Fatalf("UpsertAllowedIPDomain() error = %v", err) + } + + var rowCount int + var firstSeen int64 + var lastRefreshed int64 + if err := hexwallStore.readOnly.QueryRow(` + SELECT COUNT(*), MIN(first_seen), MIN(last_refreshed) + FROM allowed_ip_domains + WHERE ip = ? + `, "203.0.113.50").Scan(&rowCount, &firstSeen, &lastRefreshed); err != nil { + t.Fatalf("query allowed ip domain error = %v", err) + } + + if rowCount != 1 { + t.Fatalf("row count = %d, want 1 after re-upsert", rowCount) + } + if firstSeen != originalSeen { + t.Fatalf("first_seen = %d, want preserved value %d", firstSeen, originalSeen) + } + if lastRefreshed <= originalSeen { + t.Fatalf("last_refreshed = %d, want a value newer than %d", lastRefreshed, originalSeen) + } +} + +func TestNewStoreBackfillsAllowedIPDomains(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + // UpsertAllowedIP writes only the IP-level record, mimicking a database written + // before allowed_ip_domains existed. + if err := hexwallStore.UpsertAllowedIP("203.0.113.60", "example.com"); err != nil { + t.Fatalf("UpsertAllowedIP() error = %v", err) + } + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + reopened, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() reopen error = %v", err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + domains, err := reopened.DomainsForIP("203.0.113.60") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + if len(domains) != 1 || domains[0] != "example.com" { + t.Fatalf("DomainsForIP() = %v, want [example.com] from backfill", domains) + } +} + func TestDomainCheckCacheRoundTrip(t *testing.T) { dbPath := filepath.Join(t.TempDir(), "hexwall.db") diff --git a/main.go b/main.go index 2d7aebd..d2c9df9 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,7 @@ type runConfig struct { zeekLogPath string zeekNoticeLog string enableZeek bool + enableDeghost bool mode string debug bool showVersion bool @@ -55,8 +56,11 @@ func parseFlags() (*runConfig, error) { zeekLogPath := flag.String("zeek-log", "", "path to zeek ssl.log for SNI-based domain verification (empty to skip)") zeekNoticeLog := flag.String("zeek-notice-log", "/opt/zeek/logs/current/notice.log", "path to Zeek notice.log for DNS-bypass alerts") enableZeek := flag.Bool("enable-zeek", false, "enable Zeek-based DNS-bypass detection") + // deghost sends every unrecognized domain the machine contacts to a third-party API. + // That is a real privacy cost and should be opt-in, not default. + enableDeghost := flag.Bool("enable-deghost", false, "enable third-party IP/domain reputation checks (privacy-sensitive; sends queried domains externally)") mode := flag.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") - debug := flag.Bool("debug", false, "enable verbose per-connection scan logging") + debug := flag.Bool("debug", false, "enable debug-level logging, including verbose per-connection scan logging") showVersion := flag.Bool("version", false, "print version and exit") flag.Parse() @@ -66,6 +70,7 @@ func parseFlags() (*runConfig, error) { zeekLogPath: strings.TrimSpace(*zeekLogPath), zeekNoticeLog: strings.TrimSpace(*zeekNoticeLog), enableZeek: *enableZeek, + enableDeghost: *enableDeghost, mode: strings.TrimSpace(*mode), debug: *debug, showVersion: *showVersion, @@ -99,6 +104,13 @@ func run() int { return 1 } + // Install the log handler first so every later slog call honours --debug. + logLevel := slog.LevelInfo + if cfg.debug { + logLevel = slog.LevelDebug + } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))) + if cfg.showVersion { fmt.Printf("%s (%s)\n", version, platform) return 0 @@ -138,6 +150,21 @@ func run() int { } }() + // 3b. Log policy list summary from gravity.db. + if counts, err := checker.PolicyCounts(); err != nil { + slog.Warn("failed to read pi-hole policy counts", "err", err) + } else if counts != nil { + slog.Info("pihole policy lists loaded", + "allow", counts["vw_allowlist"], + "deny", counts["vw_denylist"], + "gravity", counts["vw_gravity"], + "regex_allow", counts["vw_regex_allowlist"], + "regex_deny", counts["vw_regex_denylist"]) + if counts["vw_gravity"] == 0 { + slog.Warn("pi-hole gravity list is empty; blocklist may be broken or not yet downloaded") + } + } + // 4. Open the local hexwall database, creating it if needed. hexwallStore, err := store.NewStore(cfg.hexwallDBPath) if err != nil { @@ -187,7 +214,14 @@ func run() int { }() } - deghostClient := deghost.NewClient(deghostBaseURL, deghostTimeout) + // deghost is opt-in: it sends every unrecognized domain to a third-party API, + // which is a real privacy cost. Only construct the client when explicitly enabled. + var deghostClient *deghost.Client + if cfg.enableDeghost { + deghostClient = deghost.NewClient(deghostBaseURL, deghostTimeout) + } else { + slog.Info("deghost disabled; unrecognized connections will not be escalated to third-party API") + } // 5. Refresh trusted IPs before starting the monitor so the first tick does not kill legitimate connections. cache := pihole.NewIPCache(checker, hexwallStore) From 72a2ade082e3986da985c40c8a22038c250a9d71 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 3 Aug 2026 17:30:07 +0600 Subject: [PATCH 03/56] add (enforcer): enforcer package terminates processess --- internal/enforcer/enforcer.go | 114 ++++++++++++++++++++++ internal/enforcer/enforcer_test.go | 150 +++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 internal/enforcer/enforcer.go create mode 100644 internal/enforcer/enforcer_test.go diff --git a/internal/enforcer/enforcer.go b/internal/enforcer/enforcer.go new file mode 100644 index 0000000..ae77a25 --- /dev/null +++ b/internal/enforcer/enforcer.go @@ -0,0 +1,114 @@ +// Package enforcer terminates processes that hold blocked connections. +// +// STOPGAP: this is deliberately coarser than intended. The goal is to terminate a single +// connection; killing the owning process takes down every other connection it holds too. +// It exists because somo offers no non-interactive per-connection kill. Replace with +// `ss -K dst dport ` (requires CONFIG_INET_DIAG_DESTROY) when that work is done. +package enforcer + +import ( + "fmt" + "log/slog" + "os" + "strconv" + "strings" + "syscall" + "time" +) + +// protectedPrograms is the default set of programs that must never be killed. +// All keys MUST be lowercase: isProtected lowercases the input before lookup. +var protectedPrograms = map[string]bool{ + "systemd": true, + "init": true, + "sshd": true, + "networkmanager": true, + "systemd-resolved": true, + "systemd-networkd": true, + "dbus-daemon": true, + "containerd": true, + "dockerd": true, +} + +// SetProtectedPrograms replaces the default protected-program set. +// Keys are normalized to lowercase on entry because isProtected lowercases +// its input, so stored keys must match for the lookup to work. +func SetProtectedPrograms(programs map[string]bool) { + normalized := make(map[string]bool, len(programs)) + for k, v := range programs { + normalized[strings.ToLower(k)] = v + } + protectedPrograms = normalized +} + +// isProtected reports whether the program name (case-insensitive) is in the protected set. +func isProtected(name string) bool { + lower := strings.ToLower(name) + return protectedPrograms[lower] +} + +// KillProcess terminates the process holding a connection. +// +// Guardrails: +// - Refuses PID 1 (init). +// - Refuses hexwall's own PID. +// - Refuses any PID <= 0. +// - Refuses protected programs (case-insensitive match). +// - Sends SIGTERM first, waits a grace period, then SIGKILL if still alive. +func KillProcess(pid int, program string, remoteAddr string) error { + if pid <= 0 { + return fmt.Errorf("invalid PID %d", pid) + } + + if pid == 1 { + return fmt.Errorf("refusing to signal init (PID 1)") + } + + if pid == os.Getpid() { + return fmt.Errorf("refusing to signal own process (PID %d)", pid) + } + + if isProtected(program) { + return fmt.Errorf("refusing to kill protected program %q (PID %d)", program, pid) + } + + slog.Warn("killing connection", "pid", pid, "program", program, "address", remoteAddr) + + // SIGTERM first — allows graceful shutdown. + if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { + return fmt.Errorf("send SIGTERM to PID %d: %w", pid, err) + } + + // Grace period: wait up to 2 seconds for the process to exit. + time.Sleep(2 * time.Second) + + // Check whether the process still exists. + if err := syscall.Kill(pid, 0); err != nil { + // Process is gone — SIGTERM worked. + slog.Info("process terminated by SIGTERM", "pid", pid, "program", program) + return nil + } + + // Process still alive — escalate to SIGKILL. + slog.Warn("process survived SIGTERM, sending SIGKILL", "pid", pid, "program", program) + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + return fmt.Errorf("send SIGKILL to PID %d: %w", pid, err) + } + + return nil +} + +// ParsePID converts a string PID to int, returning an error for invalid input. +func ParsePID(s string) (int, error) { + s = strings.TrimSpace(s) + if s == "" || s == "-" { + return 0, fmt.Errorf("empty or dash PID %q", s) + } + + pid, err := strconv.Atoi(s) + if err != nil { + return 0, fmt.Errorf("invalid PID %q: %w", s, err) + } + + return pid, nil +} diff --git a/internal/enforcer/enforcer_test.go b/internal/enforcer/enforcer_test.go new file mode 100644 index 0000000..16fe2bd --- /dev/null +++ b/internal/enforcer/enforcer_test.go @@ -0,0 +1,150 @@ +package enforcer + +import ( + "os" + "strings" + "testing" +) + +func TestKillProcessRefusesPIDZero(t *testing.T) { + t.Parallel() + + err := KillProcess(0, "curl", "1.2.3.4:443") + if err == nil { + t.Fatal("KillProcess(0) = nil, want error") + } +} + +func TestKillProcessRefusesNegativePID(t *testing.T) { + t.Parallel() + + err := KillProcess(-1, "curl", "1.2.3.4:443") + if err == nil { + t.Fatal("KillProcess(-1) = nil, want error") + } +} + +func TestKillProcessRefusesPIDOne(t *testing.T) { + t.Parallel() + + err := KillProcess(1, "systemd", "1.2.3.4:443") + if err == nil { + t.Fatal("KillProcess(1) = nil, want error for init") + } +} + +func TestKillProcessRefusesOwnPID(t *testing.T) { + t.Parallel() + + err := KillProcess(os.Getpid(), "test", "1.2.3.4:443") + if err == nil { + t.Fatal("KillProcess(own PID) = nil, want error") + } +} + +func TestKillProcessRefusesProtectedPrograms(t *testing.T) { + t.Parallel() + + protected := []string{ + "systemd", "init", "sshd", "NetworkManager", + "systemd-resolved", "systemd-networkd", "dbus-daemon", + "containerd", "dockerd", + } + + for _, prog := range protected { + // Use a non-init, non-own PID that we know exists won't be matched + // by the other guards. PID 2 is usually kthreadd on Linux. + err := KillProcess(2, prog, "1.2.3.4:443") + if err == nil { + t.Errorf("KillProcess(2, %q) = nil, want error for protected program", prog) + continue + } + if !strings.Contains(err.Error(), "protected") { + t.Errorf("KillProcess(2, %q) error = %q, want it to mention 'protected'", prog, err) + } + } +} + +func TestKillProcessCaseInsensitiveProtectedMatch(t *testing.T) { + t.Parallel() + + cases := []string{"SSHD", "SystemD", "NETWORKMANAGER", "containerd", "DockerD"} + for _, prog := range cases { + err := KillProcess(2, prog, "1.2.3.4:443") + if err == nil { + t.Errorf("KillProcess(2, %q) = nil, want error for case-insensitive protected match", prog) + continue + } + if !strings.Contains(err.Error(), "protected") { + t.Errorf("KillProcess(2, %q) error = %q, want it to mention 'protected'", prog, err) + } + } +} + +func TestParsePIDValid(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want int + }{ + {"1234", 1234}, + {" 5678 ", 5678}, + {"0", 0}, + {"1", 1}, + } + + for _, tt := range tests { + got, err := ParsePID(tt.input) + if err != nil { + t.Errorf("ParsePID(%q) error = %v", tt.input, err) + } + if got != tt.want { + t.Errorf("ParsePID(%q) = %d, want %d", tt.input, got, tt.want) + } + } +} + +func TestParsePIDInvalid(t *testing.T) { + t.Parallel() + + tests := []string{"", "-", "abc", "12.34", "12ab"} + for _, input := range tests { + _, err := ParsePID(input) + if err == nil { + t.Errorf("ParsePID(%q) = nil, want error", input) + } + } +} + +func TestIsProtected(t *testing.T) { + t.Parallel() + + if !isProtected("sshd") { + t.Error("isProtected(\"sshd\") = false, want true") + } + if !isProtected("SSHD") { + t.Error("isProtected(\"SSHD\") = false, want true (case-insensitive)") + } + if !isProtected("SystemD") { + t.Error("isProtected(\"SystemD\") = false, want true (case-insensitive)") + } + if isProtected("curl") { + t.Error("isProtected(\"curl\") = true, want false") + } + if isProtected("") { + t.Error("isProtected(\"\") = true, want false") + } +} + +func TestIsProtectedMixedCaseAllEntries(t *testing.T) { + t.Parallel() + + for name := range protectedPrograms { + // Construct a mixed-case variant: uppercase first char + lowercase rest. + mixed := strings.ToUpper(name[:1]) + name[1:] + if !isProtected(mixed) { + t.Errorf("isProtected(%q) = false, want true for mixed-case spelling of protected program", mixed) + } + } +} From 4c0b523fb02c953db66c89662d3369ca383faa3e Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 3 Aug 2026 17:31:57 +0600 Subject: [PATCH 04/56] add (store): ip_observations table, SNIOutcome handling, and Prune method for clearing the database --- internal/store/store.go | 236 ++++++++++++- internal/store/store_test.go | 664 +++++++++++++++++++++++++++++++++++ 2 files changed, 888 insertions(+), 12 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 4f6945c..6ee62b2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -7,6 +7,7 @@ import ( "database/sql" "errors" "fmt" + "log/slog" "net/url" "strings" "time" @@ -20,6 +21,12 @@ const ( establishedTrustWindow = time.Minute fraudCheckCacheWindow = 6 * time.Hour sqliteBusyTimeout = 5 * time.Second + + // Prune windows: each table is deleted on a different schedule. + pruneSNIWindow = 7 * 24 * time.Hour // SNI observations: operational state data; 7 days is enough for hand-inspection. + pruneKilledAlertsWindow = 90 * 24 * time.Hour // Audit tables (killed_connections, zeek_alerts): rare events, forensically valuable. + // fraud_checks and domain_checks are bounded by upsert-per-key and fraudCheckCacheWindow (6h); + // they never grow beyond one row per IP/domain and are self-cleaning on read, so pruning is unnecessary. ) const schema = ` @@ -54,12 +61,14 @@ CREATE TABLE IF NOT EXISTS fraud_checks ( ); CREATE TABLE IF NOT EXISTS sni_observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, domain TEXT NOT NULL, local_port TEXT NOT NULL, - known INTEGER NOT NULL, - observed_at INTEGER NOT NULL + outcome TEXT NOT NULL, + first_seen INTEGER NOT NULL, + last_seen INTEGER NOT NULL, + times_seen INTEGER NOT NULL, + PRIMARY KEY (ip, domain, local_port) ); CREATE TABLE IF NOT EXISTS zeek_alerts ( @@ -79,6 +88,16 @@ CREATE TABLE IF NOT EXISTS domain_checks ( should_block INTEGER NOT NULL, checked_at INTEGER NOT NULL ); + +CREATE TABLE IF NOT EXISTS ip_observations ( + ip TEXT NOT NULL, + program TEXT NOT NULL, + outcome TEXT NOT NULL, + first_seen INTEGER NOT NULL, + last_seen INTEGER NOT NULL, + times_seen INTEGER NOT NULL, + PRIMARY KEY (ip, program) +); ` // backfillAllowedIPDomains seeds the per-IP domain set from the older IP-level trust table. @@ -101,6 +120,64 @@ type DomainCheckCacheEntry struct { CheckedAt int64 } +// SNIOutcome records which rung of the decision ladder classified a connection. +type SNIOutcome string + +const ( + OutcomePolicyAllowed SNIOutcome = "policy-allowed" + OutcomePolicyBlocked SNIOutcome = "policy-blocked" + OutcomeIPDomainMatch SNIOutcome = "ip-domain-match" + OutcomeUnknown SNIOutcome = "unknown" +) + +// Valid reports whether the outcome is one of the known values. +func (o SNIOutcome) Valid() bool { + switch o { + case OutcomePolicyAllowed, OutcomePolicyBlocked, OutcomeIPDomainMatch, OutcomeUnknown: + return true + } + return false +} + +// ParseSNIOutcome converts a string read from the database into a typed SNIOutcome. +// Rows written by older builds may contain unrecognized values; those parse to OutcomeUnknown with false. +func ParseSNIOutcome(s string) (SNIOutcome, bool) { + o := SNIOutcome(s) + if o.Valid() { + return o, true + } + return OutcomeUnknown, false +} + +// IPOutcome records how a connection was classified on the IP-only fallback path. +type IPOutcome string + +const ( + OutcomeIPTrusted IPOutcome = "ip-trusted" // present and fresh in allowed_ips + OutcomeIPReserved IPOutcome = "private-reserved" // deghost returned 403: private or reserved address + OutcomeIPBlocked IPOutcome = "deghost-blocked" // deghost verdict said kill + OutcomeIPUnknown IPOutcome = "unknown" // no verdict available or check disabled +) + +// Valid reports whether the outcome is one of the known values. +func (o IPOutcome) Valid() bool { + switch o { + case OutcomeIPTrusted, OutcomeIPReserved, OutcomeIPBlocked, OutcomeIPUnknown: + return true + } + return false +} + +// ParseIPOutcome converts a string read from the database into a typed IPOutcome. +// Rows written by older builds may contain unrecognized values; those parse to OutcomeIPUnknown with false. +func ParseIPOutcome(s string) (IPOutcome, bool) { + o := IPOutcome(s) + if o.Valid() { + return o, true + } + return OutcomeIPUnknown, false +} + // ZeekAlertEntry stores a Zeek-derived alert that was persisted for audit. type ZeekAlertEntry struct { ID int64 @@ -120,6 +197,46 @@ type Store struct { readOnly *sql.DB } +// migrateSNITable detects the old event-log schema (identified by a "known" column) +// and drops it so the new state-table schema is applied by the CREATE TABLE IF NOT EXISTS. +// This is safe specifically because nothing in the codebase reads this table; a future reader +// must not assume this reasoning generalises to other tables. +func migrateSNITable(db *sql.DB) error { + cols, err := db.Query(`PRAGMA table_info(sni_observations)`) + if err != nil { + return fmt.Errorf("query table info: %w", err) + } + defer cols.Close() + + hasOldSchema := false + for cols.Next() { + var cid int + var name, ctype string + var notNull int + var dfltValue sql.NullString + var pk int + if err := cols.Scan(&cid, &name, &ctype, ¬Null, &dfltValue, &pk); err != nil { + return fmt.Errorf("scan table info: %w", err) + } + if name == "known" { + hasOldSchema = true + } + } + if err := cols.Err(); err != nil { + return fmt.Errorf("iterate table info: %w", err) + } + + if !hasOldSchema { + return nil + } + + slog.Warn("dropping old sni_observations table (write-only audit data; no readers)", "reason", "schema migration to state-table format") + if _, err := db.Exec(`DROP TABLE sni_observations`); err != nil { + return fmt.Errorf("drop old sni_observations: %w", err) + } + return nil +} + // NewStore opens or creates the hexwall database at dbPath and applies the schema. func NewStore(dbPath string) (*Store, error) { readWrite, err := sql.Open("sqlite", sqliteDSN(dbPath, "rwc")) @@ -137,6 +254,21 @@ func NewStore(dbPath string) (*Store, error) { return nil, fmt.Errorf("failed to apply schema: %w", err) } + // Migration: detect and replace old sni_observations event-log schema with the new state table. + // The old table used an autoincrement id + known INTEGER, which is structurally incompatible + // with the new (ip, domain, local_port) primary key. This is safe to drop because nothing in + // the codebase reads this table — it was only ever written to and consulted by hand. + if err := migrateSNITable(readWrite); err != nil { + _ = readWrite.Close() + return nil, fmt.Errorf("failed to migrate sni_observations: %w", err) + } + + // Re-apply schema after migration so that any table dropped by migrateSNITable is recreated. + if _, err := readWrite.Exec(schema); err != nil { + _ = readWrite.Close() + return nil, fmt.Errorf("failed to re-apply schema after migration: %w", err) + } + if _, err := readWrite.Exec(backfillAllowedIPDomains); err != nil { _ = readWrite.Close() return nil, fmt.Errorf("failed to backfill allowed ip domains: %w", err) @@ -509,17 +641,21 @@ func (s *Store) RecentZeekAlert(srcIP, sni string) (*ZeekAlertEntry, error) { }, nil } -// LogSNIObservation records an SNI domain decision for auditability. -func (s *Store) LogSNIObservation(ip, domain, localPort string, known bool) error { - knownInt := 0 - if known { - knownInt = 1 - } +// LogSNIObservation records an SNI domain decision as a state-table upsert. +// On first sight it inserts the row; on subsequent observations it overwrites the outcome, +// updates last_seen, and increments times_seen while preserving first_seen. +func (s *Store) LogSNIObservation(ip, domain, localPort string, outcome SNIOutcome) error { + domain = strings.ToLower(strings.TrimSpace(domain)) + now := time.Now().Unix() _, err := s.readWrite.Exec(` - INSERT INTO sni_observations (ip, domain, local_port, known, observed_at) - VALUES (?, ?, ?, ?, ?) - `, ip, domain, localPort, knownInt, time.Now().Unix()) + INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?, 1) + ON CONFLICT(ip, domain, local_port) DO UPDATE SET + outcome = excluded.outcome, + last_seen = excluded.last_seen, + times_seen = sni_observations.times_seen + 1 + `, ip, domain, localPort, string(outcome), now, now) if err != nil { return fmt.Errorf("log sni observation %s: %w", ip, err) @@ -527,3 +663,79 @@ func (s *Store) LogSNIObservation(ip, domain, localPort string, known bool) erro return nil } + +// LogIPObservation records an IP-only fallback decision as a state-table upsert. +// On first sight it inserts the row; on subsequent observations it overwrites the outcome, +// updates last_seen, and increments times_seen while preserving first_seen. +func (s *Store) LogIPObservation(ip, program string, outcome IPOutcome) error { + now := time.Now().Unix() + + _, err := s.readWrite.Exec(` + INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, 1) + ON CONFLICT(ip, program) DO UPDATE SET + outcome = excluded.outcome, + last_seen = excluded.last_seen, + times_seen = ip_observations.times_seen + 1 + `, ip, program, string(outcome), now, now) + + if err != nil { + return fmt.Errorf("log ip observation %s: %w", ip, err) + } + + return nil +} + +// PruneResult reports how many rows were removed per table in a single Prune call. +type PruneResult struct { + AllowedIPDomains int64 + SNIObservations int64 + IPObservations int64 + KilledConnections int64 + ZeekAlerts int64 +} + +// Prune removes stale rows from all pruneable tables. +// It returns the number of rows deleted per table so the caller can log a summary. +func (s *Store) Prune() (*PruneResult, error) { + now := time.Now().Unix() + result := &PruneResult{} + + // allowed_ip_domains: rows older than refreshTrustWindow are already excluded by + // DomainsForIP and can never grant trust again. Safe to remove. + r, err := s.readWrite.Exec(`DELETE FROM allowed_ip_domains WHERE last_refreshed < ?`, now-int64(refreshTrustWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune allowed_ip_domains: %w", err) + } + result.AllowedIPDomains, _ = r.RowsAffected() + + // sni_observations: operational state data with a 7-day window. + r, err = s.readWrite.Exec(`DELETE FROM sni_observations WHERE last_seen < ?`, now-int64(pruneSNIWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune sni_observations: %w", err) + } + result.SNIObservations, _ = r.RowsAffected() + + // ip_observations: same 7-day window as sni_observations. + r, err = s.readWrite.Exec(`DELETE FROM ip_observations WHERE last_seen < ?`, now-int64(pruneSNIWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune ip_observations: %w", err) + } + result.IPObservations, _ = r.RowsAffected() + + // killed_connections and zeek_alerts: audit records of rare, potentially forensically + // valuable events. 90-day window keeps a meaningful history without unbounded growth. + r, err = s.readWrite.Exec(`DELETE FROM killed_connections WHERE killed_at < ?`, now-int64(pruneKilledAlertsWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune killed_connections: %w", err) + } + result.KilledConnections, _ = r.RowsAffected() + + r, err = s.readWrite.Exec(`DELETE FROM zeek_alerts WHERE detected_at < ?`, now-int64(pruneKilledAlertsWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune zeek_alerts: %w", err) + } + result.ZeekAlerts, _ = r.RowsAffected() + + return result, nil +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index c0c3fd3..3f6eac5 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -372,3 +372,667 @@ func TestDomainCheckCacheExpiresAfterWindow(t *testing.T) { t.Fatalf("GetRecentDomainCheck() = %#v, want nil for stale entry", entry) } } + +func TestParseSNIOutcomeValid(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want SNIOutcome + }{ + {"policy-allowed", OutcomePolicyAllowed}, + {"policy-blocked", OutcomePolicyBlocked}, + {"ip-domain-match", OutcomeIPDomainMatch}, + {"unknown", OutcomeUnknown}, + } + + for _, tt := range tests { + got, ok := ParseSNIOutcome(tt.input) + if !ok { + t.Errorf("ParseSNIOutcome(%q) ok = false, want true", tt.input) + } + if got != tt.want { + t.Errorf("ParseSNIOutcome(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestParseSNIOutcomeInvalid(t *testing.T) { + t.Parallel() + + tests := []string{"", "bogus", "true", "false", "1", "0"} + for _, input := range tests { + got, ok := ParseSNIOutcome(input) + if ok { + t.Errorf("ParseSNIOutcome(%q) ok = true, want false", input) + } + if got != OutcomeUnknown { + t.Errorf("ParseSNIOutcome(%q) = %q, want %q", input, got, OutcomeUnknown) + } + } +} + +func TestSNIOutcomeValid(t *testing.T) { + t.Parallel() + + valid := []SNIOutcome{OutcomePolicyAllowed, OutcomePolicyBlocked, OutcomeIPDomainMatch, OutcomeUnknown} + for _, o := range valid { + if !o.Valid() { + t.Errorf("SNIOutcome(%q).Valid() = false, want true", o) + } + } + + invalid := []SNIOutcome{"bogus", "", "true"} + for _, o := range invalid { + if o.Valid() { + t.Errorf("SNIOutcome(%q).Valid() = true, want false", o) + } + } +} + +func TestLogSNIObservationInsertAndUpsert(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + // First insert. + if err := hexwallStore.LogSNIObservation("203.0.113.10", "example.com", "443", OutcomePolicyAllowed); err != nil { + t.Fatalf("LogSNIObservation() error = %v", err) + } + + var timesSeen int + var firstSeen, lastSeen int64 + var outcome string + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, first_seen, last_seen, outcome + FROM sni_observations + WHERE ip = ? AND domain = ? AND local_port = ? + `, "203.0.113.10", "example.com", "443").Scan(×Seen, &firstSeen, &lastSeen, &outcome) + if err != nil { + t.Fatalf("query sni_observations error = %v", err) + } + if timesSeen != 1 { + t.Fatalf("times_seen = %d, want 1 after first insert", timesSeen) + } + if firstSeen <= 0 { + t.Fatalf("first_seen = %d, want positive unix timestamp", firstSeen) + } + if lastSeen != firstSeen { + t.Fatalf("last_seen = %d, want equal to first_seen on first insert, got %d", lastSeen, firstSeen) + } + if outcome != "policy-allowed" { + t.Fatalf("outcome = %q, want %q", outcome, "policy-allowed") + } + + // Second insert with different outcome — should upsert. + if err := hexwallStore.LogSNIObservation("203.0.113.10", "example.com", "443", OutcomeIPDomainMatch); err != nil { + t.Fatalf("LogSNIObservation() upsert error = %v", err) + } + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, first_seen, last_seen, outcome + FROM sni_observations + WHERE ip = ? AND domain = ? AND local_port = ? + `, "203.0.113.10", "example.com", "443").Scan(×Seen, &firstSeen, &lastSeen, &outcome) + if err != nil { + t.Fatalf("query sni_observations after upsert error = %v", err) + } + if timesSeen != 2 { + t.Fatalf("times_seen = %d, want 2 after upsert", timesSeen) + } + if outcome != "ip-domain-match" { + t.Fatalf("outcome = %q, want %q after upsert", outcome, "ip-domain-match") + } + if lastSeen < firstSeen { + t.Fatalf("last_seen = %d should not be older than first_seen = %d", lastSeen, firstSeen) + } + + // Third insert — times_seen should be 3. + if err := hexwallStore.LogSNIObservation("203.0.113.10", "example.com", "443", OutcomePolicyBlocked); err != nil { + t.Fatalf("LogSNIObservation() third insert error = %v", err) + } + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, outcome + FROM sni_observations + WHERE ip = ? AND domain = ? AND local_port = ? + `, "203.0.113.10", "example.com", "443").Scan(×Seen, &outcome) + if err != nil { + t.Fatalf("query sni_observations after third insert error = %v", err) + } + if timesSeen != 3 { + t.Fatalf("times_seen = %d, want 3 after third insert", timesSeen) + } + if outcome != "policy-blocked" { + t.Fatalf("outcome = %q, want %q after third insert", outcome, "policy-blocked") + } +} + +func TestLogSNIObservationNormalizesDomain(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + if err := hexwallStore.LogSNIObservation("203.0.113.10", " EXAMPLE.COM ", "443", OutcomePolicyAllowed); err != nil { + t.Fatalf("LogSNIObservation() error = %v", err) + } + + // Lookup by the normalized form. + var outcome string + err = hexwallStore.readOnly.QueryRow(` + SELECT outcome FROM sni_observations + WHERE ip = ? AND domain = ? AND local_port = ? + `, "203.0.113.10", "example.com", "443").Scan(&outcome) + if err != nil { + t.Fatalf("query sni_observations for normalized domain error = %v", err) + } + if outcome != "policy-allowed" { + t.Fatalf("outcome = %q, want %q", outcome, "policy-allowed") + } +} + +func TestNewStoreMigratesOldSNITable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + // Create a store with the old schema manually. + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + // Drop the new table and recreate with the old schema. + if _, err := hexwallStore.readWrite.Exec(`DROP TABLE sni_observations`); err != nil { + t.Fatalf("drop sni_observations error = %v", err) + } + if _, err := hexwallStore.readWrite.Exec(` + CREATE TABLE sni_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL, domain TEXT NOT NULL, local_port TEXT NOT NULL, + known INTEGER NOT NULL, observed_at INTEGER NOT NULL + ) + `); err != nil { + t.Fatalf("create old sni_observations error = %v", err) + } + // Insert a row in the old format. + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, known, observed_at) + VALUES (?, ?, ?, ?, ?) + `, "203.0.113.20", "old.example.com", "443", 1, time.Now().Unix()); err != nil { + t.Fatalf("insert old sni_observations error = %v", err) + } + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + // Reopen — migration should drop the old table and create the new one. + reopened, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() reopen error = %v", err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + // The old row should be gone. + var count int + err = reopened.readOnly.QueryRow(`SELECT COUNT(*) FROM sni_observations`).Scan(&count) + if err != nil { + t.Fatalf("query sni_observations count error = %v", err) + } + if count != 0 { + t.Fatalf("row count = %d, want 0 after migration dropped old table", count) + } + + // The new schema should work. + if err := reopened.LogSNIObservation("203.0.113.20", "new.example.com", "443", OutcomePolicyAllowed); err != nil { + t.Fatalf("LogSNIObservation() after migration error = %v", err) + } + + err = reopened.readOnly.QueryRow(`SELECT COUNT(*) FROM sni_observations`).Scan(&count) + if err != nil { + t.Fatalf("query sni_observations count after insert error = %v", err) + } + if count != 1 { + t.Fatalf("row count = %d, want 1 after insert with new schema", count) + } +} + +func TestPruneRemovesStaleRows(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + now := time.Now().Unix() + + // Insert stale allowed_ip_domains (older than refreshTrustWindow). + staleRefreshed := now - int64(refreshTrustWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.30", "stale.example.com", staleRefreshed, staleRefreshed); err != nil { + t.Fatalf("insert stale allowed_ip_domains error = %v", err) + } + + // Insert fresh allowed_ip_domains. + freshRefreshed := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.30", "fresh.example.com", freshRefreshed, freshRefreshed); err != nil { + t.Fatalf("insert fresh allowed_ip_domains error = %v", err) + } + + // Insert stale sni_observations (older than 7 days). + staleSNI := now - int64(pruneSNIWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, "203.0.113.30", "stale.example.com", "443", "unknown", staleSNI, staleSNI, 5); err != nil { + t.Fatalf("insert stale sni_observations error = %v", err) + } + + // Insert fresh sni_observations. + freshSNI := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, "203.0.113.30", "fresh.example.com", "443", "unknown", freshSNI, freshSNI, 1); err != nil { + t.Fatalf("insert fresh sni_observations error = %v", err) + } + + // Insert stale killed_connections (older than 90 days). + staleKill := now - int64(pruneKilledAlertsWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO killed_connections (ip, pid, program, killed_at) + VALUES (?, ?, ?, ?) + `, "203.0.113.30", "1234", "curl", staleKill); err != nil { + t.Fatalf("insert stale killed_connections error = %v", err) + } + + // Insert fresh killed_connections. + freshKill := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO killed_connections (ip, pid, program, killed_at) + VALUES (?, ?, ?, ?) + `, "203.0.113.30", "5678", "wget", freshKill); err != nil { + t.Fatalf("insert fresh killed_connections error = %v", err) + } + + // Insert stale zeek_alerts. + staleZeek := now - int64(pruneKilledAlertsWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "10.0.0.1", "203.0.113.30", "443", "old.example.com", 0, "low", staleZeek, "logged"); err != nil { + t.Fatalf("insert stale zeek_alerts error = %v", err) + } + + // Insert fresh zeek_alerts. + freshZeek := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "10.0.0.2", "203.0.113.30", "443", "new.example.com", 1, "high", freshZeek, "logged"); err != nil { + t.Fatalf("insert fresh zeek_alerts error = %v", err) + } + + // Run prune. + result, err := hexwallStore.Prune() + if err != nil { + t.Fatalf("Prune() error = %v", err) + } + + if result.AllowedIPDomains != 1 { + t.Fatalf("Prune().AllowedIPDomains = %d, want 1", result.AllowedIPDomains) + } + if result.SNIObservations != 1 { + t.Fatalf("Prune().SNIObservations = %d, want 1", result.SNIObservations) + } + if result.KilledConnections != 1 { + t.Fatalf("Prune().KilledConnections = %d, want 1", result.KilledConnections) + } + if result.ZeekAlerts != 1 { + t.Fatalf("Prune().ZeekAlerts = %d, want 1", result.ZeekAlerts) + } + + // Verify fresh rows remain. + var count int + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM allowed_ip_domains`).Scan(&count); err != nil { + t.Fatalf("query allowed_ip_domains count error = %v", err) + } + if count != 1 { + t.Fatalf("allowed_ip_domains count = %d, want 1 fresh row remaining", count) + } + + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM sni_observations`).Scan(&count); err != nil { + t.Fatalf("query sni_observations count error = %v", err) + } + if count != 1 { + t.Fatalf("sni_observations count = %d, want 1 fresh row remaining", count) + } + + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM killed_connections`).Scan(&count); err != nil { + t.Fatalf("query killed_connections count error = %v", err) + } + if count != 1 { + t.Fatalf("killed_connections count = %d, want 1 fresh row remaining", count) + } + + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM zeek_alerts`).Scan(&count); err != nil { + t.Fatalf("query zeek_alerts count error = %v", err) + } + if count != 1 { + t.Fatalf("zeek_alerts count = %d, want 1 fresh row remaining", count) + } +} + +func TestPruneNoopWhenNothingExpired(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + now := time.Now().Unix() + + // Insert fresh rows in all pruneable tables. + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.40", "fresh.example.com", now, now); err != nil { + t.Fatalf("insert fresh allowed_ip_domains error = %v", err) + } + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, "203.0.113.40", "fresh.example.com", "443", "unknown", now, now, 1); err != nil { + t.Fatalf("insert fresh sni_observations error = %v", err) + } + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO killed_connections (ip, pid, program, killed_at) + VALUES (?, ?, ?, ?) + `, "203.0.113.40", "1234", "curl", now); err != nil { + t.Fatalf("insert fresh killed_connections error = %v", err) + } + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "10.0.0.1", "203.0.113.40", "443", "fresh.example.com", 0, "low", now, "logged"); err != nil { + t.Fatalf("insert fresh zeek_alerts error = %v", err) + } + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?) + `, "203.0.113.40", "curl", "unknown", now, now, 1); err != nil { + t.Fatalf("insert fresh ip_observations error = %v", err) + } + + result, err := hexwallStore.Prune() + if err != nil { + t.Fatalf("Prune() error = %v", err) + } + + total := result.AllowedIPDomains + result.SNIObservations + result.IPObservations + result.KilledConnections + result.ZeekAlerts + if total != 0 { + t.Fatalf("Prune() deleted %d total rows, want 0 when nothing is expired", total) + } +} + +func TestParseIPOutcomeValid(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want IPOutcome + }{ + {"ip-trusted", OutcomeIPTrusted}, + {"private-reserved", OutcomeIPReserved}, + {"deghost-blocked", OutcomeIPBlocked}, + {"unknown", OutcomeIPUnknown}, + } + + for _, tt := range tests { + got, ok := ParseIPOutcome(tt.input) + if !ok { + t.Errorf("ParseIPOutcome(%q) ok = false, want true", tt.input) + } + if got != tt.want { + t.Errorf("ParseIPOutcome(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestParseIPOutcomeInvalid(t *testing.T) { + t.Parallel() + + tests := []string{"", "bogus", "true", "false", "1", "0"} + for _, input := range tests { + got, ok := ParseIPOutcome(input) + if ok { + t.Errorf("ParseIPOutcome(%q) ok = true, want false", input) + } + if got != OutcomeIPUnknown { + t.Errorf("ParseIPOutcome(%q) = %q, want %q", input, got, OutcomeIPUnknown) + } + } +} + +func TestIPOutcomeValid(t *testing.T) { + t.Parallel() + + valid := []IPOutcome{OutcomeIPTrusted, OutcomeIPReserved, OutcomeIPBlocked, OutcomeIPUnknown} + for _, o := range valid { + if !o.Valid() { + t.Errorf("IPOutcome(%q).Valid() = false, want true", o) + } + } + + invalid := []IPOutcome{"bogus", "", "true"} + for _, o := range invalid { + if o.Valid() { + t.Errorf("IPOutcome(%q).Valid() = true, want false", o) + } + } +} + +func TestLogIPObservationInsertAndUpsert(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + // First insert. + if err := hexwallStore.LogIPObservation("203.0.113.10", "curl", OutcomeIPTrusted); err != nil { + t.Fatalf("LogIPObservation() error = %v", err) + } + + var timesSeen int + var firstSeen, lastSeen int64 + var outcome string + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, first_seen, last_seen, outcome + FROM ip_observations + WHERE ip = ? AND program = ? + `, "203.0.113.10", "curl").Scan(×Seen, &firstSeen, &lastSeen, &outcome) + if err != nil { + t.Fatalf("query ip_observations error = %v", err) + } + if timesSeen != 1 { + t.Fatalf("times_seen = %d, want 1 after first insert", timesSeen) + } + if firstSeen <= 0 { + t.Fatalf("first_seen = %d, want positive unix timestamp", firstSeen) + } + if lastSeen != firstSeen { + t.Fatalf("last_seen = %d, want equal to first_seen on first insert, got %d", lastSeen, firstSeen) + } + if outcome != "ip-trusted" { + t.Fatalf("outcome = %q, want %q", outcome, "ip-trusted") + } + + // Second insert with different outcome — should upsert. + if err := hexwallStore.LogIPObservation("203.0.113.10", "curl", OutcomeIPBlocked); err != nil { + t.Fatalf("LogIPObservation() upsert error = %v", err) + } + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, first_seen, last_seen, outcome + FROM ip_observations + WHERE ip = ? AND program = ? + `, "203.0.113.10", "curl").Scan(×Seen, &firstSeen, &lastSeen, &outcome) + if err != nil { + t.Fatalf("query ip_observations after upsert error = %v", err) + } + if timesSeen != 2 { + t.Fatalf("times_seen = %d, want 2 after upsert", timesSeen) + } + if outcome != "deghost-blocked" { + t.Fatalf("outcome = %q, want %q after upsert", outcome, "deghost-blocked") + } + if lastSeen < firstSeen { + t.Fatalf("last_seen = %d should not be older than first_seen = %d", lastSeen, firstSeen) + } + + // Third insert — times_seen should be 3. + if err := hexwallStore.LogIPObservation("203.0.113.10", "curl", OutcomeIPReserved); err != nil { + t.Fatalf("LogIPObservation() third insert error = %v", err) + } + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, outcome + FROM ip_observations + WHERE ip = ? AND program = ? + `, "203.0.113.10", "curl").Scan(×Seen, &outcome) + if err != nil { + t.Fatalf("query ip_observations after third insert error = %v", err) + } + if timesSeen != 3 { + t.Fatalf("times_seen = %d, want 3 after third insert", timesSeen) + } + if outcome != "private-reserved" { + t.Fatalf("outcome = %q, want %q after third insert", outcome, "private-reserved") + } +} + +func TestLogIPObservationDifferentPrograms(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + // Same IP, different programs — should create separate rows. + if err := hexwallStore.LogIPObservation("203.0.113.20", "curl", OutcomeIPTrusted); err != nil { + t.Fatalf("LogIPObservation() error = %v", err) + } + if err := hexwallStore.LogIPObservation("203.0.113.20", "wget", OutcomeIPBlocked); err != nil { + t.Fatalf("LogIPObservation() error = %v", err) + } + + var count int + err = hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, "203.0.113.20").Scan(&count) + if err != nil { + t.Fatalf("query ip_observations count error = %v", err) + } + if count != 2 { + t.Fatalf("row count = %d, want 2 for same IP with different programs", count) + } +} + +func TestPruneRemovesStaleIPObservations(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + now := time.Now().Unix() + + // Insert stale ip_observations (older than 7 days). + staleIP := now - int64(pruneSNIWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?) + `, "203.0.113.50", "curl", "unknown", staleIP, staleIP, 5); err != nil { + t.Fatalf("insert stale ip_observations error = %v", err) + } + + // Insert fresh ip_observations. + freshIP := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?) + `, "203.0.113.50", "wget", "unknown", freshIP, freshIP, 1); err != nil { + t.Fatalf("insert fresh ip_observations error = %v", err) + } + + // Run prune. + result, err := hexwallStore.Prune() + if err != nil { + t.Fatalf("Prune() error = %v", err) + } + + if result.IPObservations != 1 { + t.Fatalf("Prune().IPObservations = %d, want 1", result.IPObservations) + } + + // Verify fresh row remains. + var count int + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM ip_observations`).Scan(&count); err != nil { + t.Fatalf("query ip_observations count error = %v", err) + } + if count != 1 { + t.Fatalf("ip_observations count = %d, want 1 fresh row remaining", count) + } +} From 4d8d9781a0f6ea3b97bee8b182210e2dc620b3db Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 3 Aug 2026 17:32:32 +0600 Subject: [PATCH 05/56] update (somo): moved process kill to enforcer package --- internal/somo/somo.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/internal/somo/somo.go b/internal/somo/somo.go index 79d9a73..b3bbb94 100644 --- a/internal/somo/somo.go +++ b/internal/somo/somo.go @@ -46,28 +46,6 @@ func GetEstablishedConnections() ([]Connection, error) { return result, nil } -// KillConnection kills a process by PID via somo. -func KillConnection(pid string) error { - pid = strings.TrimSpace(pid) - if pid == "" || pid == "-" { - return fmt.Errorf("invalid PID %q", pid) - } - - for _, r := range pid { - if r < '0' || r > '9' { - return fmt.Errorf("invalid PID %q", pid) - } - } - - cmd := exec.Command("somo", "-k", "-p", pid) - output, err := cmd.CombinedOutput() - if err != nil { - return commandError(fmt.Sprintf("failed to kill connection with PID %s", pid), err, output) - } - - return nil -} - func commandError(message string, err error, output []byte) error { trimmed := strings.TrimSpace(string(output)) if trimmed == "" { From c00e181feb0435139976a7e6b97ecab154dc39b3 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 3 Aug 2026 17:33:18 +0600 Subject: [PATCH 06/56] update (monitor): RunScan improved with IP evaluation --- internal/monitor/monitor.go | 97 ++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index c32d42b..a1e77ad 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -12,6 +12,7 @@ import ( "github.com/hexbytedev/hexwall/internal/allowlist" "github.com/hexbytedev/hexwall/internal/deghost" + "github.com/hexbytedev/hexwall/internal/enforcer" "github.com/hexbytedev/hexwall/internal/pihole" "github.com/hexbytedev/hexwall/internal/somo" "github.com/hexbytedev/hexwall/internal/store" @@ -28,6 +29,7 @@ const ( // scanSummary tracks per-scan outcome counts for the summary line. type scanSummary struct { connections int + allowlisted int withSNI int withoutSNI int trusted int @@ -117,7 +119,7 @@ func HandleZeekEvent(checker *pihole.Checker, hexwallStore *store.Store, mode st // 7. IP-only fallback → deghost escalation if enabled; no action if disabled // // Safety property: with deghost disabled, only rung 4 (Pi-hole blocklist match) can kill. -func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.Store, deghostClient *deghost.Client, zeekClient *zeek.Client, mode string, debug bool) { +func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.Store, deghostClient *deghost.Client, deghostIP bool, deghostDomain bool, zeekClient *zeek.Client, mode string, debug bool) { if hexwallStore == nil { slog.Error("scan aborted: nil hexwall store") return @@ -154,6 +156,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S // Rung 1: static CIDR allowlist. if allowlist.Contains(ip) { + summary.allowlisted++ summary.trusted++ continue } @@ -175,7 +178,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S if !sniFound { // Rung 7: IP-only fallback — no SNI means non-TLS, pre-existing connection, or ECH. - action := evaluateIPOnly(ctx, hexwallStore, deghostClient, checker, ipStr, conn, selectedMode, debug) + action := evaluateIPOnly(ctx, hexwallStore, deghostClient, deghostIP, checker, ipStr, conn, selectedMode, debug) applyOutcome(&summary, action) continue } @@ -183,10 +186,15 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S domain := strings.ToLower(strings.TrimSpace(sniDomain)) // Rung 3: Pi-hole explicit allowlist — user intent outranks everything inferred. - if allowed, err := checker.IsAllowedByPolicy(domain); err != nil { - slog.Debug("domain allowlist lookup failed", "domain", domain, "err", err) + allowed, err := checker.IsAllowedByPolicy(domain) + if err != nil { + // Fail-open: if we cannot determine whether a domain is explicitly allowed, + // we must not act on a block from rung 4. Rung 4 is the only rung that can + // kill a connection, and killing a legitimate connection on the basis of + // incomplete policy data is worse than missing one block. + slog.Warn("domain allowlist lookup failed, skipping blocklist check", "domain", domain, "err", err) } else if allowed { - if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, true); logErr != nil { + if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyAllowed); logErr != nil { slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } summary.trusted++ @@ -194,16 +202,18 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S } // Rung 4: Pi-hole explicit denylist / gravity — only rung that can kill. - if blocked, err := checker.IsBlockedByPolicy(domain); err != nil { - slog.Debug("domain denylist lookup failed", "domain", domain, "err", err) - } else if blocked { - if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, false); logErr != nil { - slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) + if err == nil { + if blocked, err := checker.IsBlockedByPolicy(domain); err != nil { + slog.Warn("domain denylist lookup failed", "domain", domain, "err", err) + } else if blocked { + if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyBlocked); logErr != nil { + slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) + } + slog.Warn("policy-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + applyBlockedOutcome(hexwallStore, selectedMode, ipStr, conn) + summary.blocked++ + continue } - slog.Warn("policy-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - applyBlockedOutcome(hexwallStore, selectedMode, ipStr, conn) - summary.blocked++ - continue } // Rung 5: inferred trust from DNS history — scoped to this IP. @@ -219,7 +229,11 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S } } } - if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, known); logErr != nil { + outcome := store.OutcomeUnknown + if known { + outcome = store.OutcomeIPDomainMatch + } + if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, outcome); logErr != nil { slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } @@ -229,7 +243,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S } // Rung 6: unknown domain — deghost escalation if enabled. - if deghostClient == nil { + if !deghostDomain { if debug { slog.Info("unknown domain, deghost disabled; no action", "domain", domain, "ip", ipStr, "program", conn.Program) } @@ -276,16 +290,20 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S summary.blocked++ } - fmt.Printf("[%s] Scan complete: %d connections, %d trusted, %d blocked, %d unknown (sni=%d no-sni=%d errored=%d)\n", - time.Now().Format("15:04:05"), summary.connections, summary.trusted, summary.blocked, summary.unknown, summary.withSNI, summary.withoutSNI, summary.errored) + fmt.Printf("[%s] Scan complete: %d connections, %d trusted, %d blocked, %d unknown (allowlisted=%d sni=%d no-sni=%d errored=%d)\n", + time.Now().Format("15:04:05"), summary.connections, summary.trusted, summary.blocked, summary.unknown, summary.allowlisted, summary.withSNI, summary.withoutSNI, summary.errored) } // evaluateIPOnly handles the IP-only fallback path (rung 7) when no SNI is available. // It returns "trusted", "blocked", "unknown", or "error". -func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClient *deghost.Client, checker *pihole.Checker, ipStr string, conn somo.Connection, mode string, debug bool) string { +// Every exit path records an IP observation before returning. +func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClient *deghost.Client, deghostIP bool, checker *pihole.Checker, ipStr string, conn somo.Connection, mode string, debug bool) string { allowed, err := hexwallStore.IsAllowed(ipStr) if err != nil { slog.Error("store lookup failed", "address", conn.RAddress, "err", err) + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } return "error" } @@ -293,19 +311,28 @@ func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClien if err := hexwallStore.UpdateEstablished(ipStr); err != nil { slog.Error("failed to update established", "address", conn.RAddress, "err", err) } + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPTrusted); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } return "trusted" } - if deghostClient == nil { + if !deghostIP { if debug { slog.Info("unknown IP, deghost disabled; no action", "ip", ipStr, "program", conn.Program) } + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } return "unknown" } cachedFraudCheck, err := hexwallStore.GetRecentFraudCheck(ipStr) if err != nil { slog.Error("fraud cache lookup failed", "ip", ipStr, "program", conn.Program, "err", err) + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } return "error" } @@ -314,10 +341,16 @@ func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClien if debug { slog.Info("unknown but clean IP (cached)", "ip", ipStr, "program", conn.Program) } + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } return "unknown" } slog.Warn("deghost-blocked IP (cached)", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } applyBlockedOutcome(hexwallStore, mode, ipStr, conn) return "blocked" } @@ -325,6 +358,9 @@ func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClien report, err := deghostClient.CheckIP(ctx, ipStr) if err != nil { slog.Error("deghost check failed", "ip", ipStr, "program", conn.Program, "err", err) + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } return "error" } @@ -335,6 +371,9 @@ func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClien if debug { slog.Info("unknown but clean IP (private/reserved)", "ip", ipStr, "program", conn.Program) } + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPReserved); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } return "unknown" } @@ -347,10 +386,16 @@ func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClien if debug { slog.Info("unknown but clean IP", "ip", ipStr, "program", conn.Program) } + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } return "unknown" } slog.Warn("deghost-blocked IP", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } applyBlockedOutcome(hexwallStore, mode, ipStr, conn) return "blocked" } @@ -366,8 +411,16 @@ func applyBlockedOutcome(hexwallStore *store.Store, mode, ipStr string, conn som if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { slog.Error("failed to log kill", "address", conn.RAddress, "err", err) } - if err := somo.KillConnection(conn.PID); err != nil { - slog.Error("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) + + // Connection-level termination via `ss -K` (requires CONFIG_INET_DIAG_DESTROY) + // is deferred work. KillProcess terminates the owning process, which is coarser. + pidInt, err := enforcer.ParsePID(conn.PID) + if err != nil { + slog.Warn("invalid PID, skipping kill", "pid", conn.PID, "address", conn.RAddress, "err", err) + return + } + if err := enforcer.KillProcess(pidInt, conn.Program, conn.RAddress); err != nil { + slog.Warn("failed to kill process", "address", conn.RAddress, "pid", conn.PID, "err", err) } else { slog.Info("killed connection", "address", conn.RAddress) } From 89274cbca4f7e2ea04e9804c8e00341b391d0c1c Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 3 Aug 2026 17:34:07 +0600 Subject: [PATCH 07/56] update (main): Prune database for old records --- main.go | 89 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index d2c9df9..584e9d7 100644 --- a/main.go +++ b/main.go @@ -35,15 +35,16 @@ var ( ) type runConfig struct { - dbPath string - hexwallDBPath string - zeekLogPath string - zeekNoticeLog string - enableZeek bool - enableDeghost bool - mode string - debug bool - showVersion bool + dbPath string + hexwallDBPath string + zeekLogPath string + zeekNoticeLog string + enableZeek bool + enableDeghostIP bool + enableDeghostDomain bool + mode string + debug bool + showVersion bool } func main() { @@ -56,24 +57,24 @@ func parseFlags() (*runConfig, error) { zeekLogPath := flag.String("zeek-log", "", "path to zeek ssl.log for SNI-based domain verification (empty to skip)") zeekNoticeLog := flag.String("zeek-notice-log", "/opt/zeek/logs/current/notice.log", "path to Zeek notice.log for DNS-bypass alerts") enableZeek := flag.Bool("enable-zeek", false, "enable Zeek-based DNS-bypass detection") - // deghost sends every unrecognized domain the machine contacts to a third-party API. - // That is a real privacy cost and should be opt-in, not default. - enableDeghost := flag.Bool("enable-deghost", false, "enable third-party IP/domain reputation checks (privacy-sensitive; sends queried domains externally)") + enableDeghostIP := flag.Bool("enable-deghost-ip", true, "enable third-party IP reputation checks on the IP-only fallback path") + enableDeghostDomain := flag.Bool("enable-deghost-domain", true, "enable third-party domain reputation checks at rung 6") mode := flag.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") debug := flag.Bool("debug", false, "enable debug-level logging, including verbose per-connection scan logging") showVersion := flag.Bool("version", false, "print version and exit") flag.Parse() cfg := &runConfig{ - dbPath: strings.TrimSpace(*dbPath), - hexwallDBPath: strings.TrimSpace(*hexwallDB), - zeekLogPath: strings.TrimSpace(*zeekLogPath), - zeekNoticeLog: strings.TrimSpace(*zeekNoticeLog), - enableZeek: *enableZeek, - enableDeghost: *enableDeghost, - mode: strings.TrimSpace(*mode), - debug: *debug, - showVersion: *showVersion, + dbPath: strings.TrimSpace(*dbPath), + hexwallDBPath: strings.TrimSpace(*hexwallDB), + zeekLogPath: strings.TrimSpace(*zeekLogPath), + zeekNoticeLog: strings.TrimSpace(*zeekNoticeLog), + enableZeek: *enableZeek, + enableDeghostIP: *enableDeghostIP, + enableDeghostDomain: *enableDeghostDomain, + mode: strings.TrimSpace(*mode), + debug: *debug, + showVersion: *showVersion, } if cfg.showVersion { @@ -179,6 +180,18 @@ func run() int { slog.Info("hexwall database ready", "path", cfg.hexwallDBPath) + // 4a. Prune stale rows from prior runs at startup. + if result, err := hexwallStore.Prune(); err != nil { + slog.Warn("initial prune failed", "err", err) + } else if result != nil && (result.AllowedIPDomains+result.SNIObservations+result.IPObservations+result.KilledConnections+result.ZeekAlerts) > 0 { + slog.Info("pruned stale rows", + "allowed_ip_domains", result.AllowedIPDomains, + "sni_observations", result.SNIObservations, + "ip_observations", result.IPObservations, + "killed_connections", result.KilledConnections, + "zeek_alerts", result.ZeekAlerts) + } + // 4b. Start the Zeek ssl.log tailer when a path is provided. var zeekClient *zeek.Client if cfg.zeekLogPath != "" { @@ -215,14 +228,18 @@ func run() int { } // deghost is opt-in: it sends every unrecognized domain to a third-party API, - // which is a real privacy cost. Only construct the client when explicitly enabled. + // which is a real privacy cost. Only construct the client when at least one check is enabled. var deghostClient *deghost.Client - if cfg.enableDeghost { + if cfg.enableDeghostIP || cfg.enableDeghostDomain { deghostClient = deghost.NewClient(deghostBaseURL, deghostTimeout) } else { slog.Info("deghost disabled; unrecognized connections will not be escalated to third-party API") } + if cfg.mode == monitor.ModeEnforce && cfg.enableDeghostIP { + slog.Warn("enforce mode with IP deghost enabled: an external reputation verdict can now terminate a local process; shared cloud provider ranges are known to produce false positives on such feeds") + } + // 5. Refresh trusted IPs before starting the monitor so the first tick does not kill legitimate connections. cache := pihole.NewIPCache(checker, hexwallStore) cache.Refresh(ctx) @@ -242,6 +259,30 @@ func run() int { } }() + // 6b. Periodically prune stale rows from the hexwall database. + go func() { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if result, err := hexwallStore.Prune(); err != nil { + slog.Warn("periodic prune failed", "err", err) + } else if result != nil && (result.AllowedIPDomains+result.SNIObservations+result.IPObservations+result.KilledConnections+result.ZeekAlerts) > 0 { + slog.Info("pruned stale rows", + "allowed_ip_domains", result.AllowedIPDomains, + "sni_observations", result.SNIObservations, + "ip_observations", result.IPObservations, + "killed_connections", result.KilledConnections, + "zeek_alerts", result.ZeekAlerts) + } + } + } + }() + fmt.Println("Starting network monitor...") fmt.Printf("> Connections will be checked every %s in %s mode.\n", connectionScanInterval, cfg.mode) if cfg.debug { @@ -258,7 +299,7 @@ func run() int { slog.Info("shutting down") return 0 case <-ticker.C: - monitor.RunScan(ctx, checker, hexwallStore, deghostClient, zeekClient, cfg.mode, cfg.debug) + monitor.RunScan(ctx, checker, hexwallStore, deghostClient, cfg.enableDeghostIP, cfg.enableDeghostDomain, zeekClient, cfg.mode, cfg.debug) } } } From 86ee30394c307da4e92d01d922841bce627cf9b0 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 3 Aug 2026 18:22:39 +0600 Subject: [PATCH 08/56] chore: ignore local design notes --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index a53ed68..f46064e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,12 @@ coverage.txt .build pihole-guard.db* inventory.ndjson + +# local design notes and session artifacts — not for publication +CHAT_SUMMARY_*.md +CLAUDE.md +OVERVIEW.md +SETUP.md +SNI_DETECTION_DESIGN.md +zeek-integration-guide.md +IMPLEMENTATION_NOTES.md From a78953de7294d7b3719690f61e355f2a8ae443b9 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 4 Aug 2026 13:57:59 +0600 Subject: [PATCH 09/56] fix: golangci-lint errors --- internal/deghost/deghost_test.go | 16 ++++--- internal/monitor/monitor.go | 4 +- internal/pihole/pihole.go | 9 ++-- internal/pihole/policy_test.go | 78 ++++++++++++++++++++++++++------ internal/store/store.go | 24 +++++++--- internal/zeek/zeek.go | 6 ++- 6 files changed, 103 insertions(+), 34 deletions(-) diff --git a/internal/deghost/deghost_test.go b/internal/deghost/deghost_test.go index 8529a3c..7f2153f 100644 --- a/internal/deghost/deghost_test.go +++ b/internal/deghost/deghost_test.go @@ -68,7 +68,7 @@ func TestCheckDomainStatusForbiddenDecodesBody(t *testing.T) { DomainAgeInDays: 0, } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusForbidden) _ = json.NewEncoder(w).Encode(want) })) @@ -97,7 +97,7 @@ func TestCheckDomainStatusBadRequestDecodesBody(t *testing.T) { DomainAgeInDays: 0, } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadRequest) _ = json.NewEncoder(w).Encode(want) })) @@ -119,7 +119,7 @@ func TestCheckDomainStatusBadRequestDecodesBody(t *testing.T) { func TestCheckDomainStatusInternalServerError(t *testing.T) { t.Parallel() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) defer srv.Close() @@ -134,7 +134,7 @@ func TestCheckDomainStatusInternalServerError(t *testing.T) { func TestCheckDomainStatusGatewayTimeout(t *testing.T) { t.Parallel() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusGatewayTimeout) })) defer srv.Close() @@ -149,7 +149,7 @@ func TestCheckDomainStatusGatewayTimeout(t *testing.T) { func TestCheckDomainStatusClientClosedRequest(t *testing.T) { t.Parallel() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(499) })) defer srv.Close() @@ -164,9 +164,11 @@ func TestCheckDomainStatusClientClosedRequest(t *testing.T) { func TestCheckDomainMalformedJSON(t *testing.T) { t.Parallel() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") - fmt.Fprint(w, `{"status": "allowed" bad json`) + if _, err := fmt.Fprint(w, `{"status": "allowed" bad json`); err != nil { + t.Errorf("fmt.Fprint: %v", err) + } })) defer srv.Close() diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index a1e77ad..daa7ecd 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -178,7 +178,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S if !sniFound { // Rung 7: IP-only fallback — no SNI means non-TLS, pre-existing connection, or ECH. - action := evaluateIPOnly(ctx, hexwallStore, deghostClient, deghostIP, checker, ipStr, conn, selectedMode, debug) + action := evaluateIPOnly(ctx, hexwallStore, deghostClient, deghostIP, ipStr, conn, selectedMode, debug) applyOutcome(&summary, action) continue } @@ -297,7 +297,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S // evaluateIPOnly handles the IP-only fallback path (rung 7) when no SNI is available. // It returns "trusted", "blocked", "unknown", or "error". // Every exit path records an IP observation before returning. -func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClient *deghost.Client, deghostIP bool, checker *pihole.Checker, ipStr string, conn somo.Connection, mode string, debug bool) string { +func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClient *deghost.Client, deghostIP bool, ipStr string, conn somo.Connection, mode string, debug bool) string { allowed, err := hexwallStore.IsAllowed(ipStr) if err != nil { slog.Error("store lookup failed", "address", conn.RAddress, "err", err) diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index 043c897..d5fd0c2 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -202,13 +202,12 @@ func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { WHERE domain = ? COLLATE NOCASE LIMIT 1 `, domain).Scan(&found) - if errors.Is(err, sql.ErrNoRows) { - // Not in denylist; check gravity. - } else if err != nil { - return false, fmt.Errorf("denylist lookup failed: %w", err) - } else { + if err == nil { return true, nil } + if !errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf("denylist lookup failed: %w", err) + } err = c.gravityDB.QueryRow(` SELECT 1 FROM vw_gravity diff --git a/internal/pihole/policy_test.go b/internal/pihole/policy_test.go index 2dffbef..fb722b9 100644 --- a/internal/pihole/policy_test.go +++ b/internal/pihole/policy_test.go @@ -98,7 +98,11 @@ func TestIsBlockedByPolicy_Denylist(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() blocked, err := checker.IsBlockedByPolicy("blocked.example.com") if err != nil { @@ -116,7 +120,11 @@ func TestIsBlockedByPolicy_Gravity(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() blocked, err := checker.IsBlockedByPolicy("malware.example.net") if err != nil { @@ -134,7 +142,11 @@ func TestIsBlockedByPolicy_NoMatch(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() blocked, err := checker.IsBlockedByPolicy("safe.example.org") if err != nil { @@ -152,7 +164,11 @@ func TestIsAllowedByPolicy_Allowlist(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() allowed, err := checker.IsAllowedByPolicy("trusted.example.com") if err != nil { @@ -170,7 +186,11 @@ func TestIsAllowedByPolicy_NoMatch(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() allowed, err := checker.IsAllowedByPolicy("unknown.example.org") if err != nil { @@ -190,7 +210,11 @@ func TestPolicyLookup_CaseNormalization(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() // Denylist: case-insensitive match blocked, err := checker.IsBlockedByPolicy(" casesensitive.com ") @@ -227,7 +251,11 @@ func TestPolicyLookup_EmptyDomain(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() blocked, err := checker.IsBlockedByPolicy("") if err != nil { @@ -287,7 +315,11 @@ func TestPolicyCounts(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() counts, err := checker.PolicyCounts() if err != nil { @@ -358,7 +390,11 @@ func TestGravityDBPath_Derived(t *testing.T) { if err != nil { t.Fatalf("NewChecker() error = %v", err) } - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() if checker.gravityDB == nil { t.Fatal("gravityDB is nil, want it to be auto-discovered") @@ -385,7 +421,11 @@ func TestGravityDBPath_Missing(t *testing.T) { if err != nil { t.Fatalf("NewChecker() error = %v, want nil when gravity.db is missing", err) } - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() if checker.gravityDB != nil { t.Fatal("gravityDB is non-nil, want nil when gravity.db is missing") @@ -429,7 +469,11 @@ func TestIsBlockedByPolicy_DenylistBeatsGravity(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() blocked, err := checker.IsBlockedByPolicy("both.example.com") if err != nil { @@ -448,7 +492,11 @@ func TestIsAllowedByPolicy_TakesPrecedenceOverGravity(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() // Allowlist match should be independent of gravity presence. allowed, err := checker.IsAllowedByPolicy("special.example.com") @@ -467,7 +515,11 @@ func TestIsBlockedByPolicy_WhitespaceNormalization(t *testing.T) { _ = gravDB.Close() checker := newTestChecker(t, gravPath) - defer checker.Close() + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() blocked, err := checker.IsBlockedByPolicy(" Trim.Example.COM ") if err != nil { diff --git a/internal/store/store.go b/internal/store/store.go index 6ee62b2..658777a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -124,10 +124,14 @@ type DomainCheckCacheEntry struct { type SNIOutcome string const ( + // OutcomePolicyAllowed indicates the connection was allowed by Pi-hole policy. OutcomePolicyAllowed SNIOutcome = "policy-allowed" + // OutcomePolicyBlocked indicates the connection was blocked by Pi-hole policy. OutcomePolicyBlocked SNIOutcome = "policy-blocked" + // OutcomeIPDomainMatch indicates the SNI domain matched a known domain for the IP. OutcomeIPDomainMatch SNIOutcome = "ip-domain-match" - OutcomeUnknown SNIOutcome = "unknown" + // OutcomeUnknown indicates the connection could not be classified. + OutcomeUnknown SNIOutcome = "unknown" ) // Valid reports whether the outcome is one of the known values. @@ -153,10 +157,14 @@ func ParseSNIOutcome(s string) (SNIOutcome, bool) { type IPOutcome string const ( - OutcomeIPTrusted IPOutcome = "ip-trusted" // present and fresh in allowed_ips - OutcomeIPReserved IPOutcome = "private-reserved" // deghost returned 403: private or reserved address - OutcomeIPBlocked IPOutcome = "deghost-blocked" // deghost verdict said kill - OutcomeIPUnknown IPOutcome = "unknown" // no verdict available or check disabled + // OutcomeIPTrusted indicates the IP is present and fresh in allowed_ips. + OutcomeIPTrusted IPOutcome = "ip-trusted" + // OutcomeIPReserved indicates deghost returned 403: private or reserved address. + OutcomeIPReserved IPOutcome = "private-reserved" + // OutcomeIPBlocked indicates deghost verdict said kill. + OutcomeIPBlocked IPOutcome = "deghost-blocked" + // OutcomeIPUnknown indicates no verdict available or check disabled. + OutcomeIPUnknown IPOutcome = "unknown" ) // Valid reports whether the outcome is one of the known values. @@ -206,7 +214,11 @@ func migrateSNITable(db *sql.DB) error { if err != nil { return fmt.Errorf("query table info: %w", err) } - defer cols.Close() + defer func() { + if err := cols.Close(); err != nil { + slog.Debug("failed to close columns", "err", err) + } + }() hasOldSchema := false for cols.Next() { diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go index 01cd31e..3a9cdc0 100644 --- a/internal/zeek/zeek.go +++ b/internal/zeek/zeek.go @@ -254,7 +254,11 @@ func (c *Client) processNewLines() { slog.Debug("zeek: cannot open ssl.log", "path", c.logPath, "err", err) return } - defer file.Close() + defer func() { + if err := file.Close(); err != nil { + slog.Debug("zeek: failed to close ssl.log", "path", c.logPath, "err", err) + } + }() fi, err := file.Stat() if err != nil { From 9728e0ba63ae07a1dfd1851d48ea595da4608163 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 20 Sep 2026 17:06:43 +0600 Subject: [PATCH 10/56] test: add coverage for somo, monitor, and pihole cache internal/somo and internal/monitor had zero tests despite monitor.RunScan being the core kill-decision path. Adds unit tests for both, plus for pihole's DNS-history cache refresh. cache.go gains injectable domainsSeenSince/lookupHost fields on IPCache so the refresh logic can be tested without a real Pi-hole DB or live DNS resolution. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018yCDQPH4oP4xQcGZUKMqaH --- internal/monitor/monitor_test.go | 560 +++++++++++++++++++++++++++++++ internal/pihole/cache.go | 23 +- internal/pihole/cache_test.go | 59 ++++ internal/somo/somo_test.go | 163 +++++++++ 4 files changed, 797 insertions(+), 8 deletions(-) create mode 100644 internal/monitor/monitor_test.go create mode 100644 internal/pihole/cache_test.go create mode 100644 internal/somo/somo_test.go diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go new file mode 100644 index 0000000..6c33328 --- /dev/null +++ b/internal/monitor/monitor_test.go @@ -0,0 +1,560 @@ +package monitor + +import ( + "context" + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hexbytedev/hexwall/internal/deghost" + "github.com/hexbytedev/hexwall/internal/pihole" + "github.com/hexbytedev/hexwall/internal/somo" + "github.com/hexbytedev/hexwall/internal/store" + "github.com/hexbytedev/hexwall/internal/zeek" + + _ "modernc.org/sqlite" +) + +// ---- fixtures -------------------------------------------------------- + +// newTestStore creates a real *store.Store backed by a temp SQLite file and +// returns it along with its path, so tests can also open a second read-only +// connection to inspect tables the Store package doesn't expose a reader for +// (e.g. killed_connections, allowed_ips.last_established). +func newTestStore(t *testing.T) (*store.Store, string) { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + s, err := store.NewStore(dbPath) + if err != nil { + t.Fatalf("store.NewStore() error = %v", err) + } + t.Cleanup(func() { + if err := s.Close(); err != nil { + t.Errorf("Store.Close() error = %v", err) + } + }) + + return s, dbPath +} + +// queryInt opens its own read-only connection to the sqlite file at dbPath and +// scans a single integer column. Used to inspect tables that *store.Store +// doesn't expose an exported reader for. +func queryInt(t *testing.T, dbPath, query string, args ...any) int { + t.Helper() + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: dbPath, RawQuery: "mode=ro"}).String()) + if err != nil { + t.Fatalf("open %s for inspection: %v", dbPath, err) + } + defer func() { + _ = db.Close() + }() + + var got int + if err := db.QueryRow(query, args...).Scan(&got); err != nil { + t.Fatalf("query %q: %v", query, err) + } + return got +} + +// openTestGravityDB creates a temp gravity.db with the Pi-hole v6 views monitor's +// checker relies on. Mirrors internal/pihole/policy_test.go's helper of the same +// name, duplicated here because it is unexported in its own package. +func openTestGravityDB(t *testing.T) (*sql.DB, string) { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "gravity.db") + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: dbPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open test gravity db: %v", err) + } + + schema := ` + CREATE TABLE domainlist (id INTEGER PRIMARY KEY, type INTEGER, domain TEXT, enabled INTEGER, date_added INTEGER, date_modified INTEGER, comment TEXT, groups TEXT); + CREATE TABLE gravity (id INTEGER PRIMARY KEY, domain TEXT, adlist_id INTEGER); + + CREATE VIEW vw_allowlist AS SELECT domain FROM domainlist WHERE type = 0 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_denylist AS SELECT domain FROM domainlist WHERE type = 1 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_gravity AS SELECT domain FROM gravity; + CREATE VIEW vw_regex_allowlist AS SELECT domain FROM domainlist WHERE type = 2 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_regex_denylist AS SELECT domain FROM domainlist WHERE type = 3 AND enabled = 1 AND TRIM(domain) <> ''; + ` + if _, err := db.Exec(schema); err != nil { + _ = db.Close() + t.Fatalf("create gravity schema: %v", err) + } + + return db, dbPath +} + +// insertDomain inserts a domain into domainlist. type 0 = allowlist, 1 = denylist. +func insertDomain(t *testing.T, db *sql.DB, domain string, listType int) { + t.Helper() + if _, err := db.Exec(`INSERT INTO domainlist (type, domain, enabled) VALUES (?, ?, 1)`, listType, domain); err != nil { + t.Fatalf("insert domain %q type %d: %v", domain, listType, err) + } +} + +// newTestChecker builds a real *pihole.Checker backed by temp SQLite files: +// an empty FTL query-history db and a gravity db seeded via gravitySetup. +func newTestChecker(t *testing.T, gravitySetup func(db *sql.DB)) *pihole.Checker { + t.Helper() + + ftlPath := filepath.Join(t.TempDir(), "pihole-FTL.db") + ftlDB, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("create ftl db: %v", err) + } + if _, err := ftlDB.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY, timestamp INTEGER, type INTEGER, domain TEXT, status INTEGER, ip TEXT)`); err != nil { + t.Fatalf("create ftl schema: %v", err) + } + if err := ftlDB.Close(); err != nil { + t.Fatalf("close ftl db: %v", err) + } + + gravDB, gravPath := openTestGravityDB(t) + if gravitySetup != nil { + gravitySetup(gravDB) + } + if err := gravDB.Close(); err != nil { + t.Fatalf("close gravity db: %v", err) + } + + checker, err := pihole.NewChecker(&pihole.Config{DBPath: ftlPath, GravityDBPath: gravPath}) + if err != nil { + t.Fatalf("pihole.NewChecker() error = %v", err) + } + t.Cleanup(func() { + if err := checker.Close(); err != nil { + t.Errorf("Checker.Close() error = %v", err) + } + }) + + return checker +} + +// installFakeSomo writes an executable shell script named "somo" into a temp +// directory and prepends it to PATH, mirroring internal/somo/somo_test.go's helper. +func installFakeSomo(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + path := filepath.Join(dir, "somo") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake somo: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +// failIfCalled returns an httptest.Server that fails the test if it receives any request. +// Used to assert a code path never makes a live deghost call. +func failIfCalled(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected HTTP call to deghost: %s", r.URL.Path) + })) + t.Cleanup(srv.Close) + return srv +} + +// ---- normalizeMode ----------------------------------------------------- + +func TestNormalizeMode(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + mode string + want string + }{ + {"enforce lowercase", "enforce", ModeEnforce}, + {"enforce uppercase", "ENFORCE", ModeEnforce}, + {"enforce padded mixed case", " Enforce ", ModeEnforce}, + {"watch", "watch", ModeWatch}, + {"empty", "", ModeWatch}, + {"garbage", "banana", ModeWatch}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := normalizeMode(tt.mode); got != tt.want { + t.Fatalf("normalizeMode(%q) = %q, want %q", tt.mode, got, tt.want) + } + }) + } +} + +// ---- remoteIP ------------------------------------------------------------ + +func TestRemoteIP(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + address string + wantIP string + wantErr bool + }{ + {"ipv4 with port", "203.0.113.10:443", "203.0.113.10", false}, + {"bare ipv4", "203.0.113.10", "203.0.113.10", false}, + {"bracketed ipv6 with port", "[::1]:443", "::1", false}, + {"bare ipv6", "::1", "::1", false}, + {"invalid host with port", "not-an-ip:443", "", true}, + {"empty", "", "", true}, + {"malformed", "not-an-ip", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ip, err := remoteIP(tt.address) + if tt.wantErr { + if err == nil { + t.Fatalf("remoteIP(%q) error = nil, want error", tt.address) + } + return + } + if err != nil { + t.Fatalf("remoteIP(%q) error = %v, want nil", tt.address, err) + } + if ip.String() != tt.wantIP { + t.Fatalf("remoteIP(%q) = %q, want %q", tt.address, ip.String(), tt.wantIP) + } + }) + } +} + +// ---- applyOutcome -------------------------------------------------------- + +func TestApplyOutcome(t *testing.T) { + t.Parallel() + + tests := []struct { + outcome string + get func(s scanSummary) int + }{ + {"trusted", func(s scanSummary) int { return s.trusted }}, + {"blocked", func(s scanSummary) int { return s.blocked }}, + {"unknown", func(s scanSummary) int { return s.unknown }}, + {"error", func(s scanSummary) int { return s.errored }}, + } + + for _, tt := range tests { + t.Run(tt.outcome, func(t *testing.T) { + t.Parallel() + var summary scanSummary + applyOutcome(&summary, tt.outcome) + if got := tt.get(summary); got != 1 { + t.Fatalf("applyOutcome(%q) counter = %d, want 1", tt.outcome, got) + } + }) + } + + t.Run("unrecognized outcome increments nothing", func(t *testing.T) { + t.Parallel() + var want scanSummary + got := want + applyOutcome(&got, "garbage") + if got != want { + t.Fatalf("applyOutcome(garbage) summary = %+v, want unchanged %+v", got, want) + } + }) +} + +// ---- HandleZeekEvent ------------------------------------------------------ + +func TestHandleZeekEventBlockedDomainPersisted(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "evil.example.com", 1) // denylist + }) + hexwallStore, _ := newTestStore(t) + + event := zeek.Event{SrcIP: "10.0.0.5", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"} + HandleZeekEvent(checker, hexwallStore, ModeWatch, event) + + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil { + t.Fatalf("RecentZeekAlert() error = %v", err) + } + if alert == nil { + t.Fatal("RecentZeekAlert() = nil, want persisted alert") + } + if !alert.Blocked { + t.Fatal("alert.Blocked = false, want true for denylisted domain") + } +} + +func TestHandleZeekEventUnblockedDomainPersisted(t *testing.T) { + checker := newTestChecker(t, nil) + hexwallStore, _ := newTestStore(t) + + event := zeek.Event{SrcIP: "10.0.0.5", DstIP: "203.0.113.20", DstPort: "443", SNI: "clean.example.com"} + HandleZeekEvent(checker, hexwallStore, ModeWatch, event) + + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil { + t.Fatalf("RecentZeekAlert() error = %v", err) + } + if alert == nil { + t.Fatal("RecentZeekAlert() = nil, want persisted alert") + } + if alert.Blocked { + t.Fatal("alert.Blocked = true, want false for domain not on denylist") + } +} + +func TestHandleZeekEventNoopCases(t *testing.T) { + t.Parallel() + checker := newTestChecker(t, nil) + hexwallStore, _ := newTestStore(t) + + // None of these should panic or write anything. + HandleZeekEvent(nil, hexwallStore, ModeWatch, zeek.Event{SNI: "example.com"}) + HandleZeekEvent(checker, nil, ModeWatch, zeek.Event{SNI: "example.com"}) + HandleZeekEvent(checker, hexwallStore, ModeWatch, zeek.Event{SrcIP: "10.0.0.1", SNI: ""}) +} + +// ---- applyBlockedOutcome --------------------------------------------------- + +func TestApplyBlockedOutcomeWatchModeNeverLogsKill(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + + conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} + applyBlockedOutcome(hexwallStore, ModeWatch, "203.0.113.10", conn) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0 in watch mode", got) + } +} + +func TestApplyBlockedOutcomeEnforceModeLogsKillBeforeAttemptingIt(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + + // PID 1 (init) is guaranteed to be refused by enforcer.KillProcess, but the + // audit row must still be written first ("audit survives the kill"). + conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} + applyBlockedOutcome(hexwallStore, ModeEnforce, "203.0.113.10", conn) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.10"); got != 1 { + t.Fatalf("killed_connections count = %d, want 1 even though the kill itself was refused", got) + } +} + +func TestApplyBlockedOutcomeInvalidPIDLogsWarningAndReturns(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + + conn := somo.Connection{PID: "not-a-pid", Program: "curl", RAddress: "203.0.113.11:443"} + applyBlockedOutcome(hexwallStore, ModeEnforce, "203.0.113.11", conn) + + // LogKill happens before ParsePID, so the audit row is still written. + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.11"); got != 1 { + t.Fatalf("killed_connections count = %d, want 1 (audit before parse failure)", got) + } +} + +// ---- evaluateIPOnly -------------------------------------------------------- + +func TestEvaluateIPOnlyTrustedIPUpdatesEstablished(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + if err := hexwallStore.UpsertAllowedIP("203.0.113.20", "example.com"); err != nil { + t.Fatalf("UpsertAllowedIP() error = %v", err) + } + + srv := failIfCalled(t) + client := deghost.NewClient(srv.URL, time.Second) + + conn := somo.Connection{Program: "curl", RAddress: "203.0.113.20:443"} + got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.20", conn, ModeWatch, false) + if got != "trusted" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "trusted") + } + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM allowed_ips WHERE ip = ? AND last_established IS NOT NULL`, "203.0.113.20"); got != 1 { + t.Fatal("UpdateEstablished was not applied for a trusted IP") + } +} + +func TestEvaluateIPOnlyDeghostDisabledReturnsUnknown(t *testing.T) { + hexwallStore, _ := newTestStore(t) + srv := failIfCalled(t) + client := deghost.NewClient(srv.URL, time.Second) + + conn := somo.Connection{Program: "curl", RAddress: "203.0.113.21:443"} + got := evaluateIPOnly(context.Background(), hexwallStore, client, false, "203.0.113.21", conn, ModeWatch, false) + if got != "unknown" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") + } +} + +func TestEvaluateIPOnlyCachedCleanReturnsUnknownNoHTTPCall(t *testing.T) { + hexwallStore, _ := newTestStore(t) + if err := hexwallStore.UpsertFraudCheck("203.0.113.22", false); err != nil { + t.Fatalf("UpsertFraudCheck() error = %v", err) + } + + srv := failIfCalled(t) + client := deghost.NewClient(srv.URL, time.Second) + + conn := somo.Connection{Program: "curl", RAddress: "203.0.113.22:443"} + got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.22", conn, ModeWatch, false) + if got != "unknown" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") + } +} + +func TestEvaluateIPOnlyCachedBlockedReturnsBlockedAndKills(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + if err := hexwallStore.UpsertFraudCheck("203.0.113.23", true); err != nil { + t.Fatalf("UpsertFraudCheck() error = %v", err) + } + + srv := failIfCalled(t) + client := deghost.NewClient(srv.URL, time.Second) + + // PID 1 keeps the actual kill attempt safely refused by enforcer guardrails. + conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.23:443"} + got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.23", conn, ModeEnforce, false) + if got != "blocked" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "blocked") + } + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.23"); got != 1 { + t.Fatal("applyBlockedOutcome was not invoked for a cached blocked verdict") + } +} + +func TestEvaluateIPOnlyLiveCheckPrivateReservedCachesClean(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + client := deghost.NewClient(srv.URL, time.Second) + + hexwallStore, _ := newTestStore(t) + conn := somo.Connection{Program: "curl", RAddress: "203.0.113.24:443"} + got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.24", conn, ModeWatch, false) + if got != "unknown" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") + } + + entry, err := hexwallStore.GetRecentFraudCheck("203.0.113.24") + if err != nil { + t.Fatalf("GetRecentFraudCheck() error = %v", err) + } + if entry == nil { + t.Fatal("GetRecentFraudCheck() = nil, want a cached clean entry after private/reserved verdict") + } + if entry.ShouldKill { + t.Fatal("GetRecentFraudCheck().ShouldKill = true, want false for private/reserved verdict") + } +} + +func TestEvaluateIPOnlyLiveCheckShouldKillCachesAndBlocks(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(deghost.IPReport{ + IP: "203.0.113.25", + Security: deghost.SecurityReport{IsAbuser: true}, + }) + })) + defer srv.Close() + client := deghost.NewClient(srv.URL, time.Second) + + hexwallStore, dbPath := newTestStore(t) + conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.25:443"} + got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.25", conn, ModeEnforce, false) + if got != "blocked" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "blocked") + } + + entry, err := hexwallStore.GetRecentFraudCheck("203.0.113.25") + if err != nil { + t.Fatalf("GetRecentFraudCheck() error = %v", err) + } + if entry == nil || !entry.ShouldKill { + t.Fatalf("GetRecentFraudCheck() = %+v, want cached ShouldKill = true", entry) + } + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.25"); got != 1 { + t.Fatal("applyBlockedOutcome was not invoked for a live should-kill verdict") + } +} + +// NOTE: store.IsAllowed returning a hard error is not practically reachable with a +// real, healthy *store.Store — it only happens on a corrupted/closed DB handle, which +// would also break every other store call in the same scan. Not covered here; would +// require poking store internals from outside its package, which isn't a safe seam. + +// ---- RunScan --------------------------------------------------------------- + +func TestRunScanZeroConnections(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho '[]'\n") + hexwallStore, _ := newTestStore(t) + + // Must not panic even with nil checker/deghost/zeek dependencies. + RunScan(context.Background(), nil, hexwallStore, nil, false, false, nil, ModeWatch, false) +} + +func TestRunScanAllowlistedConnectionSkipsIPFallback(t *testing.T) { + installFakeSomo(t, `#!/bin/sh +echo '[{"proto":"tcp","local_port":"51820","remote_address":"127.0.0.1","remote_port":"443","program":"curl","pid":"1","state":"ESTABLISHED","address_type":"ipv4"}]' +`) + hexwallStore, dbPath := newTestStore(t) + + RunScan(context.Background(), nil, hexwallStore, nil, false, false, nil, ModeWatch, false) + + // Rung 1 (static allowlist) short-circuits before the IP-only fallback, so + // no ip_observations row should exist for a loopback address. + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, "127.0.0.1"); got != 0 { + t.Fatalf("ip_observations count for allowlisted ip = %d, want 0", got) + } +} + +func TestRunScanInvalidAddressDoesNotStopScan(t *testing.T) { + installFakeSomo(t, `#!/bin/sh +echo '[{"proto":"tcp","local_port":"1","remote_address":"not-an-ip","remote_port":"443","program":"curl","pid":"1","state":"ESTABLISHED","address_type":"ipv4"},{"proto":"tcp","local_port":"2","remote_address":"203.0.113.30","remote_port":"443","program":"curl","pid":"2","state":"ESTABLISHED","address_type":"ipv4"}]' +`) + hexwallStore, dbPath := newTestStore(t) + + RunScan(context.Background(), nil, hexwallStore, nil, false, false, nil, ModeWatch, false) + + // The second, valid connection must still be processed through the IP-only + // fallback despite the first connection's address being unparsable. + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, "203.0.113.30"); got != 1 { + t.Fatalf("ip_observations count for valid ip after invalid entry = %d, want 1", got) + } +} + +func TestRunScanUnknownIPRecordsIPObservation(t *testing.T) { + installFakeSomo(t, `#!/bin/sh +echo '[{"proto":"tcp","local_port":"1","remote_address":"203.0.113.31","remote_port":"443","program":"curl","pid":"1","state":"ESTABLISHED","address_type":"ipv4"}]' +`) + hexwallStore, dbPath := newTestStore(t) + + RunScan(context.Background(), nil, hexwallStore, nil, false, false, nil, ModeWatch, false) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ? AND outcome = 'unknown'`, "203.0.113.31"); got != 1 { + t.Fatalf("ip_observations unknown count = %d, want 1", got) + } +} + +// NOTE on rung 2/3/4 (Zeek SNI -> Pi-hole allow/deny) coverage inside RunScan: +// zeek.Client has no exported constructor or seam that lets an external package +// (monitor_test.go is package monitor, not package zeek) populate its SNI cache +// without going through the real background tailer, which only polls every 5s +// (zeek.tickInterval) with no synchronous first pass. Driving that path from here +// would mean sleeping 5+ seconds per test or poking zeek's unexported fields from +// outside its own package, neither of which fits the constraints for this task. +// Rungs 3/4 themselves (pihole.Checker.IsAllowedByPolicy / IsBlockedByPolicy) are +// exercised directly by internal/pihole's own tests, and HandleZeekEvent above +// exercises the same Checker.IsBlockedByPolicy call against a real gravity db. diff --git a/internal/pihole/cache.go b/internal/pihole/cache.go index 5979c4b..6e479af 100644 --- a/internal/pihole/cache.go +++ b/internal/pihole/cache.go @@ -4,7 +4,7 @@ import ( "context" "log/slog" "net" - "sort" + "slices" "sync" "time" @@ -22,13 +22,18 @@ const ( type IPCache struct { checker *Checker store *store.Store + + domainsSeenSince func(int64) ([]string, error) + lookupHost func(context.Context, string) ([]string, error) } // NewIPCache creates an IPCache backed by the given Checker and Store. func NewIPCache(checker *Checker, store *store.Store) *IPCache { return &IPCache{ - checker: checker, - store: store, + checker: checker, + store: store, + domainsSeenSince: checker.DomainsSeenSince, + lookupHost: net.DefaultResolver.LookupHost, } } @@ -41,7 +46,7 @@ func (c *IPCache) Refresh(ctx context.Context) { since := time.Now().Add(-refreshLookback).Unix() - domains, err := c.checker.DomainsSeenSince(since) + domains, err := c.domainsSeenSince(since) if err != nil { slog.Error("cache refresh: failed to query pi-hole domains", "err", err) return @@ -51,7 +56,7 @@ func (c *IPCache) Refresh(ctx context.Context) { return } - sort.Strings(domains) + slices.Sort(domains) sem := make(chan struct{}, maxConcurrentLookups) @@ -62,14 +67,16 @@ func (c *IPCache) Refresh(ctx context.Context) { resolved := make(map[string][]string, len(domains)) var wg sync.WaitGroup -spawnLoop: for _, domain := range domains { select { case <-ctx.Done(): - break spawnLoop case sem <- struct{}{}: } + if ctx.Err() != nil { + break + } + wg.Add(1) go func(d string) { defer wg.Done() @@ -78,7 +85,7 @@ spawnLoop: rctx, cancel := context.WithTimeout(ctx, lookupTimeout) defer cancel() - ips, err := net.DefaultResolver.LookupHost(rctx, d) + ips, err := c.lookupHost(rctx, d) if err != nil { // DNS failure is normal for blocked domains, expired records, and similar cases. return diff --git a/internal/pihole/cache_test.go b/internal/pihole/cache_test.go new file mode 100644 index 0000000..44e3881 --- /dev/null +++ b/internal/pihole/cache_test.go @@ -0,0 +1,59 @@ +package pihole + +import ( + "context" + "path/filepath" + "reflect" + "testing" + + "github.com/hexbytedev/hexwall/internal/store" +) + +func TestIPCacheRefreshWritesTrustedIPAndDomains(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + hexwallStore, err := store.NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + cache := &IPCache{ + store: hexwallStore, + domainsSeenSince: func(int64) ([]string, error) { + return []string{"example.com", "example.org"}, nil + }, + lookupHost: func(_ context.Context, host string) ([]string, error) { + switch host { + case "example.com": + return []string{"93.184.216.34", "93.184.216.35"}, nil + case "example.org": + return []string{"93.184.216.34"}, nil + default: + return nil, nil + } + }, + } + + cache.Refresh(context.Background()) + + allowed, err := hexwallStore.IsAllowed("93.184.216.34") + if err != nil { + t.Fatalf("IsAllowed() error = %v", err) + } + if !allowed { + t.Fatal("IsAllowed() = false, want true") + } + + domains, err := hexwallStore.DomainsForIP("93.184.216.34") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + want := []string{"example.com", "example.org"} + if !reflect.DeepEqual(domains, want) { + t.Fatalf("DomainsForIP() = %v, want %v", domains, want) + } +} diff --git a/internal/somo/somo_test.go b/internal/somo/somo_test.go new file mode 100644 index 0000000..0db6450 --- /dev/null +++ b/internal/somo/somo_test.go @@ -0,0 +1,163 @@ +package somo + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// installFakeSomo writes an executable shell script named "somo" into a temp +// directory and prepends that directory to PATH, so exec.LookPath("somo") and +// exec.Command("somo", ...) resolve to the fake instead of a real install. +func installFakeSomo(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + path := filepath.Join(dir, "somo") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake somo: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestCheckInstalledSucceedsWhenOnPath(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\nexit 0\n") + + if err := CheckInstalled(); err != nil { + t.Fatalf("CheckInstalled() error = %v, want nil", err) + } +} + +func TestCheckInstalledFailsWhenPathEmpty(t *testing.T) { + t.Setenv("PATH", "") + + if err := CheckInstalled(); err == nil { + t.Fatal("CheckInstalled() error = nil, want error for empty PATH") + } +} + +func TestCheckInstalledFailsWhenNotOnPath(t *testing.T) { + // A PATH pointing only at an empty directory has no somo binary. + t.Setenv("PATH", t.TempDir()) + + if err := CheckInstalled(); err == nil { + t.Fatal("CheckInstalled() error = nil, want error when somo is not on PATH") + } +} + +func TestGetEstablishedConnectionsParsesAllFields(t *testing.T) { + installFakeSomo(t, `#!/bin/sh +echo '[{"proto":"tcp","local_port":"51820","remote_address":"203.0.113.10","remote_port":"443","program":"curl","pid":"1234","state":"ESTABLISHED","address_type":"ipv4"}]' +`) + + conns, err := GetEstablishedConnections() + if err != nil { + t.Fatalf("GetEstablishedConnections() error = %v", err) + } + if len(conns) != 1 { + t.Fatalf("GetEstablishedConnections() len = %d, want 1", len(conns)) + } + + want := Connection{ + Protocol: "tcp", + LPort: "51820", + RAddress: "203.0.113.10", + RPort: "443", + Program: "curl", + PID: "1234", + State: "ESTABLISHED", + AddressType: "ipv4", + } + if conns[0] != want { + t.Fatalf("GetEstablishedConnections()[0] = %+v, want %+v", conns[0], want) + } +} + +func TestGetEstablishedConnectionsEmptyArray(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho '[]'\n") + + conns, err := GetEstablishedConnections() + if err != nil { + t.Fatalf("GetEstablishedConnections() error = %v", err) + } + if len(conns) != 0 { + t.Fatalf("GetEstablishedConnections() len = %d, want 0", len(conns)) + } +} + +func TestGetEstablishedConnectionsMalformedJSON(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho 'not json'\n") + + _, err := GetEstablishedConnections() + if err == nil { + t.Fatal("GetEstablishedConnections() error = nil, want error for malformed JSON") + } + if !strings.Contains(err.Error(), "failed to parse") { + t.Fatalf("GetEstablishedConnections() error = %q, want it to mention %q", err.Error(), "failed to parse") + } +} + +func TestGetEstablishedConnectionsNonZeroExit(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho 'somo: permission denied' 1>&2\nexit 1\n") + + _, err := GetEstablishedConnections() + if err == nil { + t.Fatal("GetEstablishedConnections() error = nil, want error for non-zero exit") + } + if !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("GetEstablishedConnections() error = %q, want it to include command output", err.Error()) + } +} + +func TestCommandError(t *testing.T) { + t.Parallel() + + wrapped := errors.New("exit status 1") + + tests := []struct { + name string + message string + err error + output []byte + want string + }{ + { + name: "empty output", + message: "failed to run somo", + err: wrapped, + output: nil, + want: "failed to run somo: exit status 1", + }, + { + name: "whitespace-only output", + message: "failed to run somo", + err: wrapped, + output: []byte(" \n\t "), + want: "failed to run somo: exit status 1", + }, + { + name: "non-empty output", + message: "failed to run somo", + err: wrapped, + output: []byte("permission denied\n"), + want: "failed to run somo: exit status 1: permission denied", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := commandError(tt.message, tt.err, tt.output) + if got.Error() != tt.want { + t.Fatalf("commandError() = %q, want %q", got.Error(), tt.want) + } + if !errors.Is(got, wrapped) { + t.Fatal("commandError() does not wrap the original error") + } + }) + } +} From a41293b95af828ba27065cc60f22bafaf7c1e232 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 21 Sep 2026 13:32:02 +0600 Subject: [PATCH 11/56] refactor(zeek): split ssl.log SNI cache from notice.log alert path zeek.go bundled two unrelated systems: the ssl.log tailer that builds the in-memory SNI cache monitor.RunScan queries, and the independent notice.log watcher that feeds the DNS-bypass alert path. Split into zeek.go (Client/SNI cache) and notice.go (Event/ParseNoticeLine/ WatchNoticeLog), with tests split the same way. Also added missing doc comments on the two exported notice-parsing errors, per golangci-lint. No behavior change. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018yCDQPH4oP4xQcGZUKMqaH --- internal/zeek/notice.go | 152 +++++++++++++++++++++++++++++++++++ internal/zeek/notice_test.go | 41 ++++++++++ internal/zeek/zeek.go | 135 ++----------------------------- internal/zeek/zeek_test.go | 35 -------- 4 files changed, 199 insertions(+), 164 deletions(-) create mode 100644 internal/zeek/notice.go create mode 100644 internal/zeek/notice_test.go diff --git a/internal/zeek/notice.go b/internal/zeek/notice.go new file mode 100644 index 0000000..76a2898 --- /dev/null +++ b/internal/zeek/notice.go @@ -0,0 +1,152 @@ +package zeek + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "strings" + "time" +) + +var ( + // ErrNoticeLineEmpty indicates a notice log line was blank. + ErrNoticeLineEmpty = errors.New("empty notice line") + // ErrNoticeLineNoFields indicates a notice log line parsed as JSON but had none of the recognized fields. + ErrNoticeLineNoFields = errors.New("notice line did not contain recognizable fields") +) + +// Event is a parsed DNS bypass notice from Zeek. +type Event struct { + Timestamp time.Time + SrcIP string + DstIP string + DstPort string + SNI string +} + +// ParseNoticeLine parses a single Zeek notice JSON line into an Event. +func ParseNoticeLine(line string) (Event, error) { + line = strings.TrimSpace(line) + if line == "" { + return Event{}, ErrNoticeLineEmpty + } + + var payload map[string]any + if err := json.Unmarshal([]byte(line), &payload); err != nil { + return Event{}, fmt.Errorf("parse notice line: %w", err) + } + + event := Event{ + Timestamp: time.Now(), + } + + if ts, ok := payload["ts"].(string); ok { + if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil { + event.Timestamp = parsed + } + } + + if ts, ok := payload["timestamp"].(string); ok { + if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil { + event.Timestamp = parsed + } + } + + event.SrcIP = stringValue(payload["src_ip"], payload["src"], payload["id.orig_h"]) + event.DstIP = stringValue(payload["dst_ip"], payload["dst"], payload["id.resp_h"]) + event.DstPort = stringValue(payload["dst_port"], payload["p"], payload["id.resp_p"]) + event.SNI = stringValue(payload["sni"], payload["server_name"]) + + if event.SrcIP == "" && event.DstIP == "" && event.SNI == "" { + return Event{}, ErrNoticeLineNoFields + } + + return event, nil +} + +// WatchNoticeLog tails a Zeek notice log and emits parsed Events. +func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error { + logPath := strings.TrimSpace(path) + if ctx == nil || logPath == "" { + return nil + } + + var offset int64 + poller := time.NewTicker(2 * time.Second) + defer poller.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-poller.C: + file, err := os.Open(logPath) + if err != nil { + slog.Debug("zeek: cannot open notice log", "path", logPath, "err", err) + continue + } + + fi, err := file.Stat() + if err != nil { + _ = file.Close() + slog.Debug("zeek: cannot stat notice log", "path", logPath, "err", err) + continue + } + + if fi.Size() < offset { + offset = 0 + } + if fi.Size() > offset { + if _, err := file.Seek(offset, io.SeekStart); err != nil { + _ = file.Close() + slog.Debug("zeek: seek notice log failed", "path", logPath, "err", err) + continue + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + event, err := ParseNoticeLine(line) + if err != nil { + continue + } + + select { + case events <- event: + default: + } + } + if err := scanner.Err(); err != nil { + slog.Debug("zeek: notice log scan failed", "path", logPath, "err", err) + } + } + _ = file.Close() + offset = fi.Size() + } + } +} + +func stringValue(values ...any) string { + for _, value := range values { + switch v := value.(type) { + case string: + return strings.TrimSpace(v) + case fmt.Stringer: + return strings.TrimSpace(v.String()) + case nil: + continue + default: + return fmt.Sprint(v) + } + } + return "" +} diff --git a/internal/zeek/notice_test.go b/internal/zeek/notice_test.go new file mode 100644 index 0000000..0aa6769 --- /dev/null +++ b/internal/zeek/notice_test.go @@ -0,0 +1,41 @@ +package zeek + +import ( + "testing" + "time" +) + +func TestParseNoticeLine(t *testing.T) { + line := `{"note":"DNSBypass::PossibleBypass","src_ip":"192.0.2.10","dst_ip":"203.0.113.20","dst_port":"443","sni":"evil.example","msg":"SNI evil.example from 192.0.2.10"}` + + event, err := ParseNoticeLine(line) + if err != nil { + t.Fatalf("ParseNoticeLine() error = %v", err) + } + + if event.SrcIP != "192.0.2.10" { + t.Fatalf("ParseNoticeLine().SrcIP = %q, want %q", event.SrcIP, "192.0.2.10") + } + if event.DstIP != "203.0.113.20" { + t.Fatalf("ParseNoticeLine().DstIP = %q, want %q", event.DstIP, "203.0.113.20") + } + if event.DstPort != "443" { + t.Fatalf("ParseNoticeLine().DstPort = %q, want %q", event.DstPort, "443") + } + if event.SNI != "evil.example" { + t.Fatalf("ParseNoticeLine().SNI = %q, want %q", event.SNI, "evil.example") + } + if event.Timestamp.IsZero() { + t.Fatal("ParseNoticeLine().Timestamp = zero time, want non-zero") + } + if event.Timestamp.After(time.Now().Add(time.Second)) { + t.Fatalf("ParseNoticeLine().Timestamp too far in the future: %v", event.Timestamp) + } +} + +func TestParseNoticeLineRejectsInvalidJSON(t *testing.T) { + _, err := ParseNoticeLine(`{"note":`) + if err == nil { + t.Fatal("ParseNoticeLine() error = nil, want error") + } +} diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go index 3a9cdc0..39704f3 100644 --- a/internal/zeek/zeek.go +++ b/internal/zeek/zeek.go @@ -2,11 +2,9 @@ package zeek import ( - "bufio" "bytes" "context" "encoding/json" - "errors" "fmt" "io" "log/slog" @@ -26,15 +24,6 @@ type Config struct { LogPath string } -// Event is a parsed DNS bypass notice from Zeek. -type Event struct { - Timestamp time.Time - SrcIP string - DstIP string - DstPort string - SNI string -} - type sniEntry struct { domain string observedAt time.Time @@ -78,8 +67,11 @@ func NewClient(cfg *Config) (*Client, error) { if err != nil { return nil, fmt.Errorf("failed to open zeek ssl.log %s: %w", logPath, err) } + fi, err := f.Stat() + _ = f.Close() + if err != nil { return nil, fmt.Errorf("failed to stat zeek ssl.log %s: %w", logPath, err) } @@ -104,7 +96,9 @@ func (c *Client) Close() { if c == nil { return } + close(c.stopCh) + <-c.done } @@ -128,108 +122,6 @@ func (c *Client) Lookup(localPort, remoteIP, remotePort string) (string, bool) { return entry.domain, true } -// ParseNoticeLine parses a single Zeek notice JSON line into an Event. -func ParseNoticeLine(line string) (Event, error) { - line = strings.TrimSpace(line) - if line == "" { - return Event{}, errors.New("empty notice line") - } - - var payload map[string]any - if err := json.Unmarshal([]byte(line), &payload); err != nil { - return Event{}, fmt.Errorf("parse notice line: %w", err) - } - - event := Event{Timestamp: time.Now()} - if ts, ok := payload["ts"].(string); ok { - if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil { - event.Timestamp = parsed - } - } - if ts, ok := payload["timestamp"].(string); ok { - if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil { - event.Timestamp = parsed - } - } - - event.SrcIP = stringValue(payload["src_ip"], payload["src"], payload["id.orig_h"]) - event.DstIP = stringValue(payload["dst_ip"], payload["dst"], payload["id.resp_h"]) - event.DstPort = stringValue(payload["dst_port"], payload["p"], payload["id.resp_p"]) - event.SNI = stringValue(payload["sni"], payload["server_name"]) - - if event.SrcIP == "" && event.DstIP == "" && event.SNI == "" { - return Event{}, errors.New("notice line did not contain recognizable fields") - } - - return event, nil -} - -// WatchNoticeLog tails a Zeek notice log and emits parsed Events. -func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error { - logPath := strings.TrimSpace(path) - if ctx == nil || logPath == "" { - return nil - } - - var offset int64 - poller := time.NewTicker(2 * time.Second) - defer poller.Stop() - - for { - select { - case <-ctx.Done(): - return nil - case <-poller.C: - file, err := os.Open(logPath) - if err != nil { - slog.Debug("zeek: cannot open notice log", "path", logPath, "err", err) - continue - } - - fi, err := file.Stat() - if err != nil { - _ = file.Close() - slog.Debug("zeek: cannot stat notice log", "path", logPath, "err", err) - continue - } - - if fi.Size() < offset { - offset = 0 - } - if fi.Size() > offset { - if _, err := file.Seek(offset, io.SeekStart); err != nil { - _ = file.Close() - slog.Debug("zeek: seek notice log failed", "path", logPath, "err", err) - continue - } - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - - event, err := ParseNoticeLine(line) - if err != nil { - continue - } - - select { - case events <- event: - default: - } - } - if err := scanner.Err(); err != nil { - slog.Debug("zeek: notice log scan failed", "path", logPath, "err", err) - } - } - _ = file.Close() - offset = fi.Size() - } - } -} - func (c *Client) tailLoop(ctx context.Context) { defer close(c.done) @@ -254,6 +146,7 @@ func (c *Client) processNewLines() { slog.Debug("zeek: cannot open ssl.log", "path", c.logPath, "err", err) return } + defer func() { if err := file.Close(); err != nil { slog.Debug("zeek: failed to close ssl.log", "path", c.logPath, "err", err) @@ -342,19 +235,3 @@ func (c *Client) processNewLines() { } c.mu.Unlock() } - -func stringValue(values ...any) string { - for _, value := range values { - switch v := value.(type) { - case string: - return strings.TrimSpace(v) - case fmt.Stringer: - return strings.TrimSpace(v.String()) - case nil: - continue - default: - return fmt.Sprint(v) - } - } - return "" -} diff --git a/internal/zeek/zeek_test.go b/internal/zeek/zeek_test.go index 2991a4b..9cb9c85 100644 --- a/internal/zeek/zeek_test.go +++ b/internal/zeek/zeek_test.go @@ -25,41 +25,6 @@ func writeFile(t *testing.T, path, content string) { } } -func TestParseNoticeLine(t *testing.T) { - line := `{"note":"DNSBypass::PossibleBypass","src_ip":"192.0.2.10","dst_ip":"203.0.113.20","dst_port":"443","sni":"evil.example","msg":"SNI evil.example from 192.0.2.10"}` - - event, err := ParseNoticeLine(line) - if err != nil { - t.Fatalf("ParseNoticeLine() error = %v", err) - } - - if event.SrcIP != "192.0.2.10" { - t.Fatalf("ParseNoticeLine().SrcIP = %q, want %q", event.SrcIP, "192.0.2.10") - } - if event.DstIP != "203.0.113.20" { - t.Fatalf("ParseNoticeLine().DstIP = %q, want %q", event.DstIP, "203.0.113.20") - } - if event.DstPort != "443" { - t.Fatalf("ParseNoticeLine().DstPort = %q, want %q", event.DstPort, "443") - } - if event.SNI != "evil.example" { - t.Fatalf("ParseNoticeLine().SNI = %q, want %q", event.SNI, "evil.example") - } - if event.Timestamp.IsZero() { - t.Fatal("ParseNoticeLine().Timestamp = zero time, want non-zero") - } - if event.Timestamp.After(time.Now().Add(time.Second)) { - t.Fatalf("ParseNoticeLine().Timestamp too far in the future: %v", event.Timestamp) - } -} - -func TestParseNoticeLineRejectsInvalidJSON(t *testing.T) { - _, err := ParseNoticeLine(`{"note":`) - if err == nil { - t.Fatal("ParseNoticeLine() error = nil, want error") - } -} - func TestClientProcessNewLines_BasicParse(t *testing.T) { path := filepath.Join(t.TempDir(), "ssl.log") writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}`+"\n") From 1f5b261229379ff1feeb8a85b39438d51583f443 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 21 Sep 2026 15:06:34 +0600 Subject: [PATCH 12/56] refactor(monitor): collapse RunScan's 9 params into a Scanner struct RunScan took 9 parameters and evaluateIPOnly took 8, all long-lived collaborators (checker, store, deghost/zeek clients) and config flags (mode, deghostIP, deghostDomain, debug) that don't change between scan ticks. Introduces monitor.Scanner holding them as fields; RunScan, evaluateIPOnly, applyBlockedOutcome, and HandleZeekEvent become methods on it. main.go builds one Scanner after its dependencies exist and calls scanner.Run(ctx) / scanner.HandleZeekEvent(event) instead of threading the same 8-9 arguments through every call site. No behavior change. Tests updated to construct *Scanner instead of calling the old free functions. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018yCDQPH4oP4xQcGZUKMqaH --- internal/monitor/monitor.go | 125 +++++++++++++++++-------------- internal/monitor/monitor_test.go | 48 +++++++----- main.go | 41 ++++++---- 3 files changed, 124 insertions(+), 90 deletions(-) diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index daa7ecd..e544ab2 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -26,6 +26,21 @@ const ( ModeEnforce = "enforce" ) +// Scanner holds the long-lived collaborators and configuration flags the +// decision ladder needs. Construct it with a struct literal — every field is +// a plain dependency or flag with nothing to validate, so there's no +// NewScanner constructor. +type Scanner struct { + Checker *pihole.Checker + Store *store.Store + DeghostClient *deghost.Client + ZeekClient *zeek.Client + Mode string + DeghostIP bool + DeghostDomain bool + Debug bool +} + // scanSummary tracks per-scan outcome counts for the summary line. type scanSummary struct { connections int @@ -74,16 +89,16 @@ func remoteIP(address string) (net.IP, error) { } // HandleZeekEvent evaluates a Zeek notice event against the current policy and persists it. -func HandleZeekEvent(checker *pihole.Checker, hexwallStore *store.Store, mode string, event zeek.Event) { - if checker == nil || hexwallStore == nil { +func (s *Scanner) HandleZeekEvent(event zeek.Event) { + if s.Checker == nil || s.Store == nil { return } if event.SNI == "" { return } - selectedMode := normalizeMode(mode) - blocked, err := checker.IsBlockedByPolicy(event.SNI) + selectedMode := normalizeMode(s.Mode) + blocked, err := s.Checker.IsBlockedByPolicy(event.SNI) if err != nil { slog.Warn("zeek policy lookup failed", "sni", event.SNI, "err", err) return @@ -98,7 +113,7 @@ func HandleZeekEvent(checker *pihole.Checker, hexwallStore *store.Store, mode st slog.Info("zeek notice was not policy-blocked", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI) } - if err := hexwallStore.LogZeekAlert(event.SrcIP, event.DstIP, event.DstPort, event.SNI, blocked, confidence, actionTaken); err != nil { + if err := s.Store.LogZeekAlert(event.SrcIP, event.DstIP, event.DstPort, event.SNI, blocked, confidence, actionTaken); err != nil { slog.Error("failed to persist zeek alert", "src_ip", event.SrcIP, "sni", event.SNI, "err", err) } @@ -107,7 +122,7 @@ func HandleZeekEvent(checker *pihole.Checker, hexwallStore *store.Store, mode st } } -// RunScan inspects established connections and applies the decision ladder. +// Run inspects established connections and applies the decision ladder. // // The ladder is ordered from cheapest/most-authoritative to most-expensive/least-authoritative: // 1. Static CIDR allowlist (loopback, RFC1918, cloud metadata) @@ -119,15 +134,15 @@ func HandleZeekEvent(checker *pihole.Checker, hexwallStore *store.Store, mode st // 7. IP-only fallback → deghost escalation if enabled; no action if disabled // // Safety property: with deghost disabled, only rung 4 (Pi-hole blocklist match) can kill. -func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.Store, deghostClient *deghost.Client, deghostIP bool, deghostDomain bool, zeekClient *zeek.Client, mode string, debug bool) { - if hexwallStore == nil { +func (s *Scanner) Run(ctx context.Context) { + if s.Store == nil { slog.Error("scan aborted: nil hexwall store") return } - selectedMode := normalizeMode(mode) - if selectedMode != mode { - slog.Warn("invalid scan mode; defaulting to watch", "mode", mode, "fallback", selectedMode) + selectedMode := normalizeMode(s.Mode) + if selectedMode != s.Mode { + slog.Warn("invalid scan mode; defaulting to watch", "mode", s.Mode, "fallback", selectedMode) } connections, err := somo.GetEstablishedConnections() @@ -164,8 +179,8 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S // Rung 2: did Zeek see an SNI for this connection? var sniDomain string sniFound := false - if zeekClient != nil { - if domain, ok := zeekClient.Lookup(conn.LPort, ipStr, conn.RPort); ok { + if s.ZeekClient != nil { + if domain, ok := s.ZeekClient.Lookup(conn.LPort, ipStr, conn.RPort); ok { sniDomain = domain sniFound = true summary.withSNI++ @@ -178,7 +193,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S if !sniFound { // Rung 7: IP-only fallback — no SNI means non-TLS, pre-existing connection, or ECH. - action := evaluateIPOnly(ctx, hexwallStore, deghostClient, deghostIP, ipStr, conn, selectedMode, debug) + action := s.evaluateIPOnly(ctx, ipStr, conn) applyOutcome(&summary, action) continue } @@ -186,7 +201,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S domain := strings.ToLower(strings.TrimSpace(sniDomain)) // Rung 3: Pi-hole explicit allowlist — user intent outranks everything inferred. - allowed, err := checker.IsAllowedByPolicy(domain) + allowed, err := s.Checker.IsAllowedByPolicy(domain) if err != nil { // Fail-open: if we cannot determine whether a domain is explicitly allowed, // we must not act on a block from rung 4. Rung 4 is the only rung that can @@ -194,7 +209,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S // incomplete policy data is worse than missing one block. slog.Warn("domain allowlist lookup failed, skipping blocklist check", "domain", domain, "err", err) } else if allowed { - if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyAllowed); logErr != nil { + if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyAllowed); logErr != nil { slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } summary.trusted++ @@ -203,14 +218,14 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S // Rung 4: Pi-hole explicit denylist / gravity — only rung that can kill. if err == nil { - if blocked, err := checker.IsBlockedByPolicy(domain); err != nil { + if blocked, err := s.Checker.IsBlockedByPolicy(domain); err != nil { slog.Warn("domain denylist lookup failed", "domain", domain, "err", err) } else if blocked { - if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyBlocked); logErr != nil { + if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyBlocked); logErr != nil { slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } slog.Warn("policy-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - applyBlockedOutcome(hexwallStore, selectedMode, ipStr, conn) + s.applyBlockedOutcome(ipStr, conn) summary.blocked++ continue } @@ -218,7 +233,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S // Rung 5: inferred trust from DNS history — scoped to this IP. known := false - knownDomains, err := hexwallStore.DomainsForIP(ipStr) + knownDomains, err := s.Store.DomainsForIP(ipStr) if err != nil { slog.Debug("sni domain check failed", "domain", domain, "ip", ipStr, "err", err) } else { @@ -233,7 +248,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S if known { outcome = store.OutcomeIPDomainMatch } - if logErr := hexwallStore.LogSNIObservation(ipStr, domain, conn.LPort, outcome); logErr != nil { + if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, outcome); logErr != nil { slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } @@ -243,8 +258,8 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S } // Rung 6: unknown domain — deghost escalation if enabled. - if !deghostDomain { - if debug { + if !s.DeghostDomain { + if s.Debug { slog.Info("unknown domain, deghost disabled; no action", "domain", domain, "ip", ipStr, "program", conn.Program) } summary.unknown++ @@ -253,7 +268,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S shouldBlock, reason := false, "" - if cached, err := hexwallStore.GetRecentDomainCheck(domain); err != nil { + if cached, err := s.Store.GetRecentDomainCheck(domain); err != nil { slog.Error("domain cache lookup failed", "domain", domain, "err", err) summary.errored++ continue @@ -263,7 +278,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S } if !shouldBlock && reason == "" { - report, err := deghostClient.CheckDomain(ctx, domain) + report, err := s.DeghostClient.CheckDomain(ctx, domain) if err != nil { slog.Error("deghost domain check failed", "domain", domain, "program", conn.Program, "err", err) summary.errored++ @@ -272,13 +287,13 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S shouldBlock = deghost.ShouldBlockDomain(report) reason = "live-domain-check" - if err := hexwallStore.UpsertDomainCheck(domain, shouldBlock); err != nil { + if err := s.Store.UpsertDomainCheck(domain, shouldBlock); err != nil { slog.Error("failed to cache domain check", "domain", domain, "program", conn.Program, "err", err) } } if !shouldBlock { - if debug { + if s.Debug { slog.Info("unknown but clean domain", "domain", domain, "ip", ipStr, "program", conn.Program, "reason", reason) } summary.unknown++ @@ -286,7 +301,7 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S } slog.Warn("deghost-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", reason) - applyBlockedOutcome(hexwallStore, selectedMode, ipStr, conn) + s.applyBlockedOutcome(ipStr, conn) summary.blocked++ } @@ -297,40 +312,40 @@ func RunScan(ctx context.Context, checker *pihole.Checker, hexwallStore *store.S // evaluateIPOnly handles the IP-only fallback path (rung 7) when no SNI is available. // It returns "trusted", "blocked", "unknown", or "error". // Every exit path records an IP observation before returning. -func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClient *deghost.Client, deghostIP bool, ipStr string, conn somo.Connection, mode string, debug bool) string { - allowed, err := hexwallStore.IsAllowed(ipStr) +func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn somo.Connection) string { + allowed, err := s.Store.IsAllowed(ipStr) if err != nil { slog.Error("store lookup failed", "address", conn.RAddress, "err", err) - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } return "error" } if allowed { - if err := hexwallStore.UpdateEstablished(ipStr); err != nil { + if err := s.Store.UpdateEstablished(ipStr); err != nil { slog.Error("failed to update established", "address", conn.RAddress, "err", err) } - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPTrusted); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPTrusted); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } return "trusted" } - if !deghostIP { - if debug { + if !s.DeghostIP { + if s.Debug { slog.Info("unknown IP, deghost disabled; no action", "ip", ipStr, "program", conn.Program) } - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } return "unknown" } - cachedFraudCheck, err := hexwallStore.GetRecentFraudCheck(ipStr) + cachedFraudCheck, err := s.Store.GetRecentFraudCheck(ipStr) if err != nil { slog.Error("fraud cache lookup failed", "ip", ipStr, "program", conn.Program, "err", err) - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } return "error" @@ -338,77 +353,77 @@ func evaluateIPOnly(ctx context.Context, hexwallStore *store.Store, deghostClien if cachedFraudCheck != nil { if !cachedFraudCheck.ShouldKill { - if debug { + if s.Debug { slog.Info("unknown but clean IP (cached)", "ip", ipStr, "program", conn.Program) } - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } return "unknown" } slog.Warn("deghost-blocked IP (cached)", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } - applyBlockedOutcome(hexwallStore, mode, ipStr, conn) + s.applyBlockedOutcome(ipStr, conn) return "blocked" } - report, err := deghostClient.CheckIP(ctx, ipStr) + report, err := s.DeghostClient.CheckIP(ctx, ipStr) if err != nil { slog.Error("deghost check failed", "ip", ipStr, "program", conn.Program, "err", err) - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } return "error" } if report == nil { - if err := hexwallStore.UpsertFraudCheck(ipStr, false); err != nil { + if err := s.Store.UpsertFraudCheck(ipStr, false); err != nil { slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) } - if debug { + if s.Debug { slog.Info("unknown but clean IP (private/reserved)", "ip", ipStr, "program", conn.Program) } - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPReserved); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPReserved); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } return "unknown" } shouldKill := deghost.ShouldKill(report) - if err := hexwallStore.UpsertFraudCheck(ipStr, shouldKill); err != nil { + if err := s.Store.UpsertFraudCheck(ipStr, shouldKill); err != nil { slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) } if !shouldKill { - if debug { + if s.Debug { slog.Info("unknown but clean IP", "ip", ipStr, "program", conn.Program) } - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } return "unknown" } slog.Warn("deghost-blocked IP", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - if logErr := hexwallStore.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } - applyBlockedOutcome(hexwallStore, mode, ipStr, conn) + s.applyBlockedOutcome(ipStr, conn) return "blocked" } // applyBlockedOutcome enforces the kill/watch policy for a blocked connection. -func applyBlockedOutcome(hexwallStore *store.Store, mode, ipStr string, conn somo.Connection) { - selectedMode := normalizeMode(mode) +func (s *Scanner) applyBlockedOutcome(ipStr string, conn somo.Connection) { + selectedMode := normalizeMode(s.Mode) if selectedMode == ModeWatch { slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) return } - if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { + if err := s.Store.LogKill(ipStr, conn.PID, conn.Program); err != nil { slog.Error("failed to log kill", "address", conn.RAddress, "err", err) } diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index 6c33328..9693a7c 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -280,7 +280,8 @@ func TestHandleZeekEventBlockedDomainPersisted(t *testing.T) { hexwallStore, _ := newTestStore(t) event := zeek.Event{SrcIP: "10.0.0.5", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"} - HandleZeekEvent(checker, hexwallStore, ModeWatch, event) + scanner := &Scanner{Checker: checker, Store: hexwallStore, Mode: ModeWatch} + scanner.HandleZeekEvent(event) alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) if err != nil { @@ -299,7 +300,8 @@ func TestHandleZeekEventUnblockedDomainPersisted(t *testing.T) { hexwallStore, _ := newTestStore(t) event := zeek.Event{SrcIP: "10.0.0.5", DstIP: "203.0.113.20", DstPort: "443", SNI: "clean.example.com"} - HandleZeekEvent(checker, hexwallStore, ModeWatch, event) + scanner := &Scanner{Checker: checker, Store: hexwallStore, Mode: ModeWatch} + scanner.HandleZeekEvent(event) alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) if err != nil { @@ -319,9 +321,9 @@ func TestHandleZeekEventNoopCases(t *testing.T) { hexwallStore, _ := newTestStore(t) // None of these should panic or write anything. - HandleZeekEvent(nil, hexwallStore, ModeWatch, zeek.Event{SNI: "example.com"}) - HandleZeekEvent(checker, nil, ModeWatch, zeek.Event{SNI: "example.com"}) - HandleZeekEvent(checker, hexwallStore, ModeWatch, zeek.Event{SrcIP: "10.0.0.1", SNI: ""}) + (&Scanner{Store: hexwallStore, Mode: ModeWatch}).HandleZeekEvent(zeek.Event{SNI: "example.com"}) + (&Scanner{Checker: checker, Mode: ModeWatch}).HandleZeekEvent(zeek.Event{SNI: "example.com"}) + (&Scanner{Checker: checker, Store: hexwallStore, Mode: ModeWatch}).HandleZeekEvent(zeek.Event{SrcIP: "10.0.0.1", SNI: ""}) } // ---- applyBlockedOutcome --------------------------------------------------- @@ -330,7 +332,7 @@ func TestApplyBlockedOutcomeWatchModeNeverLogsKill(t *testing.T) { hexwallStore, dbPath := newTestStore(t) conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} - applyBlockedOutcome(hexwallStore, ModeWatch, "203.0.113.10", conn) + (&Scanner{Store: hexwallStore, Mode: ModeWatch}).applyBlockedOutcome("203.0.113.10", conn) if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { t.Fatalf("killed_connections count = %d, want 0 in watch mode", got) @@ -343,7 +345,7 @@ func TestApplyBlockedOutcomeEnforceModeLogsKillBeforeAttemptingIt(t *testing.T) // PID 1 (init) is guaranteed to be refused by enforcer.KillProcess, but the // audit row must still be written first ("audit survives the kill"). conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} - applyBlockedOutcome(hexwallStore, ModeEnforce, "203.0.113.10", conn) + (&Scanner{Store: hexwallStore, Mode: ModeEnforce}).applyBlockedOutcome("203.0.113.10", conn) if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.10"); got != 1 { t.Fatalf("killed_connections count = %d, want 1 even though the kill itself was refused", got) @@ -354,7 +356,7 @@ func TestApplyBlockedOutcomeInvalidPIDLogsWarningAndReturns(t *testing.T) { hexwallStore, dbPath := newTestStore(t) conn := somo.Connection{PID: "not-a-pid", Program: "curl", RAddress: "203.0.113.11:443"} - applyBlockedOutcome(hexwallStore, ModeEnforce, "203.0.113.11", conn) + (&Scanner{Store: hexwallStore, Mode: ModeEnforce}).applyBlockedOutcome("203.0.113.11", conn) // LogKill happens before ParsePID, so the audit row is still written. if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.11"); got != 1 { @@ -374,7 +376,8 @@ func TestEvaluateIPOnlyTrustedIPUpdatesEstablished(t *testing.T) { client := deghost.NewClient(srv.URL, time.Second) conn := somo.Connection{Program: "curl", RAddress: "203.0.113.20:443"} - got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.20", conn, ModeWatch, false) + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.20", conn) if got != "trusted" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "trusted") } @@ -390,7 +393,8 @@ func TestEvaluateIPOnlyDeghostDisabledReturnsUnknown(t *testing.T) { client := deghost.NewClient(srv.URL, time.Second) conn := somo.Connection{Program: "curl", RAddress: "203.0.113.21:443"} - got := evaluateIPOnly(context.Background(), hexwallStore, client, false, "203.0.113.21", conn, ModeWatch, false) + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: false, Mode: ModeWatch} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.21", conn) if got != "unknown" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") } @@ -406,7 +410,8 @@ func TestEvaluateIPOnlyCachedCleanReturnsUnknownNoHTTPCall(t *testing.T) { client := deghost.NewClient(srv.URL, time.Second) conn := somo.Connection{Program: "curl", RAddress: "203.0.113.22:443"} - got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.22", conn, ModeWatch, false) + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.22", conn) if got != "unknown" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") } @@ -423,7 +428,8 @@ func TestEvaluateIPOnlyCachedBlockedReturnsBlockedAndKills(t *testing.T) { // PID 1 keeps the actual kill attempt safely refused by enforcer guardrails. conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.23:443"} - got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.23", conn, ModeEnforce, false) + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeEnforce} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.23", conn) if got != "blocked" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "blocked") } @@ -442,7 +448,8 @@ func TestEvaluateIPOnlyLiveCheckPrivateReservedCachesClean(t *testing.T) { hexwallStore, _ := newTestStore(t) conn := somo.Connection{Program: "curl", RAddress: "203.0.113.24:443"} - got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.24", conn, ModeWatch, false) + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.24", conn) if got != "unknown" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") } @@ -472,7 +479,8 @@ func TestEvaluateIPOnlyLiveCheckShouldKillCachesAndBlocks(t *testing.T) { hexwallStore, dbPath := newTestStore(t) conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.25:443"} - got := evaluateIPOnly(context.Background(), hexwallStore, client, true, "203.0.113.25", conn, ModeEnforce, false) + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeEnforce} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.25", conn) if got != "blocked" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "blocked") } @@ -495,14 +503,14 @@ func TestEvaluateIPOnlyLiveCheckShouldKillCachesAndBlocks(t *testing.T) { // would also break every other store call in the same scan. Not covered here; would // require poking store internals from outside its package, which isn't a safe seam. -// ---- RunScan --------------------------------------------------------------- +// ---- Scanner.Run ------------------------------------------------------------- func TestRunScanZeroConnections(t *testing.T) { installFakeSomo(t, "#!/bin/sh\necho '[]'\n") hexwallStore, _ := newTestStore(t) // Must not panic even with nil checker/deghost/zeek dependencies. - RunScan(context.Background(), nil, hexwallStore, nil, false, false, nil, ModeWatch, false) + (&Scanner{Store: hexwallStore, Mode: ModeWatch}).Run(context.Background()) } func TestRunScanAllowlistedConnectionSkipsIPFallback(t *testing.T) { @@ -511,7 +519,7 @@ echo '[{"proto":"tcp","local_port":"51820","remote_address":"127.0.0.1","remote_ `) hexwallStore, dbPath := newTestStore(t) - RunScan(context.Background(), nil, hexwallStore, nil, false, false, nil, ModeWatch, false) + (&Scanner{Store: hexwallStore, Mode: ModeWatch}).Run(context.Background()) // Rung 1 (static allowlist) short-circuits before the IP-only fallback, so // no ip_observations row should exist for a loopback address. @@ -526,7 +534,7 @@ echo '[{"proto":"tcp","local_port":"1","remote_address":"not-an-ip","remote_port `) hexwallStore, dbPath := newTestStore(t) - RunScan(context.Background(), nil, hexwallStore, nil, false, false, nil, ModeWatch, false) + (&Scanner{Store: hexwallStore, Mode: ModeWatch}).Run(context.Background()) // The second, valid connection must still be processed through the IP-only // fallback despite the first connection's address being unparsable. @@ -541,14 +549,14 @@ echo '[{"proto":"tcp","local_port":"1","remote_address":"203.0.113.31","remote_p `) hexwallStore, dbPath := newTestStore(t) - RunScan(context.Background(), nil, hexwallStore, nil, false, false, nil, ModeWatch, false) + (&Scanner{Store: hexwallStore, Mode: ModeWatch}).Run(context.Background()) if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ? AND outcome = 'unknown'`, "203.0.113.31"); got != 1 { t.Fatalf("ip_observations unknown count = %d, want 1", got) } } -// NOTE on rung 2/3/4 (Zeek SNI -> Pi-hole allow/deny) coverage inside RunScan: +// NOTE on rung 2/3/4 (Zeek SNI -> Pi-hole allow/deny) coverage inside Scanner.Run: // zeek.Client has no exported constructor or seam that lets an external package // (monitor_test.go is package monitor, not package zeek) populate its SNI cache // without going through the real background tailer, which only polls every 5s diff --git a/main.go b/main.go index 584e9d7..b071e54 100644 --- a/main.go +++ b/main.go @@ -208,6 +208,30 @@ func run() int { zeekClient = zc } + // deghost is opt-in: it sends every unrecognized domain to a third-party API, + // which is a real privacy cost. Only construct the client when at least one check is enabled. + var deghostClient *deghost.Client + if cfg.enableDeghostIP || cfg.enableDeghostDomain { + deghostClient = deghost.NewClient(deghostBaseURL, deghostTimeout) + } else { + slog.Info("deghost disabled; unrecognized connections will not be escalated to third-party API") + } + + if cfg.mode == monitor.ModeEnforce && cfg.enableDeghostIP { + slog.Warn("enforce mode with IP deghost enabled: an external reputation verdict can now terminate a local process; shared cloud provider ranges are known to produce false positives on such feeds") + } + + scanner := &monitor.Scanner{ + Checker: checker, + Store: hexwallStore, + DeghostClient: deghostClient, + ZeekClient: zeekClient, + Mode: cfg.mode, + DeghostIP: cfg.enableDeghostIP, + DeghostDomain: cfg.enableDeghostDomain, + Debug: cfg.debug, + } + var zeekEvents = make(chan zeek.Event, 100) if cfg.enableZeek && cfg.zeekNoticeLog != "" { go func() { @@ -221,25 +245,12 @@ func run() int { case <-ctx.Done(): return case event := <-zeekEvents: - monitor.HandleZeekEvent(checker, hexwallStore, cfg.mode, event) + scanner.HandleZeekEvent(event) } } }() } - // deghost is opt-in: it sends every unrecognized domain to a third-party API, - // which is a real privacy cost. Only construct the client when at least one check is enabled. - var deghostClient *deghost.Client - if cfg.enableDeghostIP || cfg.enableDeghostDomain { - deghostClient = deghost.NewClient(deghostBaseURL, deghostTimeout) - } else { - slog.Info("deghost disabled; unrecognized connections will not be escalated to third-party API") - } - - if cfg.mode == monitor.ModeEnforce && cfg.enableDeghostIP { - slog.Warn("enforce mode with IP deghost enabled: an external reputation verdict can now terminate a local process; shared cloud provider ranges are known to produce false positives on such feeds") - } - // 5. Refresh trusted IPs before starting the monitor so the first tick does not kill legitimate connections. cache := pihole.NewIPCache(checker, hexwallStore) cache.Refresh(ctx) @@ -299,7 +310,7 @@ func run() int { slog.Info("shutting down") return 0 case <-ticker.C: - monitor.RunScan(ctx, checker, hexwallStore, deghostClient, cfg.enableDeghostIP, cfg.enableDeghostDomain, zeekClient, cfg.mode, cfg.debug) + scanner.Run(ctx) } } } From acbc62ad7d396833b41bded7f7505d7368d0bc41 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 21 Sep 2026 18:54:30 +0600 Subject: [PATCH 13/56] docs: bring README in line with the Zeek/deghost decision ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI flags table was missing --enable-zeek, --zeek-log, --zeek-notice-log, --enable-deghost-ip, --enable-deghost-domain, and --version entirely. Worse, "When a connection is killed" only described the IP-only fraud-API path, implying that was the sole trigger — it omitted that a Pi-hole domain denylist match (via Zeek SNI) can kill on its own with both deghost flags disabled. Also refreshed the stale three-category debug-logging description and added Zeek to Dependencies. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018yCDQPH4oP4xQcGZUKMqaH --- README.md | 91 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 2e2c4f5..8c24574 100644 --- a/README.md +++ b/README.md @@ -31,17 +31,20 @@ In short: `Pi-hole + somo + hexwall` gives you a practical containment layer for Pi-hole operates at the DNS layer. It can block a domain, but it has no visibility into connections that bypass DNS entirely — processes that dial a hard-coded IP address directly. Malware and compromised packages commonly use this technique to phone home without triggering any DNS-based block. -hexwall works from the inverse assumption: **if an IP is not trusted, the connection is suspicious by default.** Trust is granted when the IP matches the built-in CIDR allowlist, was refreshed from Pi-hole query history within the last hour, or was already observed as an established connection within the last 60 seconds. On startup it immediately refreshes trusted IPs from Pi-hole, then refreshes them every 30 seconds while scanning connections every 10 seconds. Anything that falls outside that trust set is checked against an external fraud API, with results cached locally for 6 hours per IP. Confirmed threats are either logged (watch mode) or killed (enforce mode). +hexwall works from the inverse assumption: **if a connection is not trusted, it is suspicious by default.** Trust is granted when the IP matches the built-in CIDR allowlist, was refreshed from Pi-hole query history within the last hour, or was already observed as an established connection within the last 60 seconds. On startup it immediately refreshes trusted IPs from Pi-hole, then refreshes them every 30 seconds while scanning connections every 10 seconds. + +When Zeek is enabled (`--zeek-log`), hexwall also captures the TLS SNI for each connection and checks that exact domain against Pi-hole's own allow/deny lists before ever falling back to third-party reputation — this is what lets it tell apart a blocked site from a harmless one even when both happen to share the same IP, as is common behind a CDN. Anything that still falls outside every trust check is optionally escalated to an external reputation API, with results cached locally for 6 hours. Confirmed threats are either logged (watch mode) or killed (enforce mode). --- ## Dependencies -| Dependency | Role | Required | -| ----------------------------------------- | --------------------------------------------------- | --------------------------- | -| [`somo`](https://github.com/theopfr/somo) | Lists established TCP/UDP connections; kills by PID | Must be in `$PATH` | -| Pi-hole FTL database | Source of trusted DNS history | Auto-detected or via `--db` | -| `deghostapi.hexbyte.dev` | Fraud/threat classification for unknown IPs | Network access required | +| Dependency | Role | Required | +| ----------------------------------------- | ----------------------------------------------------------- | ------------------------------ | +| [`somo`](https://github.com/theopfr/somo) | Lists established TCP/UDP connections; kills by PID | Must be in `$PATH` | +| Pi-hole FTL database | Source of trusted DNS history and allow/deny policy lists | Auto-detected or via `--db` | +| `deghostapi.hexbyte.dev` | Fraud/threat classification for unrecognized IPs and domains | Optional — see `--enable-deghost-*` below | +| Zeek | Passive capture of TLS SNI (`ssl.log`) and DNS-bypass notices (`notice.log`) | Optional — see `--zeek-*`/`--enable-zeek` below | **Build dependency:** `modernc.org/sqlite` (pure-Go SQLite, no CGo or system libsqlite3 required). @@ -95,63 +98,67 @@ If neither is found, start fails. Use `--db` to specify the path manually. ## CLI flags -| Flag | Default | Description | -| ------------ | ------------------- | ---------------------------------------------------------------- | -| `--db` | _(auto-detected)_ | Path to `pihole-FTL.db` | -|`--hexwall-db`| `./hexwall.db` | Path to the local hexwall database (created on first run) | -| `--mode` | `watch` | `watch` — detect and log only; `enforce` — detect, log, and kill | -| `--debug` | `false` | Enable verbose per-connection scan logging | +| Flag | Default | Description | +| ------------------------- | ------------------------------------- | ---------------------------------------------------------------------------- | +| `--db` | _(auto-detected)_ | Path to `pihole-FTL.db` | +| `--hexwall-db` | `./hexwall.db` | Path to the local hexwall database (created on first run) | +| `--mode` | `watch` | `watch` — detect and log only; `enforce` — detect, log, and kill | +| `--debug` | `false` | Enable verbose per-connection scan logging | +| `--enable-zeek` | `false` | Enable the Zeek `notice.log` DNS-bypass alert path (independent of `--zeek-log`) | +| `--zeek-log` | _(unset — SNI matching skipped)_ | Path to Zeek's `ssl.log`, used to match a connection's TLS SNI against Pi-hole's allow/deny lists | +| `--zeek-notice-log` | `/opt/zeek/logs/current/notice.log` | Path to Zeek's `notice.log`; only read when `--enable-zeek` is set | +| `--enable-deghost-ip` | `true` | Enable third-party IP reputation checks for unrecognized connections with no SNI | +| `--enable-deghost-domain` | `true` | Enable third-party domain reputation checks for unrecognized SNI domains | +| `--version` | `false` | Print version and exit | `--mode` accepts `watch` or `enforce` (case-insensitive). Any other value exits with an error. -### Debug logging +`--enable-zeek` and `--zeek-log` are independent switches for two unrelated Zeek consumers: `--zeek-log` feeds SNI-based domain matching into the main decision ladder, while `--enable-zeek` (plus `--zeek-notice-log`) turns on a separate, standalone DNS-bypass alert path. Either, both, or neither can be set. -When `--debug` is enabled, each scan cycle logs every connection it encounters with its status: +### Debug logging -- **allowed** — IP is in the local trust cache or allowlist -- **unrecognized-clean** — IP is not in cache, but the fraud API returned a clean verdict (or 403 for private/reserved ranges) -- **vulnerable** — IP is not in cache, fraud API flagged it as abuser/attacker/threat +When `--debug` is enabled, hexwall also logs intermediate, non-actionable outcomes it would otherwise stay quiet about — a domain or IP coming back clean from a reputation check, or a check being skipped because its flag is disabled. Warnings, blocks, and kills are always logged regardless of `--debug`. -When debug is disabled (default), only non-allowed results are logged. Additionally, empty scans (somo returning zero connections) are always logged regardless of debug mode. +Every scan tick prints a one-line summary regardless of `--debug` — connection count, and how many were trusted, blocked, unknown, or errored — and an empty scan (`somo` returning zero connections) is always logged too. --- ## When a connection is killed -A connection is killed only when all of the following are true: +A connection is only ever killed in `--mode enforce`, for a connection `somo` reports as currently established. Three independent checks can trigger it: + +1. **Pi-hole domain denylist match** — Zeek captured the connection's TLS SNI (via `--zeek-log`) and that domain is on Pi-hole's own denylist or gravity blocklist. Requires only Pi-hole; no third-party service is involved, and this check runs regardless of the `--enable-deghost-*` flags. +2. **Third-party domain reputation match** — an SNI was captured, the domain is on neither Pi-hole list and isn't already known-trusted for this IP, and the reputation API's verdict (fetched live or reused from a 6-hour cache) says to block it. Requires `--enable-deghost-domain` (default `true`). +3. **Third-party IP reputation match** — no SNI was available at all (non-TLS traffic, ECH, or a connection predating startup), the IP is not trusted, and the reputation API's verdict on the bare IP (live or 6-hour cached) marks it `is_abuser`, `is_attacker`, or `is_threat`. Requires `--enable-deghost-ip` (default `true`). -1. `--mode enforce` is active. -2. `somo` reports the connection as currently established. -3. The remote IP is not trusted. -4. The remote IP's current fraud decision, either fetched live or reused from the local 6-hour cache, marks it as `is_abuser`, `is_attacker`, or `is_threat`. +With both `--enable-deghost-ip` and `--enable-deghost-domain` set to `false`, only the first check — Pi-hole's own denylist — can ever trigger a kill. -An IP is considered trusted when any of these are true: +A connection is treated as trusted, and skipped entirely, when any of these are true: -- It matches the built-in CIDR allowlist. -- It was refreshed from Pi-hole history within the last hour. -- It was already seen as an established allowed connection within the last 60 seconds. +- Its IP matches the built-in CIDR allowlist. +- Its domain is explicitly on Pi-hole's allowlist. +- Its IP was refreshed from Pi-hole query history within the last hour, or was already seen as an established connection within the last 60 seconds. +- Its domain has already been seen for this exact IP in hexwall's own history. Connections are not killed in these cases: -- `--mode watch` is active. The connection is only logged as `would kill`. -- The IP is on the built-in allowlist. -- The IP is still trusted from a recent Pi-hole refresh or recent established-connection activity. -- The IP already has a cached clean fraud verdict from the last 6 hours. -- The fraud API returns HTTP `403`, which is treated as clean/private/reserved. -- The fraud API returns a report but none of `is_abuser`, `is_attacker`, or `is_threat` are true. -- The fraud lookup itself fails. +- `--mode watch` is active — a would-be kill is only logged as `would kill`. +- The relevant reputation check is disabled via its flag. +- A cached reputation verdict from the last 6 hours says clean. +- The IP reputation API returns HTTP `403`, treated as clean/private/reserved. +- The reputation lookup itself fails — a failed lookup is never treated as a block. -When a kill does happen, hexwall first records the event in the local `killed_connections` audit table and then asks `somo` to kill the owning PID. +When a kill does happen, hexwall first records the event in the local `killed_connections` audit table, then signals the owning PID: `SIGTERM` first, escalating to `SIGKILL` after a 2-second grace period if the process is still alive. A hardcoded set of protected system processes (`systemd`, `sshd`, `dockerd`, and similar) can never be killed this way, whichever check triggered the match. -### Fraud API cache behavior +### Reputation API cache behavior -Fraud lookups are cached in the local SQLite database for 6 hours per IP. +Both IP and domain reputation lookups are cached in the local SQLite database for 6 hours, keyed by IP or by domain respectively. -- If an untrusted IP has a cached fraud decision newer than 6 hours, hexwall reuses that cached result and does not call the fraud API again. -- If the cached decision is older than 6 hours, the next scan calls the fraud API again and refreshes the cache timestamp. -- Both clean and kill-worthy fraud decisions are cached. -- HTTP `403` responses are treated as clean/private/reserved and cached as a non-kill result. -- Fraud lookup failures are not cached. +- If a cached decision is newer than 6 hours, hexwall reuses it and does not call the reputation API again. +- If the cached decision is older than 6 hours, the next scan calls the API again and refreshes the cache timestamp. +- Both clean and block-worthy decisions are cached. +- HTTP `403` responses (IP checks only) are treated as clean/private/reserved and cached as a non-kill result. +- A failed lookup is not cached, so it will be retried on the next scan. This cache lives in the local hexwall database as Unix timestamps (`INTEGER` / int64-style seconds) and survives restarts. From d4fae4acaee822430487549e7e5cdb475865ae0e Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 21 Sep 2026 18:55:45 +0600 Subject: [PATCH 14/56] test: add coverage for internal/detector detector had zero tests. Split FindDBPath's body into an unexported findDBPath that takes the physical path and Docker fallback as parameters (same injectable-dependency pattern already used in pihole.IPCache), so tests can exercise the physical-found, physical-is-a-directory, and Docker-fallback branches without touching the real /etc/pihole or requiring a Docker daemon. findPiholeContainer and findDockerDBPath are covered via the same fake-executable-on-PATH technique already used for somo. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018yCDQPH4oP4xQcGZUKMqaH --- internal/detector/detector.go | 17 +- internal/detector/detector_test.go | 290 +++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 internal/detector/detector_test.go diff --git a/internal/detector/detector.go b/internal/detector/detector.go index 86f9678..47a5958 100644 --- a/internal/detector/detector.go +++ b/internal/detector/detector.go @@ -23,19 +23,26 @@ const ( // // Returns an error if neither is found. func FindDBPath() (string, error) { + return findDBPath(physicalDBPath, findDockerDBPath) +} + +// findDBPath is FindDBPath's logic with the physical path and Docker fallback +// injectable, so tests can exercise both branches without touching +// /etc/pihole or requiring a real Docker daemon. +func findDBPath(physicalPath string, dockerFallback func() (string, error)) (string, error) { // Check the default host installation first. - if err := validateDBFile(physicalDBPath); err == nil { - return physicalDBPath, nil + if err := validateDBFile(physicalPath); err == nil { + return physicalPath, nil } else if !errors.Is(err, os.ErrNotExist) { - return "", fmt.Errorf("failed to access physical pi-hole DB at %s: %w", physicalDBPath, err) + return "", fmt.Errorf("failed to access physical pi-hole DB at %s: %w", physicalPath, err) } // Fall back to a running Docker installation. - path, err := findDockerDBPath() + path, err := dockerFallback() if err != nil { return "", fmt.Errorf( "pi-hole not found: no physical installation at %s and no running pi-hole Docker container (%w)", - physicalDBPath, err, + physicalPath, err, ) } diff --git a/internal/detector/detector_test.go b/internal/detector/detector_test.go new file mode 100644 index 0000000..b924c73 --- /dev/null +++ b/internal/detector/detector_test.go @@ -0,0 +1,290 @@ +package detector + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// installFakeDocker writes an executable shell script named "docker" into a +// temp directory and prepends it to PATH, mirroring the same trick used for +// somo elsewhere in this project (see internal/somo/somo_test.go). +func installFakeDocker(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + path := filepath.Join(dir, "docker") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake docker: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func writeRegularFile(t *testing.T, path string) { + t.Helper() + if err := os.WriteFile(path, []byte("fake db"), 0o644); err != nil { + t.Fatalf("write file %s: %v", path, err) + } +} + +// failDockerFallback returns a fallback that fails the test if it is ever +// called, for asserting the physical-path branch short-circuits correctly. +func failDockerFallback(t *testing.T) func() (string, error) { + t.Helper() + return func() (string, error) { + t.Fatal("docker fallback should not have been called") + return "", nil + } +} + +// ---- validateDBFile -------------------------------------------------- + +func TestValidateDBFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + regular := filepath.Join(dir, "pihole-FTL.db") + writeRegularFile(t, regular) + + if err := validateDBFile(regular); err != nil { + t.Fatalf("validateDBFile(regular file) error = %v, want nil", err) + } + + if err := validateDBFile(dir); err == nil { + t.Fatal("validateDBFile(directory) error = nil, want error") + } else if !strings.Contains(err.Error(), "not a regular file") { + t.Fatalf("validateDBFile(directory) error = %q, want it to mention 'not a regular file'", err) + } + + missing := filepath.Join(dir, "nope.db") + if err := validateDBFile(missing); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("validateDBFile(missing) error = %v, want os.ErrNotExist", err) + } +} + +// ---- findDBPath (physical vs. Docker fallback) ------------------------- + +func TestFindDBPathPhysicalFound(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + physical := filepath.Join(dir, "pihole-FTL.db") + writeRegularFile(t, physical) + + got, err := findDBPath(physical, failDockerFallback(t)) + if err != nil { + t.Fatalf("findDBPath() error = %v", err) + } + if got != physical { + t.Fatalf("findDBPath() = %q, want %q", got, physical) + } +} + +func TestFindDBPathPhysicalIsDirectoryShortCircuits(t *testing.T) { + t.Parallel() + + dir := t.TempDir() // a directory, not a regular file + + _, err := findDBPath(dir, failDockerFallback(t)) + if err == nil { + t.Fatal("findDBPath() error = nil, want error") + } + if !strings.Contains(err.Error(), "failed to access physical pi-hole DB") { + t.Fatalf("findDBPath() error = %q, want it to mention accessing the physical DB", err) + } +} + +func TestFindDBPathFallsBackToDockerWhenPhysicalMissing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + missingPhysical := filepath.Join(dir, "nope.db") + dockerPath := filepath.Join(dir, "docker-mounted.db") + + got, err := findDBPath(missingPhysical, func() (string, error) { + return dockerPath, nil + }) + if err != nil { + t.Fatalf("findDBPath() error = %v", err) + } + if got != dockerPath { + t.Fatalf("findDBPath() = %q, want %q", got, dockerPath) + } +} + +func TestFindDBPathBothMissing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + missingPhysical := filepath.Join(dir, "nope.db") + + _, err := findDBPath(missingPhysical, func() (string, error) { + return "", errors.New("no running pi-hole container found") + }) + if err == nil { + t.Fatal("findDBPath() error = nil, want error") + } + if !strings.Contains(err.Error(), "pi-hole not found") { + t.Fatalf("findDBPath() error = %q, want it to mention 'pi-hole not found'", err) + } +} + +// ---- findPiholeContainer ------------------------------------------------ + +func TestFindPiholeContainerMatchesByName(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"abc123\tmy-pihole-1\tsomeimage:latest\" ;;\nesac\n") + + id, err := findPiholeContainer() + if err != nil { + t.Fatalf("findPiholeContainer() error = %v", err) + } + if id != "abc123" { + t.Fatalf("findPiholeContainer() = %q, want %q", id, "abc123") + } +} + +func TestFindPiholeContainerMatchesByImage(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"def456\tunrelated-name\tpihole/pihole:2024.07.0\" ;;\nesac\n") + + id, err := findPiholeContainer() + if err != nil { + t.Fatalf("findPiholeContainer() error = %v", err) + } + if id != "def456" { + t.Fatalf("findPiholeContainer() = %q, want %q", id, "def456") + } +} + +func TestFindPiholeContainerCaseInsensitive(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"ghi789\tMyPiHoleBox\tSomeImage\" ;;\nesac\n") + + id, err := findPiholeContainer() + if err != nil { + t.Fatalf("findPiholeContainer() error = %v", err) + } + if id != "ghi789" { + t.Fatalf("findPiholeContainer() = %q, want %q", id, "ghi789") + } +} + +func TestFindPiholeContainerNoneFound(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"xyz999\tunrelated\tother/image\" ;;\nesac\n") + + _, err := findPiholeContainer() + if err == nil { + t.Fatal("findPiholeContainer() error = nil, want error") + } + if !strings.Contains(err.Error(), "no running pi-hole container found") { + t.Fatalf("findPiholeContainer() error = %q, want it to mention no container found", err) + } +} + +func TestFindPiholeContainerDockerNotRunning(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\necho \"cannot connect to the docker daemon\" >&2\nexit 1\n") + + _, err := findPiholeContainer() + if err == nil { + t.Fatal("findPiholeContainer() error = nil, want error") + } + if !strings.Contains(err.Error(), "docker ps failed") { + t.Fatalf("findPiholeContainer() error = %q, want it to mention 'docker ps failed'", err) + } +} + +// ---- findDockerDBPath ------------------------------------------------- + +func TestFindDockerDBPathSuccess(t *testing.T) { + mountDir := t.TempDir() + writeRegularFile(t, filepath.Join(mountDir, ftlDBFilename)) + + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo '[{\"Source\":\""+mountDir+"\",\"Destination\":\"/etc/pihole\"}]' ;;\n"+ + "esac\n") + + got, err := findDockerDBPath() + if err != nil { + t.Fatalf("findDockerDBPath() error = %v", err) + } + want := filepath.Join(mountDir, ftlDBFilename) + if got != want { + t.Fatalf("findDockerDBPath() = %q, want %q", got, want) + } +} + +func TestFindDockerDBPathNoPiholeMount(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo '[{\"Source\":\"/some/other\",\"Destination\":\"/var/lib/other\"}]' ;;\n"+ + "esac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + if !strings.Contains(err.Error(), "no /etc/pihole mount detected") { + t.Fatalf("findDockerDBPath() error = %q, want it to mention no mount detected", err) + } +} + +func TestFindDockerDBPathMountedButDBMissing(t *testing.T) { + mountDir := t.TempDir() // no pihole-FTL.db written inside + + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo '[{\"Source\":\""+mountDir+"\",\"Destination\":\"/etc/pihole\"}]' ;;\n"+ + "esac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + if !strings.Contains(err.Error(), "DB not available at expected path") { + t.Fatalf("findDockerDBPath() error = %q, want it to mention DB not available", err) + } +} + +func TestFindDockerDBPathContainerNotFound(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"cid1\tunrelated\tother/image\" ;;\nesac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + if !strings.Contains(err.Error(), "no running pi-hole container found") { + t.Fatalf("findDockerDBPath() error = %q, want it to propagate the container-not-found error", err) + } +} + +func TestFindDockerDBPathInspectFails(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo \"no such container\" >&2; exit 1 ;;\n"+ + "esac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + if !strings.Contains(err.Error(), "docker inspect failed") { + t.Fatalf("findDockerDBPath() error = %q, want it to mention 'docker inspect failed'", err) + } +} + +func TestFindDockerDBPathMalformedInspectJSON(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo \"not json at all\" ;;\n"+ + "esac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + if !strings.Contains(err.Error(), "failed to parse docker inspect output") { + t.Fatalf("findDockerDBPath() error = %q, want it to mention JSON parse failure", err) + } +} From 6f21df66b85561d894da0e1de7d17a1c0abba5c6 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 21 Sep 2026 19:09:00 +0600 Subject: [PATCH 15/56] fix: enforcer.go fails to vet/build on windows (undefined: syscall.Kill) syscall.Kill is POSIX-only and doesn't exist in Go's windows syscall package, breaking `go vet`/`go build` for windows/amd64 and windows/arm64 in CI. Split the platform-specific signal-sending out of KillProcess into terminate(pid, program), implemented in enforcer_unix.go (the real SIGTERM/grace-period/SIGKILL logic, behind //go:build !windows) and enforcer_windows.go (a stub returning "not supported", behind //go:build windows). The portable guardrail checks in KillProcess itself are unchanged. Verified go vet and go build succeed for all six OS/arch combinations CI checks. --- internal/enforcer/enforcer.go | 28 +++----------------- internal/enforcer/enforcer_unix.go | 37 +++++++++++++++++++++++++++ internal/enforcer/enforcer_windows.go | 13 ++++++++++ 3 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 internal/enforcer/enforcer_unix.go create mode 100644 internal/enforcer/enforcer_windows.go diff --git a/internal/enforcer/enforcer.go b/internal/enforcer/enforcer.go index ae77a25..3eabb79 100644 --- a/internal/enforcer/enforcer.go +++ b/internal/enforcer/enforcer.go @@ -12,8 +12,6 @@ import ( "os" "strconv" "strings" - "syscall" - "time" ) // protectedPrograms is the default set of programs that must never be killed. @@ -55,6 +53,9 @@ func isProtected(name string) bool { // - Refuses any PID <= 0. // - Refuses protected programs (case-insensitive match). // - Sends SIGTERM first, waits a grace period, then SIGKILL if still alive. +// +// The actual signal-sending is platform-specific; see terminate in +// enforcer_unix.go / enforcer_windows.go. func KillProcess(pid int, program string, remoteAddr string) error { if pid <= 0 { return fmt.Errorf("invalid PID %d", pid) @@ -74,28 +75,7 @@ func KillProcess(pid int, program string, remoteAddr string) error { slog.Warn("killing connection", "pid", pid, "program", program, "address", remoteAddr) - // SIGTERM first — allows graceful shutdown. - if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { - return fmt.Errorf("send SIGTERM to PID %d: %w", pid, err) - } - - // Grace period: wait up to 2 seconds for the process to exit. - time.Sleep(2 * time.Second) - - // Check whether the process still exists. - if err := syscall.Kill(pid, 0); err != nil { - // Process is gone — SIGTERM worked. - slog.Info("process terminated by SIGTERM", "pid", pid, "program", program) - return nil - } - - // Process still alive — escalate to SIGKILL. - slog.Warn("process survived SIGTERM, sending SIGKILL", "pid", pid, "program", program) - if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { - return fmt.Errorf("send SIGKILL to PID %d: %w", pid, err) - } - - return nil + return terminate(pid, program) } // ParsePID converts a string PID to int, returning an error for invalid input. diff --git a/internal/enforcer/enforcer_unix.go b/internal/enforcer/enforcer_unix.go new file mode 100644 index 0000000..2f3f764 --- /dev/null +++ b/internal/enforcer/enforcer_unix.go @@ -0,0 +1,37 @@ +//go:build !windows + +package enforcer + +import ( + "fmt" + "log/slog" + "syscall" + "time" +) + +// terminate sends SIGTERM, waits a grace period, then escalates to SIGKILL +// if the process is still alive. +func terminate(pid int, program string) error { + // SIGTERM first — allows graceful shutdown. + if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { + return fmt.Errorf("send SIGTERM to PID %d: %w", pid, err) + } + + // Grace period: wait up to 2 seconds for the process to exit. + time.Sleep(2 * time.Second) + + // Check whether the process still exists. + if err := syscall.Kill(pid, 0); err != nil { + // Process is gone — SIGTERM worked. + slog.Info("process terminated by SIGTERM", "pid", pid, "program", program) + return nil + } + + // Process still alive — escalate to SIGKILL. + slog.Warn("process survived SIGTERM, sending SIGKILL", "pid", pid, "program", program) + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + return fmt.Errorf("send SIGKILL to PID %d: %w", pid, err) + } + + return nil +} diff --git a/internal/enforcer/enforcer_windows.go b/internal/enforcer/enforcer_windows.go new file mode 100644 index 0000000..0e116fa --- /dev/null +++ b/internal/enforcer/enforcer_windows.go @@ -0,0 +1,13 @@ +//go:build windows + +package enforcer + +import "fmt" + +// terminate is unimplemented on Windows: the SIGTERM-then-SIGKILL grace +// period this package relies on is a POSIX signal semantic that Windows does +// not have. hexwall does not publish Windows binaries; this stub exists so +// the module vets and builds cleanly cross-platform. +func terminate(pid int, program string) error { + return fmt.Errorf("killing processes is not supported on windows (pid %d, program %q)", pid, program) +} From 5b84bd6a18f4c36651b0be5aabe65aff78bccedc Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 15:36:55 +0600 Subject: [PATCH 16/56] feat(enforcer): kill the single connection via ss -K, not the whole process Resolves the STOPGAP noted in enforcer.go and ARCHITECTURE.md: killing the owning process took down every other connection it held, not just the blocked one. Adds KillConnection(pid, program, localPort, remoteIP, remotePort), which tries ss -K dst dport sport first (CONFIG_INET_DIAG_DESTROY, requires CAP_NET_ADMIN) and falls back to the existing whole-process KillProcess only when that isn't possible. ss -K's own exit code isn't trustworthy (observed exiting 0 even on EPERM), so success is confirmed by re-querying the socket table for the exact tuple afterward, not by trusting ss's return value. Guardrail checks (PID <= 0, PID 1, own PID, protected programs) are shared between both kill paths via checkGuardrails, applied once before either is attempted. monitor.Scanner.applyBlockedOutcome now calls KillConnection instead of KillProcess directly. Verified go vet/go build clean across all six OS/arch combinations CI checks, plus new tests covering KillConnection's guardrails, killSocket against a fake ss on PATH, and socketStillEstablished against a real loopback TCP connection. --- internal/enforcer/enforcer.go | 66 +++++++-- internal/enforcer/enforcer_socket.go | 59 ++++++++ internal/enforcer/enforcer_socket_test.go | 166 ++++++++++++++++++++++ internal/enforcer/enforcer_test.go | 54 +++++++ internal/monitor/monitor.go | 10 +- 5 files changed, 337 insertions(+), 18 deletions(-) create mode 100644 internal/enforcer/enforcer_socket.go create mode 100644 internal/enforcer/enforcer_socket_test.go diff --git a/internal/enforcer/enforcer.go b/internal/enforcer/enforcer.go index 3eabb79..3fc0838 100644 --- a/internal/enforcer/enforcer.go +++ b/internal/enforcer/enforcer.go @@ -1,9 +1,12 @@ -// Package enforcer terminates processes that hold blocked connections. +// Package enforcer terminates blocked connections, preferring to kill just the one +// connection over the whole owning process. // -// STOPGAP: this is deliberately coarser than intended. The goal is to terminate a single -// connection; killing the owning process takes down every other connection it holds too. -// It exists because somo offers no non-interactive per-connection kill. Replace with -// `ss -K dst dport ` (requires CONFIG_INET_DIAG_DESTROY) when that work is done. +// KillConnection is the preferred entry point: it tries a precise, connection-level +// kill via `ss -K` (requires CONFIG_INET_DIAG_DESTROY and CAP_NET_ADMIN) and falls +// back to killing the whole process — KillProcess's behavior — when that isn't +// possible: ss is missing, the kernel doesn't support it, or hexwall isn't running +// with the needed capability. The process-level fallback remains coarser than +// intended: it takes down every other connection that process holds too. package enforcer import ( @@ -45,18 +48,13 @@ func isProtected(name string) bool { return protectedPrograms[lower] } -// KillProcess terminates the process holding a connection. -// -// Guardrails: +// checkGuardrails applies the safety checks shared by both kill paths below, +// before either one is attempted: +// - Refuses any PID <= 0. // - Refuses PID 1 (init). // - Refuses hexwall's own PID. -// - Refuses any PID <= 0. // - Refuses protected programs (case-insensitive match). -// - Sends SIGTERM first, waits a grace period, then SIGKILL if still alive. -// -// The actual signal-sending is platform-specific; see terminate in -// enforcer_unix.go / enforcer_windows.go. -func KillProcess(pid int, program string, remoteAddr string) error { +func checkGuardrails(pid int, program string) error { if pid <= 0 { return fmt.Errorf("invalid PID %d", pid) } @@ -73,6 +71,46 @@ func KillProcess(pid int, program string, remoteAddr string) error { return fmt.Errorf("refusing to kill protected program %q (PID %d)", program, pid) } + return nil +} + +// KillProcess terminates the process holding a connection. +// +// Guardrails: see checkGuardrails. +// Sends SIGTERM first, waits a grace period, then SIGKILL if still alive. +// The actual signal-sending is platform-specific; see terminate in +// enforcer_unix.go / enforcer_windows.go. +func KillProcess(pid int, program string, remoteAddr string) error { + if err := checkGuardrails(pid, program); err != nil { + return err + } + + slog.Warn("killing connection", "pid", pid, "program", program, "address", remoteAddr) + + return terminate(pid, program) +} + +// KillConnection kills the single connection identified by localPort/remoteIP/remotePort, +// owned by pid/program. It prefers a precise, connection-level kill (see killSocket in +// enforcer_socket.go) that leaves every other connection the process holds untouched, +// falling back to killing the whole process only when that isn't possible. +// +// Guardrails are the same as KillProcess's, checked once before either kill path +// is attempted. +func KillConnection(pid int, program, localPort, remoteIP, remotePort string) error { + if err := checkGuardrails(pid, program); err != nil { + return err + } + + remoteAddr := remoteIP + ":" + remotePort + + socketErr := killSocket(localPort, remoteIP, remotePort) + if socketErr == nil { + slog.Info("killed single connection", "pid", pid, "program", program, "local_port", localPort, "remote", remoteAddr) + return nil + } + slog.Debug("connection-level kill unavailable, falling back to killing the process", "err", socketErr, "pid", pid, "program", program) + slog.Warn("killing connection", "pid", pid, "program", program, "address", remoteAddr) return terminate(pid, program) diff --git a/internal/enforcer/enforcer_socket.go b/internal/enforcer/enforcer_socket.go new file mode 100644 index 0000000..c5e46e7 --- /dev/null +++ b/internal/enforcer/enforcer_socket.go @@ -0,0 +1,59 @@ +package enforcer + +import ( + "fmt" + "os/exec" + "strings" +) + +// killSocket attempts to close exactly one TCP connection — identified by its +// local port and the remote IP:port it is talking to — via `ss -K`, which asks +// the kernel to destroy that single socket (CONFIG_INET_DIAG_DESTROY) without +// touching any other connection the owning process holds. +// +// ss -K's own exit code is not a reliable success signal: it has been observed +// to exit 0 even when the kernel refused the destroy request for lack of +// CAP_NET_ADMIN. So after running it, the socket table is re-queried for this +// exact tuple; if it is still there, the kill is treated as failed regardless +// of what ss itself reported. +// +// This needs no build tag: ss is a Linux-only tool (iproute2), but on any +// platform or environment where it is missing, unsupported, or unprivileged, +// this simply returns an error, and the caller falls back to killing the +// whole process. +func killSocket(localPort, remoteIP, remotePort string) error { + // Best-effort: its exit code is not trusted. The recheck below is the + // actual source of truth. + _ = exec.Command("ss", "-K", "dst", remoteIP, "dport", remotePort, "sport", localPort).Run() + + stillEstablished, err := socketStillEstablished(localPort, remoteIP, remotePort) + if err != nil { + return fmt.Errorf("could not verify socket state after ss -K: %w", err) + } + if stillEstablished { + return fmt.Errorf("ss -K did not close the connection (missing CAP_NET_ADMIN, unsupported kernel, or ss unavailable)") + } + + return nil +} + +// socketStillEstablished reports whether a TCP connection matching this exact +// local-port/remote-IP/remote-port tuple is still in the ESTABLISHED state. +func socketStillEstablished(localPort, remoteIP, remotePort string) (bool, error) { + filter := fmt.Sprintf("( sport = :%s and dst %s and dport = %s )", localPort, remoteIP, remotePort) + + out, err := exec.Command("ss", "-tn", "state", "established", filter).Output() + if err != nil { + return false, fmt.Errorf("ss state query failed: %w", err) + } + + dataLines := 0 + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if strings.TrimSpace(line) != "" { + dataLines++ + } + } + + // The first line is always the column header; anything past that is a match. + return dataLines > 1, nil +} diff --git a/internal/enforcer/enforcer_socket_test.go b/internal/enforcer/enforcer_socket_test.go new file mode 100644 index 0000000..d5d3a86 --- /dev/null +++ b/internal/enforcer/enforcer_socket_test.go @@ -0,0 +1,166 @@ +package enforcer + +import ( + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// installFakeSS writes an executable shell script named "ss" into a temp +// directory and prepends it to PATH, mirroring the same trick used for somo +// and docker elsewhere in this project. +func installFakeSS(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + path := filepath.Join(dir, "ss") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake ss: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +// ---- killSocket, via a fake ss on PATH ---------------------------------- + +func TestKillSocketSucceedsWhenConnectionGoneAfterward(t *testing.T) { + installFakeSS(t, "#!/bin/sh\n"+ + "case \"$1\" in\n"+ + " -K) exit 0 ;;\n"+ + " -tn) echo \"Recv-Q Send-Q Local Address:Port Peer Address:Port\" ;;\n"+ // header only: gone + "esac\n") + + if err := killSocket("51820", "203.0.113.10", "443"); err != nil { + t.Fatalf("killSocket() error = %v, want nil when the recheck shows the socket gone", err) + } +} + +func TestKillSocketFailsWhenConnectionStillThere(t *testing.T) { + installFakeSS(t, "#!/bin/sh\n"+ + "case \"$1\" in\n"+ + " -K) exit 0 ;;\n"+ // ss -K can exit 0 even when it changed nothing (e.g. EPERM) + " -tn) printf \"Recv-Q Send-Q Local Address:Port Peer Address:Port\\n0 0 10.0.0.5:51820 203.0.113.10:443\\n\" ;;\n"+ + "esac\n") + + err := killSocket("51820", "203.0.113.10", "443") + if err == nil { + t.Fatal("killSocket() error = nil, want error when the recheck still shows the connection") + } + if !strings.Contains(err.Error(), "did not close") { + t.Fatalf("killSocket() error = %q, want it to mention the connection was not closed", err) + } +} + +func TestKillSocketFailsWhenSSUnavailable(t *testing.T) { + // Empty PATH: exec.Command("ss", ...) can't find the binary at all, + // simulating a platform (e.g. macOS) where ss doesn't exist. + t.Setenv("PATH", t.TempDir()) + + err := killSocket("51820", "203.0.113.10", "443") + if err == nil { + t.Fatal("killSocket() error = nil, want error when ss is not on PATH") + } +} + +// ---- socketStillEstablished, against a real loopback connection --------- + +func requireSS(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("ss"); err != nil { + t.Skip("ss not available in this environment") + } +} + +// openLoopbackConnection sets up a real, live TCP connection over loopback +// and returns the client side plus its own and the server's port, so +// socketStillEstablished can be checked against a socket that genuinely +// exists rather than a fabricated tuple. +func openLoopbackConnection(t *testing.T) (client net.Conn, clientPort, serverPort string) { + t.Helper() + + ln, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen() error = %v", err) + } + t.Cleanup(func() { _ = ln.Close() }) + + accepted := make(chan net.Conn, 1) + go func() { + c, err := ln.Accept() + if err == nil { + accepted <- c + } + }() + + conn, err := net.Dial("tcp4", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial() error = %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + + server := <-accepted + t.Cleanup(func() { _ = server.Close() }) + + local := conn.LocalAddr().(*net.TCPAddr) + remote := conn.RemoteAddr().(*net.TCPAddr) + + return conn, strconv.Itoa(local.Port), strconv.Itoa(remote.Port) +} + +func TestSocketStillEstablishedTrueForLiveConnection(t *testing.T) { + requireSS(t) + + _, clientPort, serverPort := openLoopbackConnection(t) + + still, err := socketStillEstablished(clientPort, "127.0.0.1", serverPort) + if err != nil { + t.Fatalf("socketStillEstablished() error = %v", err) + } + if !still { + t.Fatal("socketStillEstablished() = false, want true for a live connection") + } +} + +func TestSocketStillEstablishedFalseAfterClose(t *testing.T) { + requireSS(t) + + conn, clientPort, serverPort := openLoopbackConnection(t) + if err := conn.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + // Closing the local side takes the socket out of ESTABLISHED essentially + // immediately, but allow a couple of short retries to avoid flakiness. + var still bool + var err error + for i := 0; i < 10; i++ { + still, err = socketStillEstablished(clientPort, "127.0.0.1", serverPort) + if err != nil { + t.Fatalf("socketStillEstablished() error = %v", err) + } + if !still { + break + } + time.Sleep(20 * time.Millisecond) + } + if still { + t.Fatal("socketStillEstablished() = true, want false after the connection was closed") + } +} + +func TestSocketStillEstablishedNoMatch(t *testing.T) { + requireSS(t) + + still, err := socketStillEstablished("1", "203.0.113.99", "1") + if err != nil { + t.Fatalf("socketStillEstablished() error = %v", err) + } + if still { + t.Fatal("socketStillEstablished() = true, want false for a tuple that was never connected") + } +} diff --git a/internal/enforcer/enforcer_test.go b/internal/enforcer/enforcer_test.go index 16fe2bd..a6ba22b 100644 --- a/internal/enforcer/enforcer_test.go +++ b/internal/enforcer/enforcer_test.go @@ -81,6 +81,60 @@ func TestKillProcessCaseInsensitiveProtectedMatch(t *testing.T) { } } +// ---- KillConnection guardrails (same checkGuardrails as KillProcess) ----- + +func TestKillConnectionRefusesPIDZero(t *testing.T) { + t.Parallel() + + err := KillConnection(0, "curl", "51820", "1.2.3.4", "443") + if err == nil { + t.Fatal("KillConnection(0) = nil, want error") + } +} + +func TestKillConnectionRefusesNegativePID(t *testing.T) { + t.Parallel() + + err := KillConnection(-1, "curl", "51820", "1.2.3.4", "443") + if err == nil { + t.Fatal("KillConnection(-1) = nil, want error") + } +} + +func TestKillConnectionRefusesPIDOne(t *testing.T) { + t.Parallel() + + err := KillConnection(1, "systemd", "51820", "1.2.3.4", "443") + if err == nil { + t.Fatal("KillConnection(1) = nil, want error for init") + } +} + +func TestKillConnectionRefusesOwnPID(t *testing.T) { + t.Parallel() + + err := KillConnection(os.Getpid(), "test", "51820", "1.2.3.4", "443") + if err == nil { + t.Fatal("KillConnection(own PID) = nil, want error") + } +} + +func TestKillConnectionRefusesProtectedPrograms(t *testing.T) { + t.Parallel() + + protected := []string{"sshd", "dockerd", "systemd-resolved"} + for _, prog := range protected { + err := KillConnection(2, prog, "51820", "1.2.3.4", "443") + if err == nil { + t.Errorf("KillConnection(2, %q) = nil, want error for protected program", prog) + continue + } + if !strings.Contains(err.Error(), "protected") { + t.Errorf("KillConnection(2, %q) error = %q, want it to mention 'protected'", prog, err) + } + } +} + func TestParsePIDValid(t *testing.T) { t.Parallel() diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index e544ab2..b7f3e13 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -427,15 +427,17 @@ func (s *Scanner) applyBlockedOutcome(ipStr string, conn somo.Connection) { slog.Error("failed to log kill", "address", conn.RAddress, "err", err) } - // Connection-level termination via `ss -K` (requires CONFIG_INET_DIAG_DESTROY) - // is deferred work. KillProcess terminates the owning process, which is coarser. pidInt, err := enforcer.ParsePID(conn.PID) if err != nil { slog.Warn("invalid PID, skipping kill", "pid", conn.PID, "address", conn.RAddress, "err", err) return } - if err := enforcer.KillProcess(pidInt, conn.Program, conn.RAddress); err != nil { - slog.Warn("failed to kill process", "address", conn.RAddress, "pid", conn.PID, "err", err) + + // Prefers a connection-level kill (ss -K) so any other connection the same + // process holds is left untouched; falls back to killing the whole process + // when that isn't possible. See enforcer.KillConnection. + if err := enforcer.KillConnection(pidInt, conn.Program, conn.LPort, ipStr, conn.RPort); err != nil { + slog.Warn("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) } else { slog.Info("killed connection", "address", conn.RAddress) } From 7079e87555d13ca95378503ab26a2d8f41a8487d Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:38:09 +0600 Subject: [PATCH 17/56] ci: drop Windows build checks --- .github/workflows/go.yml | 30 ++++++------------- .../{somo/somo.go => connlist/connlist.go} | 0 .../connlist_test.go} | 0 3 files changed, 9 insertions(+), 21 deletions(-) rename internal/{somo/somo.go => connlist/connlist.go} (100%) rename internal/{somo/somo_test.go => connlist/connlist_test.go} (100%) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4aa85b5..f344004 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -3,16 +3,16 @@ name: Go on: push: paths: - - '**/*.go' - - 'go.mod' - - 'go.sum' - - '**/*.yml' + - "**/*.go" + - "go.mod" + - "go.sum" + - "**/*.yml" pull_request: paths: - - '**/*.go' - - 'go.mod' - - 'go.sum' - - '**/*.yml' + - "**/*.go" + - "go.mod" + - "go.sum" + - "**/*.yml" create: permissions: @@ -48,38 +48,26 @@ jobs: - name: Examine source code for MacOS AMD run: GOOS=darwin GOARCH=amd64 go vet -v ./... - - name: Examine source code for Windows AMD - run: GOOS=windows GOARCH=amd64 go vet -v ./... - - name: Examine source code for Linux ARM run: GOOS=linux GOARCH=arm64 go vet -v ./... - name: Examine source code for MacOS ARM run: GOOS=darwin GOARCH=arm64 go vet -v ./... - - name: Examine source code for Windows ARM - run: GOOS=windows GOARCH=arm64 go vet -v ./... - - name: Build for Linux AMD run: GOOS=linux GOARCH=amd64 go build -v ./... - name: Build for MacOS AMD run: GOOS=darwin GOARCH=amd64 go build -v ./... - - name: Build for Windows AMD - run: GOOS=windows GOARCH=amd64 go build -v ./... - - name: Build for Linux ARM run: GOOS=linux GOARCH=arm64 go build -v ./... - name: Build for MacOS ARM run: GOOS=darwin GOARCH=arm64 go build -v ./... - - name: Build for Windows ARM - run: GOOS=windows GOARCH=arm64 go build -v ./... - - name: Test - # https://stackoverflow.com/questions/58139406/only-run-job-on-specific-branch-with-github-actions + # https://stackoverflow.com/questions/58139406/only-run-job-on-specific-branch-with-github-actions if: github.event_name == 'push' && github.ref == 'refs/heads/main' run: go test -race -v -cover -coverpkg=./... -coverprofile=coverage.txt -covermode=atomic ./... diff --git a/internal/somo/somo.go b/internal/connlist/connlist.go similarity index 100% rename from internal/somo/somo.go rename to internal/connlist/connlist.go diff --git a/internal/somo/somo_test.go b/internal/connlist/connlist_test.go similarity index 100% rename from internal/somo/somo_test.go rename to internal/connlist/connlist_test.go From 1b5b97fc441db0ee3ad5795ebfe3bba25eaac6b9 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:38:09 +0600 Subject: [PATCH 18/56] docs: document connection listing tools --- README.md | 61 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 8c24574..099baf4 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,14 @@ In short: `Pi-hole + somo + hexwall` gives you a practical containment layer for ## At a glance -| Situation | Server without Pi-hole DNS | Server with only Pi-hole | Server with Pi-hole + somo + hexwall | -| --- | --- | --- | --- | -| Malicious domain resolved over DNS | ❌ Exposed | ✅ Can block at DNS layer | ✅ Can block at DNS layer | -| Malware connects to a hard-coded IP directly | ❌ Exposed | ❌ Pi-hole does not see it | ✅ Direct-IP connection is visible and checked | -| Identify which process owns the connection | ❌ No built-in visibility | ❌ No process visibility | ✅ `somo` shows PID/program | -| Stop outbound exfiltration after compromise | ❌ No containment layer | ⚠️ Only if the attacker still uses DNS | ✅ Can kill suspicious established connections | -| Supply-chain malware bypassing DNS | ❌ High risk | ❌ Still exposed | ✅ Stronger containment | -| Overall outbound protection posture | ❌ Weak | ⚠️ Partial | 🛡️ Strongest of the three | +| Situation | Server without Pi-hole DNS | Server with only Pi-hole | Server with Pi-hole + somo + hexwall | +| -------------------------------------------- | -------------------------- | ------------------------------------- | --------------------------------------------- | +| Malicious domain resolved over DNS | ❌ Exposed | ✅ Can block at DNS layer | ✅ Can block at DNS layer | +| Malware connects to a hard-coded IP directly | ❌ Exposed | ❌ Pi-hole does not see it | ✅ Direct-IP connection is visible and checked | +| Identify which process owns the connection | ❌ No built-in visibility | ❌ No process visibility | ✅ `somo` shows PID/program | +| Stop outbound exfiltration after compromise | ❌ No containment layer | ⚠️ Only if the attacker still uses DNS | ✅ Can kill suspicious established connections | +| Supply-chain malware bypassing DNS | ❌ High risk | ❌ Still exposed | ✅ Stronger containment | +| Overall outbound protection posture | ❌ Weak | ⚠️ Partial | 🛡️ Strongest of the three | --- @@ -39,12 +39,13 @@ When Zeek is enabled (`--zeek-log`), hexwall also captures the TLS SNI for each ## Dependencies -| Dependency | Role | Required | -| ----------------------------------------- | ----------------------------------------------------------- | ------------------------------ | -| [`somo`](https://github.com/theopfr/somo) | Lists established TCP/UDP connections; kills by PID | Must be in `$PATH` | -| Pi-hole FTL database | Source of trusted DNS history and allow/deny policy lists | Auto-detected or via `--db` | -| `deghostapi.hexbyte.dev` | Fraud/threat classification for unrecognized IPs and domains | Optional — see `--enable-deghost-*` below | -| Zeek | Passive capture of TLS SNI (`ssl.log`) and DNS-bypass notices (`notice.log`) | Optional — see `--zeek-*`/`--enable-zeek` below | +| Dependency | Role | Required | +| ------------------------------------------------------------ | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| [`ss`](https://wiki.linuxfoundation.org/networking/iproute2) | Lists established TCP connections; kills a single connection via `ss -K` | Default — part of iproute2, present on virtually all Linux systems | +| [`somo`](https://github.com/theopfr/somo) | Fallback connection lister when `ss` is unavailable; kills by PID | Fallback — only required if `ss` (iproute2) is unavailable, e.g. on macOS | +| Pi-hole FTL database | Source of trusted DNS history and allow/deny policy lists | Auto-detected or via `--db` | +| `deghostapi.hexbyte.dev` | Fraud/threat classification for unrecognized IPs and domains | Optional — see `--enable-deghost-*` below | +| Zeek | Passive capture of TLS SNI (`ssl.log`) and DNS-bypass notices (`notice.log`) | Optional — see `--zeek-*`/`--enable-zeek` below | **Build dependency:** `modernc.org/sqlite` (pure-Go SQLite, no CGo or system libsqlite3 required). @@ -61,10 +62,16 @@ You can download them from the project's GitHub Releases page: For production Linux servers, download the matching binary from Releases, place it where your deployment expects `./app/hexwall`, and use the included demo Docker Compose file at [`docker-compose.yml`](./docker-compose.yml) to launch it on the server. -For macOS and Windows, prebuilt binaries are not published by CI. Clone the repository and build the binary manually before deployment. +hexwall targets Linux, where prebuilt binaries are published by CI. macOS is best-effort: clone the repository and build the binary manually, and install `somo` as a fallback since `ss` (Linux's iproute2) isn't available there. Windows is not supported. + +### Connection backend + +At startup, hexwall picks a connection-listing backend by availability, not by OS: `ss` first if it's on `PATH`, then `somo`, then it exits with an install message if neither is found. This means `ss` is used by default on real Linux boxes (iproute2 ships with virtually every distro) and `somo` only comes into play as a fallback — on macOS, where `ss` doesn't exist, or on a stripped-down Linux system without iproute2 installed. The connection-kill path already tries `ss -K` first and falls back to a process-level signal kill regardless of which lister found the connection, so no behavior changes based on which backend is selected. ### Install `somo` +The steps below are only needed on the fallback path described above (macOS, or a Linux box without `ss`). + If `somo` is not already in your `$PATH`, this is the quickest way to install and verify it on Ubuntu or Debian: ```sh @@ -98,18 +105,18 @@ If neither is found, start fails. Use `--db` to specify the path manually. ## CLI flags -| Flag | Default | Description | -| ------------------------- | ------------------------------------- | ---------------------------------------------------------------------------- | -| `--db` | _(auto-detected)_ | Path to `pihole-FTL.db` | -| `--hexwall-db` | `./hexwall.db` | Path to the local hexwall database (created on first run) | -| `--mode` | `watch` | `watch` — detect and log only; `enforce` — detect, log, and kill | -| `--debug` | `false` | Enable verbose per-connection scan logging | -| `--enable-zeek` | `false` | Enable the Zeek `notice.log` DNS-bypass alert path (independent of `--zeek-log`) | -| `--zeek-log` | _(unset — SNI matching skipped)_ | Path to Zeek's `ssl.log`, used to match a connection's TLS SNI against Pi-hole's allow/deny lists | -| `--zeek-notice-log` | `/opt/zeek/logs/current/notice.log` | Path to Zeek's `notice.log`; only read when `--enable-zeek` is set | -| `--enable-deghost-ip` | `true` | Enable third-party IP reputation checks for unrecognized connections with no SNI | -| `--enable-deghost-domain` | `true` | Enable third-party domain reputation checks for unrecognized SNI domains | -| `--version` | `false` | Print version and exit | +| Flag | Default | Description | +| ------------------------- | ----------------------------------- | ------------------------------------------------------------------------------------------------- | +| `--db` | _(auto-detected)_ | Path to `pihole-FTL.db` | +| `--hexwall-db` | `./hexwall.db` | Path to the local hexwall database (created on first run) | +| `--mode` | `watch` | `watch` — detect and log only; `enforce` — detect, log, and kill | +| `--debug` | `false` | Enable verbose per-connection scan logging | +| `--enable-zeek` | `false` | Enable the Zeek `notice.log` DNS-bypass alert path (independent of `--zeek-log`) | +| `--zeek-log` | _(unset — SNI matching skipped)_ | Path to Zeek's `ssl.log`, used to match a connection's TLS SNI against Pi-hole's allow/deny lists | +| `--zeek-notice-log` | `/opt/zeek/logs/current/notice.log` | Path to Zeek's `notice.log`; only read when `--enable-zeek` is set | +| `--enable-deghost-ip` | `true` | Enable third-party IP reputation checks for unrecognized connections with no SNI | +| `--enable-deghost-domain` | `true` | Enable third-party domain reputation checks for unrecognized SNI domains | +| `--version` | `false` | Print version and exit | `--mode` accepts `watch` or `enforce` (case-insensitive). Any other value exits with an error. From bdd0939de237220ced6259bcbb91b28d206e826b Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 19/56] feat: select ss before somo --- internal/connlist/connlist.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/connlist/connlist.go b/internal/connlist/connlist.go index b3bbb94..01fe985 100644 --- a/internal/connlist/connlist.go +++ b/internal/connlist/connlist.go @@ -1,5 +1,6 @@ -// Package somo wraps the somo CLI for monitoring and managing network connections. -package somo +// Package connlist lists established network connections via ss (preferred) +// or somo (fallback) — see SelectBackend. +package connlist import ( "encoding/json" @@ -30,6 +31,22 @@ func CheckInstalled() error { return nil } +// SelectBackend picks a connection-listing tool by availability: ss +// (iproute2) first since it needs no extra install on virtually any Linux +// box, falling back to somo when ss isn't on PATH (e.g. macOS). name is +// "ss" or "somo", for logging which tool is in effect. +func SelectBackend() (lister func() ([]Connection, error), name string, err error) { + if SSAvailable() { + return GetEstablishedConnectionsViaSS, "ss", nil + } + + if CheckInstalled() == nil { + return GetEstablishedConnections, "somo", nil + } + + return nil, "", fmt.Errorf("no connection-listing tool available: install ss (iproute2) or somo (https://github.com/theopfr/somo)") +} + // GetEstablishedConnections returns a list of established TCP/UDP connections. func GetEstablishedConnections() ([]Connection, error) { cmd := exec.Command("somo", "-e", "--json", "--no-pager") From 5dbcc60ac3a73826f6c55abd00c60c3af6305639 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 20/56] test: cover connection backend selection --- internal/connlist/connlist_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/connlist/connlist_test.go b/internal/connlist/connlist_test.go index 0db6450..ef413f3 100644 --- a/internal/connlist/connlist_test.go +++ b/internal/connlist/connlist_test.go @@ -1,4 +1,4 @@ -package somo +package connlist import ( "errors" From 6a80b7718e596f13ba01169f47f4b02bde782406 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 21/56] test: cover backend availability --- internal/connlist/backend_test.go | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 internal/connlist/backend_test.go diff --git a/internal/connlist/backend_test.go b/internal/connlist/backend_test.go new file mode 100644 index 0000000..c2dcba9 --- /dev/null +++ b/internal/connlist/backend_test.go @@ -0,0 +1,59 @@ +package connlist + +import ( + "os" + "path/filepath" + "testing" +) + +// writeFakeBinary writes an executable script into dir under the given name. +func writeFakeBinary(t *testing.T, dir, name, script string) { + t.Helper() + + if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil { + t.Fatalf("write fake %s: %v", name, err) + } +} + +func TestSelectBackend(t *testing.T) { + okScript := "#!/bin/sh\nexit 0\n" + + tests := []struct { + name string + binaries []string // names to place on an otherwise-empty PATH + wantName string + wantErr bool + }{ + {name: "ss present picks ss", binaries: []string{"ss", "somo"}, wantName: "ss"}, + {name: "only somo present picks somo", binaries: []string{"somo"}, wantName: "somo"}, + {name: "neither present fails", binaries: nil, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + for _, bin := range tt.binaries { + writeFakeBinary(t, dir, bin, okScript) + } + t.Setenv("PATH", dir) + + lister, name, err := SelectBackend() + if tt.wantErr { + if err == nil { + t.Fatal("SelectBackend() error = nil, want error") + } + return + } + + if err != nil { + t.Fatalf("SelectBackend() error = %v, want nil", err) + } + if name != tt.wantName { + t.Fatalf("SelectBackend() name = %q, want %q", name, tt.wantName) + } + if lister == nil { + t.Fatal("SelectBackend() lister = nil, want non-nil") + } + }) + } +} From 06eb300ca20f1700686f2dff0fa54d83b037ee61 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 22/56] feat: add ss connection lister --- internal/connlist/ss_lister.go | 92 ++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 internal/connlist/ss_lister.go diff --git a/internal/connlist/ss_lister.go b/internal/connlist/ss_lister.go new file mode 100644 index 0000000..895959c --- /dev/null +++ b/internal/connlist/ss_lister.go @@ -0,0 +1,92 @@ +package connlist + +import ( + "bufio" + "fmt" + "os/exec" + "regexp" + "strings" +) + +// processFieldPattern extracts the first "prog",pid=N entry from ss's +// users:(("prog",pid=N,fd=M),...) process column. Only the first owner is +// kept, matching somo's one-program-per-connection shape. +var processFieldPattern = regexp.MustCompile(`users:\(\("([^"]+)",pid=(\d+),fd=\d+\)`) + +// SSAvailable reports whether the ss CLI (iproute2) is available in PATH. +func SSAvailable() bool { + _, err := exec.LookPath("ss") + return err == nil +} + +// GetEstablishedConnectionsViaSS lists established TCP connections using ss, +// the preferred backend when iproute2 is available (Linux, not macOS). +func GetEstablishedConnectionsViaSS() ([]Connection, error) { + cmd := exec.Command("ss", "-H", "-tnp", "state", "established") + output, err := cmd.CombinedOutput() + if err != nil { + return nil, commandError("failed to run ss", err, output) + } + + return parseSSOutput(string(output)), nil +} + +// parseSSOutput parses `ss -H -tnp state established` output into Connections. +// A malformed line is skipped rather than aborting the whole parse. A +// connection whose owning process ss couldn't identify (no users:(...) field, +// e.g. owned by another user without CAP_NET_ADMIN) yields an empty +// Program/PID, since enforcer.ParsePID already treats "" as invalid. +func parseSSOutput(output string) []Connection { + var connections []Connection + + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 4 { + continue + } + + _, localPort, err := splitSSAddress(fields[2]) + if err != nil { + continue + } + remoteHost, remotePort, err := splitSSAddress(fields[3]) + if err != nil { + continue + } + + conn := Connection{ + Protocol: "tcp", + LPort: localPort, + RAddress: remoteHost, + RPort: remotePort, + State: "ESTAB", + } + + if len(fields) >= 5 { + if m := processFieldPattern.FindStringSubmatch(fields[4]); m != nil { + conn.Program = m[1] + conn.PID = m[2] + } + } + + connections = append(connections, conn) + } + + return connections +} + +// splitSSAddress splits ss's "host:port" (or "[ipv6]:port") address column. +func splitSSAddress(field string) (host, port string, err error) { + idx := strings.LastIndex(field, ":") + if idx < 0 { + return "", "", fmt.Errorf("no port separator in address %q", field) + } + + host, port = field[:idx], field[idx+1:] + if host == "" || port == "" { + return "", "", fmt.Errorf("empty host or port in address %q", field) + } + + return host, port, nil +} From d471ba353a6ea90103c5b66fbec5a1a9f971ce82 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 23/56] test: cover ss connection parsing --- internal/connlist/ss_lister_test.go | 126 ++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 internal/connlist/ss_lister_test.go diff --git a/internal/connlist/ss_lister_test.go b/internal/connlist/ss_lister_test.go new file mode 100644 index 0000000..5d010f0 --- /dev/null +++ b/internal/connlist/ss_lister_test.go @@ -0,0 +1,126 @@ +package connlist + +import ( + "os" + "path/filepath" + "testing" +) + +// installFakeSS writes an executable shell script named "ss" into a temp +// directory and prepends it to PATH, mirroring installFakeSomo below and the +// same trick used in internal/enforcer/enforcer_socket_test.go. +func installFakeSS(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + path := filepath.Join(dir, "ss") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake ss: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +// realSSFixture is real `ss -H -tnp state established` output captured on a +// live Linux box (iproute2-6.1.0), used verbatim so the parser is validated +// against actual formatting rather than assumed grammar. +const realSSFixture = `0 0 192.168.0.223:53404 160.79.104.10:443 users:(("claude",pid=6039,fd=11)) +0 0 192.168.0.223:56926 160.79.104.10:443 users:(("claude-desktop",pid=5831,fd=22)) +0 0 192.168.0.223:41492 140.82.114.25:443 users:(("firefox-bin",pid=4076,fd=52)) +0 0 192.168.0.223:48878 64.233.170.188:5228 users:(("chrome",pid=5068,fd=37)) +` + +func TestParseSSOutputRealFixture(t *testing.T) { + conns := parseSSOutput(realSSFixture) + if len(conns) != 4 { + t.Fatalf("parseSSOutput() len = %d, want 4", len(conns)) + } + + want := Connection{Protocol: "tcp", LPort: "53404", RAddress: "160.79.104.10", RPort: "443", Program: "claude", PID: "6039", State: "ESTAB"} + if conns[0] != want { + t.Fatalf("parseSSOutput()[0] = %+v, want %+v", conns[0], want) + } + + if conns[3].RPort != "5228" || conns[3].RAddress != "64.233.170.188" { + t.Fatalf("parseSSOutput()[3] remote = %s:%s, want 64.233.170.188:5228", conns[3].RAddress, conns[3].RPort) + } +} + +func TestParseSSOutputUnattributedProcess(t *testing.T) { + // ss omits the whole process column when it can't attribute the socket + // to a process (e.g. owned by another user without CAP_NET_ADMIN). + const line = "0 0 192.168.0.223:53404 160.79.104.10:443\n" + + conns := parseSSOutput(line) + if len(conns) != 1 { + t.Fatalf("parseSSOutput() len = %d, want 1", len(conns)) + } + if conns[0].Program != "" || conns[0].PID != "" { + t.Fatalf("parseSSOutput()[0] = %+v, want empty Program/PID", conns[0]) + } + if conns[0].RAddress != "160.79.104.10" || conns[0].RPort != "443" { + t.Fatalf("parseSSOutput()[0] remote = %s:%s, want 160.79.104.10:443", conns[0].RAddress, conns[0].RPort) + } +} + +func TestParseSSOutputMultipleOwners(t *testing.T) { + const line = `0 0 192.168.0.223:53404 160.79.104.10:443 users:(("nginx",pid=100,fd=5),("nginx",pid=100,fd=6)) +` + + conns := parseSSOutput(line) + if len(conns) != 1 { + t.Fatalf("parseSSOutput() len = %d, want 1", len(conns)) + } + if conns[0].Program != "nginx" || conns[0].PID != "100" { + t.Fatalf("parseSSOutput()[0] = %+v, want Program=nginx PID=100", conns[0]) + } +} + +func TestParseSSOutputSkipsMalformedLines(t *testing.T) { + const output = `not even close to a valid line +0 0 192.168.0.223:53404 160.79.104.10:443 users:(("claude",pid=6039,fd=11)) +192.168.0.223:missingport +` + + conns := parseSSOutput(output) + if len(conns) != 1 { + t.Fatalf("parseSSOutput() len = %d, want 1 (malformed lines skipped)", len(conns)) + } + if conns[0].Program != "claude" { + t.Fatalf("parseSSOutput()[0].Program = %q, want claude", conns[0].Program) + } +} + +func TestSSAvailable(t *testing.T) { + installFakeSS(t, "#!/bin/sh\nexit 0\n") + if !SSAvailable() { + t.Fatal("SSAvailable() = false, want true with fake ss on PATH") + } +} + +func TestSSAvailableFalseWhenNotOnPath(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + if SSAvailable() { + t.Fatal("SSAvailable() = true, want false with empty PATH") + } +} + +func TestGetEstablishedConnectionsViaSS(t *testing.T) { + installFakeSS(t, "#!/bin/sh\ncat <<'EOF'\n"+realSSFixture+"EOF\n") + + conns, err := GetEstablishedConnectionsViaSS() + if err != nil { + t.Fatalf("GetEstablishedConnectionsViaSS() error = %v", err) + } + if len(conns) != 4 { + t.Fatalf("GetEstablishedConnectionsViaSS() len = %d, want 4", len(conns)) + } +} + +func TestGetEstablishedConnectionsViaSSCommandFailure(t *testing.T) { + installFakeSS(t, "#!/bin/sh\necho 'ss: permission denied' 1>&2\nexit 1\n") + + if _, err := GetEstablishedConnectionsViaSS(); err == nil { + t.Fatal("GetEstablishedConnectionsViaSS() error = nil, want error when ss fails") + } +} From e092f0f258e9e59c9d8dd12a8594c6d8785d93df Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 24/56] test: update connection tool references --- internal/detector/detector_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/detector/detector_test.go b/internal/detector/detector_test.go index b924c73..74d9e1c 100644 --- a/internal/detector/detector_test.go +++ b/internal/detector/detector_test.go @@ -10,7 +10,7 @@ import ( // installFakeDocker writes an executable shell script named "docker" into a // temp directory and prepends it to PATH, mirroring the same trick used for -// somo elsewhere in this project (see internal/somo/somo_test.go). +// somo elsewhere in this project (see internal/connlist/connlist_test.go). func installFakeDocker(t *testing.T, script string) { t.Helper() From 23419f1c9ae9d4fe7ce442b934f6edb8c89c9e70 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 25/56] docs: mark Windows enforcer stub unsupported --- internal/enforcer/enforcer_windows.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/enforcer/enforcer_windows.go b/internal/enforcer/enforcer_windows.go index 0e116fa..441d6a4 100644 --- a/internal/enforcer/enforcer_windows.go +++ b/internal/enforcer/enforcer_windows.go @@ -6,8 +6,8 @@ import "fmt" // terminate is unimplemented on Windows: the SIGTERM-then-SIGKILL grace // period this package relies on is a POSIX signal semantic that Windows does -// not have. hexwall does not publish Windows binaries; this stub exists so -// the module vets and builds cleanly cross-platform. +// not have. hexwall does not support Windows — CI no longer builds or tests +// this file, so treat it as an unmaintained stub, not a courtesy compile target. func terminate(pid int, program string) error { return fmt.Errorf("killing processes is not supported on windows (pid %d, program %q)", pid, program) } From c3f39d2667ff4e9eeda94406248c806c2acd833f Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 26/56] feat: inject connection lister into monitor --- internal/monitor/monitor.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index b7f3e13..76bcb63 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -11,10 +11,10 @@ import ( "time" "github.com/hexbytedev/hexwall/internal/allowlist" + "github.com/hexbytedev/hexwall/internal/connlist" "github.com/hexbytedev/hexwall/internal/deghost" "github.com/hexbytedev/hexwall/internal/enforcer" "github.com/hexbytedev/hexwall/internal/pihole" - "github.com/hexbytedev/hexwall/internal/somo" "github.com/hexbytedev/hexwall/internal/store" "github.com/hexbytedev/hexwall/internal/zeek" ) @@ -31,14 +31,15 @@ const ( // a plain dependency or flag with nothing to validate, so there's no // NewScanner constructor. type Scanner struct { - Checker *pihole.Checker - Store *store.Store - DeghostClient *deghost.Client - ZeekClient *zeek.Client - Mode string - DeghostIP bool - DeghostDomain bool - Debug bool + Checker *pihole.Checker + Store *store.Store + DeghostClient *deghost.Client + ZeekClient *zeek.Client + ConnectionLister func() ([]connlist.Connection, error) + Mode string + DeghostIP bool + DeghostDomain bool + Debug bool } // scanSummary tracks per-scan outcome counts for the summary line. @@ -106,6 +107,7 @@ func (s *Scanner) HandleZeekEvent(event zeek.Event) { confidence := "medium" actionTaken := "logged" + if blocked { confidence = "high" slog.Warn("zeek bypass alert", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI, "mode", selectedMode) @@ -140,12 +142,17 @@ func (s *Scanner) Run(ctx context.Context) { return } + if s.ConnectionLister == nil { + slog.Warn("connection lister unset; defaulting to somo") + s.ConnectionLister = connlist.GetEstablishedConnections + } + selectedMode := normalizeMode(s.Mode) if selectedMode != s.Mode { slog.Warn("invalid scan mode; defaulting to watch", "mode", s.Mode, "fallback", selectedMode) } - connections, err := somo.GetEstablishedConnections() + connections, err := s.ConnectionLister() if err != nil { slog.Error("error fetching connections", "err", err) return @@ -312,7 +319,7 @@ func (s *Scanner) Run(ctx context.Context) { // evaluateIPOnly handles the IP-only fallback path (rung 7) when no SNI is available. // It returns "trusted", "blocked", "unknown", or "error". // Every exit path records an IP observation before returning. -func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn somo.Connection) string { +func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlist.Connection) string { allowed, err := s.Store.IsAllowed(ipStr) if err != nil { slog.Error("store lookup failed", "address", conn.RAddress, "err", err) @@ -416,7 +423,7 @@ func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn somo.Co } // applyBlockedOutcome enforces the kill/watch policy for a blocked connection. -func (s *Scanner) applyBlockedOutcome(ipStr string, conn somo.Connection) { +func (s *Scanner) applyBlockedOutcome(ipStr string, conn connlist.Connection) { selectedMode := normalizeMode(s.Mode) if selectedMode == ModeWatch { slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) From 0fffa92d7766ed9507a8aa46e4802ec2d979d37a Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 27/56] test: cover monitor connection lister --- internal/monitor/monitor_test.go | 42 ++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index 9693a7c..9322d98 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -12,9 +12,9 @@ import ( "testing" "time" + "github.com/hexbytedev/hexwall/internal/connlist" "github.com/hexbytedev/hexwall/internal/deghost" "github.com/hexbytedev/hexwall/internal/pihole" - "github.com/hexbytedev/hexwall/internal/somo" "github.com/hexbytedev/hexwall/internal/store" "github.com/hexbytedev/hexwall/internal/zeek" @@ -142,7 +142,7 @@ func newTestChecker(t *testing.T, gravitySetup func(db *sql.DB)) *pihole.Checker } // installFakeSomo writes an executable shell script named "somo" into a temp -// directory and prepends it to PATH, mirroring internal/somo/somo_test.go's helper. +// directory and prepends it to PATH, mirroring internal/connlist/connlist_test.go's helper. func installFakeSomo(t *testing.T, script string) { t.Helper() @@ -331,7 +331,7 @@ func TestHandleZeekEventNoopCases(t *testing.T) { func TestApplyBlockedOutcomeWatchModeNeverLogsKill(t *testing.T) { hexwallStore, dbPath := newTestStore(t) - conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} + conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} (&Scanner{Store: hexwallStore, Mode: ModeWatch}).applyBlockedOutcome("203.0.113.10", conn) if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { @@ -344,7 +344,7 @@ func TestApplyBlockedOutcomeEnforceModeLogsKillBeforeAttemptingIt(t *testing.T) // PID 1 (init) is guaranteed to be refused by enforcer.KillProcess, but the // audit row must still be written first ("audit survives the kill"). - conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} + conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} (&Scanner{Store: hexwallStore, Mode: ModeEnforce}).applyBlockedOutcome("203.0.113.10", conn) if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.10"); got != 1 { @@ -355,7 +355,7 @@ func TestApplyBlockedOutcomeEnforceModeLogsKillBeforeAttemptingIt(t *testing.T) func TestApplyBlockedOutcomeInvalidPIDLogsWarningAndReturns(t *testing.T) { hexwallStore, dbPath := newTestStore(t) - conn := somo.Connection{PID: "not-a-pid", Program: "curl", RAddress: "203.0.113.11:443"} + conn := connlist.Connection{PID: "not-a-pid", Program: "curl", RAddress: "203.0.113.11:443"} (&Scanner{Store: hexwallStore, Mode: ModeEnforce}).applyBlockedOutcome("203.0.113.11", conn) // LogKill happens before ParsePID, so the audit row is still written. @@ -375,7 +375,7 @@ func TestEvaluateIPOnlyTrustedIPUpdatesEstablished(t *testing.T) { srv := failIfCalled(t) client := deghost.NewClient(srv.URL, time.Second) - conn := somo.Connection{Program: "curl", RAddress: "203.0.113.20:443"} + conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.20:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} got := scanner.evaluateIPOnly(context.Background(), "203.0.113.20", conn) if got != "trusted" { @@ -392,7 +392,7 @@ func TestEvaluateIPOnlyDeghostDisabledReturnsUnknown(t *testing.T) { srv := failIfCalled(t) client := deghost.NewClient(srv.URL, time.Second) - conn := somo.Connection{Program: "curl", RAddress: "203.0.113.21:443"} + conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.21:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: false, Mode: ModeWatch} got := scanner.evaluateIPOnly(context.Background(), "203.0.113.21", conn) if got != "unknown" { @@ -409,7 +409,7 @@ func TestEvaluateIPOnlyCachedCleanReturnsUnknownNoHTTPCall(t *testing.T) { srv := failIfCalled(t) client := deghost.NewClient(srv.URL, time.Second) - conn := somo.Connection{Program: "curl", RAddress: "203.0.113.22:443"} + conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.22:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} got := scanner.evaluateIPOnly(context.Background(), "203.0.113.22", conn) if got != "unknown" { @@ -427,7 +427,7 @@ func TestEvaluateIPOnlyCachedBlockedReturnsBlockedAndKills(t *testing.T) { client := deghost.NewClient(srv.URL, time.Second) // PID 1 keeps the actual kill attempt safely refused by enforcer guardrails. - conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.23:443"} + conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.23:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeEnforce} got := scanner.evaluateIPOnly(context.Background(), "203.0.113.23", conn) if got != "blocked" { @@ -447,7 +447,7 @@ func TestEvaluateIPOnlyLiveCheckPrivateReservedCachesClean(t *testing.T) { client := deghost.NewClient(srv.URL, time.Second) hexwallStore, _ := newTestStore(t) - conn := somo.Connection{Program: "curl", RAddress: "203.0.113.24:443"} + conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.24:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} got := scanner.evaluateIPOnly(context.Background(), "203.0.113.24", conn) if got != "unknown" { @@ -478,7 +478,7 @@ func TestEvaluateIPOnlyLiveCheckShouldKillCachesAndBlocks(t *testing.T) { client := deghost.NewClient(srv.URL, time.Second) hexwallStore, dbPath := newTestStore(t) - conn := somo.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.25:443"} + conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.25:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeEnforce} got := scanner.evaluateIPOnly(context.Background(), "203.0.113.25", conn) if got != "blocked" { @@ -510,7 +510,19 @@ func TestRunScanZeroConnections(t *testing.T) { hexwallStore, _ := newTestStore(t) // Must not panic even with nil checker/deghost/zeek dependencies. - (&Scanner{Store: hexwallStore, Mode: ModeWatch}).Run(context.Background()) + (&Scanner{Store: hexwallStore, Mode: ModeWatch, ConnectionLister: connlist.GetEstablishedConnections}).Run(context.Background()) +} + +func TestRunScanFallsBackToSomoWhenConnectionListerUnset(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho '[]'\n") + hexwallStore, _ := newTestStore(t) + + // Even when a caller forgets to populate the field, Run should still work. + scanner := &Scanner{Store: hexwallStore, Mode: ModeWatch} + scanner.Run(context.Background()) + if scanner.ConnectionLister == nil { + t.Fatal("ConnectionLister fallback is nil after Run()") + } } func TestRunScanAllowlistedConnectionSkipsIPFallback(t *testing.T) { @@ -519,7 +531,7 @@ echo '[{"proto":"tcp","local_port":"51820","remote_address":"127.0.0.1","remote_ `) hexwallStore, dbPath := newTestStore(t) - (&Scanner{Store: hexwallStore, Mode: ModeWatch}).Run(context.Background()) + (&Scanner{Store: hexwallStore, Mode: ModeWatch, ConnectionLister: connlist.GetEstablishedConnections}).Run(context.Background()) // Rung 1 (static allowlist) short-circuits before the IP-only fallback, so // no ip_observations row should exist for a loopback address. @@ -534,7 +546,7 @@ echo '[{"proto":"tcp","local_port":"1","remote_address":"not-an-ip","remote_port `) hexwallStore, dbPath := newTestStore(t) - (&Scanner{Store: hexwallStore, Mode: ModeWatch}).Run(context.Background()) + (&Scanner{Store: hexwallStore, Mode: ModeWatch, ConnectionLister: connlist.GetEstablishedConnections}).Run(context.Background()) // The second, valid connection must still be processed through the IP-only // fallback despite the first connection's address being unparsable. @@ -549,7 +561,7 @@ echo '[{"proto":"tcp","local_port":"1","remote_address":"203.0.113.31","remote_p `) hexwallStore, dbPath := newTestStore(t) - (&Scanner{Store: hexwallStore, Mode: ModeWatch}).Run(context.Background()) + (&Scanner{Store: hexwallStore, Mode: ModeWatch, ConnectionLister: connlist.GetEstablishedConnections}).Run(context.Background()) if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ? AND outcome = 'unknown'`, "203.0.113.31"); got != 1 { t.Fatalf("ip_observations unknown count = %d, want 1", got) From 472f9cee33f9aaff9db0f45e8092c060d7b36686 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 22 Sep 2026 17:39:10 +0600 Subject: [PATCH 28/56] feat: log OS and selected connection tool --- main.go | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index b071e54..a9d4040 100644 --- a/main.go +++ b/main.go @@ -13,11 +13,11 @@ import ( "syscall" "time" + "github.com/hexbytedev/hexwall/internal/connlist" "github.com/hexbytedev/hexwall/internal/deghost" "github.com/hexbytedev/hexwall/internal/detector" "github.com/hexbytedev/hexwall/internal/monitor" "github.com/hexbytedev/hexwall/internal/pihole" - "github.com/hexbytedev/hexwall/internal/somo" "github.com/hexbytedev/hexwall/internal/store" "github.com/hexbytedev/hexwall/internal/zeek" ) @@ -117,15 +117,24 @@ func run() int { return 0 } + // Windows is not supported at all, checked before any tool-availability probing. + if runtime.GOOS == "windows" { + slog.Error("windows is not supported by hexwall") + return 1 + } + slog.Info("detected OS", "os", runtime.GOOS) + // Cancel background work cleanly on Ctrl+C. ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - // 1. Verify that somo is available before starting any monitoring. - if err := somo.CheckInstalled(); err != nil { - slog.Error("somo is not installed", "err", err, "tip", "install somo from https://github.com/theopfr/somo") + // 1. Pick a connection-listing tool: ss (iproute2) if available, else somo. + connectionLister, toolName, err := connlist.SelectBackend() + if err != nil { + slog.Error("no connection-listing tool available", "err", err) return 1 } + slog.Info("using connection-listing tool", "tool", toolName) // 2. Resolve the Pi-hole database path from --db or auto-detection. if cfg.dbPath == "" { @@ -222,14 +231,15 @@ func run() int { } scanner := &monitor.Scanner{ - Checker: checker, - Store: hexwallStore, - DeghostClient: deghostClient, - ZeekClient: zeekClient, - Mode: cfg.mode, - DeghostIP: cfg.enableDeghostIP, - DeghostDomain: cfg.enableDeghostDomain, - Debug: cfg.debug, + Checker: checker, + Store: hexwallStore, + DeghostClient: deghostClient, + ZeekClient: zeekClient, + ConnectionLister: connectionLister, + Mode: cfg.mode, + DeghostIP: cfg.enableDeghostIP, + DeghostDomain: cfg.enableDeghostDomain, + Debug: cfg.debug, } var zeekEvents = make(chan zeek.Event, 100) From 716877ab908d219be3f81c3d922f7f47b246d705 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Wed, 23 Sep 2026 01:10:53 +0600 Subject: [PATCH 29/56] fix: README still called somo the primary tool; add IPv6 parser test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the ss/somo backend-selection work found two gaps: - README's intro, tagline, and two other prose spots still described somo as THE connection tool, unqualified, even though the newer Dependencies table and "Connection backend" section correctly say ss is now the default. Fixed all five stale spots to match. - The ss output parser had zero test coverage for IPv6. Verified manually against a real IPv6 loopback connection: ss brackets IPv6 addresses ("[::1]:PORT"), and the parser's LastIndex(":") split leaves the brackets in RAddress rather than stripping them. This works today only because monitor.remoteIP() strips brackets itself as a fallback — not a bug, but an untested implicit dependency between two packages. Added a test pinning the current (correct) behavior so a future change to either side can't silently break it. --- README.md | 12 ++++++------ internal/connlist/ss_lister_test.go | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 099baf4..6d6dd19 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,19 @@ [![Dependabot Updates][11]][12] [![Go][13]][14] -`hexwall` is an outbound exfiltration brake for servers already protected by Pi-hole. When a supply-chain attack, malware implant, or compromised package is already running inside the machine, it can skip DNS entirely and send stolen data straight to a hard-coded IP. Pi-hole does not see that hop. `somo` can see the live connection. `hexwall` uses that visibility to flag and, in `enforce` mode, kill suspicious direct-IP connections before data leaves the server. +`hexwall` is an outbound exfiltration brake for servers already protected by Pi-hole. When a supply-chain attack, malware implant, or compromised package is already running inside the machine, it can skip DNS entirely and send stolen data straight to a hard-coded IP. Pi-hole does not see that hop. `ss` (falling back to `somo` when unavailable) can see the live connection. `hexwall` uses that visibility to flag and, in `enforce` mode, kill suspicious direct-IP connections before data leaves the server. It monitors active network connections and kills any that connect to IPs that are not trusted by Pi-hole history, the built-in local allowlist, or recent already-established traffic — closing the gap that Pi-hole leaves open for direct-IP connections. -In short: `Pi-hole + somo + hexwall` gives you a practical containment layer for post-compromise outbound traffic, especially the kind of direct-IP exfiltration occasionally used in supply-chain attacks. +In short: `Pi-hole + hexwall` gives you a practical containment layer for post-compromise outbound traffic, especially the kind of direct-IP exfiltration occasionally used in supply-chain attacks. ## At a glance -| Situation | Server without Pi-hole DNS | Server with only Pi-hole | Server with Pi-hole + somo + hexwall | +| Situation | Server without Pi-hole DNS | Server with only Pi-hole | Server with Pi-hole + hexwall | | -------------------------------------------- | -------------------------- | ------------------------------------- | --------------------------------------------- | | Malicious domain resolved over DNS | ❌ Exposed | ✅ Can block at DNS layer | ✅ Can block at DNS layer | | Malware connects to a hard-coded IP directly | ❌ Exposed | ❌ Pi-hole does not see it | ✅ Direct-IP connection is visible and checked | -| Identify which process owns the connection | ❌ No built-in visibility | ❌ No process visibility | ✅ `somo` shows PID/program | +| Identify which process owns the connection | ❌ No built-in visibility | ❌ No process visibility | ✅ `ss`/`somo` shows PID/program | | Stop outbound exfiltration after compromise | ❌ No containment layer | ⚠️ Only if the attacker still uses DNS | ✅ Can kill suspicious established connections | | Supply-chain malware bypassing DNS | ❌ High risk | ❌ Still exposed | ✅ Stronger containment | | Overall outbound protection posture | ❌ Weak | ⚠️ Partial | 🛡️ Strongest of the three | @@ -126,13 +126,13 @@ If neither is found, start fails. Use `--db` to specify the path manually. When `--debug` is enabled, hexwall also logs intermediate, non-actionable outcomes it would otherwise stay quiet about — a domain or IP coming back clean from a reputation check, or a check being skipped because its flag is disabled. Warnings, blocks, and kills are always logged regardless of `--debug`. -Every scan tick prints a one-line summary regardless of `--debug` — connection count, and how many were trusted, blocked, unknown, or errored — and an empty scan (`somo` returning zero connections) is always logged too. +Every scan tick prints a one-line summary regardless of `--debug` — connection count, and how many were trusted, blocked, unknown, or errored — and an empty scan (the connection-listing tool returning zero connections) is always logged too. --- ## When a connection is killed -A connection is only ever killed in `--mode enforce`, for a connection `somo` reports as currently established. Three independent checks can trigger it: +A connection is only ever killed in `--mode enforce`, for a connection the active connection-listing tool (`ss` or `somo`) reports as currently established. Three independent checks can trigger it: 1. **Pi-hole domain denylist match** — Zeek captured the connection's TLS SNI (via `--zeek-log`) and that domain is on Pi-hole's own denylist or gravity blocklist. Requires only Pi-hole; no third-party service is involved, and this check runs regardless of the `--enable-deghost-*` flags. 2. **Third-party domain reputation match** — an SNI was captured, the domain is on neither Pi-hole list and isn't already known-trusted for this IP, and the reputation API's verdict (fetched live or reused from a 6-hour cache) says to block it. Requires `--enable-deghost-domain` (default `true`). diff --git a/internal/connlist/ss_lister_test.go b/internal/connlist/ss_lister_test.go index 5d010f0..d795898 100644 --- a/internal/connlist/ss_lister_test.go +++ b/internal/connlist/ss_lister_test.go @@ -46,6 +46,25 @@ func TestParseSSOutputRealFixture(t *testing.T) { } } +func TestParseSSOutputIPv6(t *testing.T) { + // Captured live from a real IPv6 loopback connection: ss brackets IPv6 + // addresses, unlike its IPv4 output. splitSSAddress's LastIndex(":") + // split leaves the brackets in RAddress rather than stripping them — + // this is only safe because monitor.remoteIP() strips brackets itself as + // a fallback. This test exists to catch it if that ever stops being true. + const line = "0 0 [::1]:56328 [::1]:12450 users:((\"nc\",pid=48399,fd=3))\n" + + conns := parseSSOutput(line) + if len(conns) != 1 { + t.Fatalf("parseSSOutput() len = %d, want 1", len(conns)) + } + + want := Connection{Protocol: "tcp", LPort: "56328", RAddress: "[::1]", RPort: "12450", Program: "nc", PID: "48399", State: "ESTAB"} + if conns[0] != want { + t.Fatalf("parseSSOutput()[0] = %+v, want %+v", conns[0], want) + } +} + func TestParseSSOutputUnattributedProcess(t *testing.T) { // ss omits the whole process column when it can't attribute the socket // to a process (e.g. owned by another user without CAP_NET_ADMIN). From a89560a04e3d113e923fd0bf8e81c4a965472aab Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Wed, 23 Sep 2026 14:45:42 +0600 Subject: [PATCH 30/56] feat: skip probing for ss on macOS, notify user of selected tool on screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ss depends on CONFIG_INET_DIAG_DESTROY, a Linux kernel feature — a binary named "ss" on macOS could never actually destroy a socket even if one existed, so probing for it there was always pointless. Split SelectBackend into a thin public wrapper and an injectable selectBackend(goos, ssAvailable, somoInstalled) so this can be tested deterministically regardless of the host OS: on darwin, ssAvailable is never even called, and the "neither found" error mentions only somo instead of suggesting an ss install that could never work. Also make the connection-listing tool choice visible on screen (fmt.Println), not just in the structured slog line, both on success and on the fatal "neither tool found" exit. --- internal/connlist/backend_test.go | 64 +++++++++++++++++++++++++++++++ internal/connlist/connlist.go | 24 ++++++++++-- main.go | 2 + 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/internal/connlist/backend_test.go b/internal/connlist/backend_test.go index c2dcba9..20470d2 100644 --- a/internal/connlist/backend_test.go +++ b/internal/connlist/backend_test.go @@ -1,8 +1,10 @@ package connlist import ( + "errors" "os" "path/filepath" + "strings" "testing" ) @@ -57,3 +59,65 @@ func TestSelectBackend(t *testing.T) { }) } } + +// ---- selectBackend: macOS never probes for ss --------------------------- + +func TestSelectBackendDarwinNeverCallsSSAvailable(t *testing.T) { + t.Parallel() + + ssAvailableCalled := func() bool { + t.Error("ssAvailable() was called on darwin, want it never probed") + return true + } + somoInstalled := func() error { return nil } + + _, name, err := selectBackend("darwin", ssAvailableCalled, somoInstalled) + if err != nil { + t.Fatalf("selectBackend(darwin) error = %v, want nil", err) + } + if name != "somo" { + t.Fatalf("selectBackend(darwin) name = %q, want %q (ss must never be picked on darwin, even if it claims available)", name, "somo") + } +} + +func TestSelectBackendDarwinFailureMessageOmitsSS(t *testing.T) { + t.Parallel() + + _, _, err := selectBackend("darwin", func() bool { return false }, func() error { return errors.New("not found") }) + if err == nil { + t.Fatal("selectBackend(darwin) error = nil, want error when somo is also missing") + } + if !strings.Contains(err.Error(), "somo") { + t.Fatalf("selectBackend(darwin) error = %q, want it to mention somo", err) + } + if strings.Contains(err.Error(), "install ss") { + t.Fatalf("selectBackend(darwin) error = %q, want it to NOT suggest installing ss on macOS", err) + } +} + +func TestSelectBackendLinuxUnaffected(t *testing.T) { + t.Parallel() + + lister, name, err := selectBackend("linux", func() bool { return true }, func() error { return nil }) + if err != nil { + t.Fatalf("selectBackend(linux) error = %v, want nil", err) + } + if name != "ss" { + t.Fatalf("selectBackend(linux) name = %q, want %q", name, "ss") + } + if lister == nil { + t.Fatal("selectBackend(linux) lister = nil, want non-nil") + } +} + +func TestSelectBackendLinuxFailureMessageMentionsBoth(t *testing.T) { + t.Parallel() + + _, _, err := selectBackend("linux", func() bool { return false }, func() error { return errors.New("not found") }) + if err == nil { + t.Fatal("selectBackend(linux) error = nil, want error") + } + if !strings.Contains(err.Error(), "ss") || !strings.Contains(err.Error(), "somo") { + t.Fatalf("selectBackend(linux) error = %q, want it to mention both ss and somo", err) + } +} diff --git a/internal/connlist/connlist.go b/internal/connlist/connlist.go index 01fe985..117ee8a 100644 --- a/internal/connlist/connlist.go +++ b/internal/connlist/connlist.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "os/exec" + "runtime" "strings" ) @@ -33,17 +34,32 @@ func CheckInstalled() error { // SelectBackend picks a connection-listing tool by availability: ss // (iproute2) first since it needs no extra install on virtually any Linux -// box, falling back to somo when ss isn't on PATH (e.g. macOS). name is -// "ss" or "somo", for logging which tool is in effect. +// box, falling back to somo when ss isn't on PATH. name is "ss" or "somo", +// for logging which tool is in effect. func SelectBackend() (lister func() ([]Connection, error), name string, err error) { - if SSAvailable() { + return selectBackend(runtime.GOOS, SSAvailable, CheckInstalled) +} + +// selectBackend is SelectBackend's logic with the OS and both availability +// checks injectable, so tests can exercise the macOS branch deterministically +// regardless of which OS the test itself runs on. +// +// ss depends on CONFIG_INET_DIAG_DESTROY, a Linux kernel feature — a binary +// named "ss" on macOS could never actually destroy a socket even if one +// existed, so macOS never probes for it at all and goes straight to somo. +func selectBackend(goos string, ssAvailable func() bool, somoInstalled func() error) (lister func() ([]Connection, error), name string, err error) { + if goos != "darwin" && ssAvailable() { return GetEstablishedConnectionsViaSS, "ss", nil } - if CheckInstalled() == nil { + if somoInstalled() == nil { return GetEstablishedConnections, "somo", nil } + if goos == "darwin" { + return nil, "", fmt.Errorf("no connection-listing tool available: install somo (https://github.com/theopfr/somo) — ss (iproute2) is Linux-only and cannot work on macOS") + } + return nil, "", fmt.Errorf("no connection-listing tool available: install ss (iproute2) or somo (https://github.com/theopfr/somo)") } diff --git a/main.go b/main.go index a9d4040..62840ab 100644 --- a/main.go +++ b/main.go @@ -131,9 +131,11 @@ func run() int { // 1. Pick a connection-listing tool: ss (iproute2) if available, else somo. connectionLister, toolName, err := connlist.SelectBackend() if err != nil { + fmt.Println("No connection-listing tool available:", err) slog.Error("no connection-listing tool available", "err", err) return 1 } + fmt.Printf("> Using %s for connection listing.\n", toolName) slog.Info("using connection-listing tool", "tool", toolName) // 2. Resolve the Pi-hole database path from --db or auto-detection. From 1df86fc7487cb8dc4d6961b94a7cc29ab89c9205 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Wed, 23 Sep 2026 18:12:59 +0600 Subject: [PATCH 31/56] fix(zeek): parse Zeek's default TSV log format, not JSON-only Real, previously undiscovered bug: zeek.go's parser only ever handled JSON-formatted Zeek logs (json.Unmarshal on every line), but a stock Zeek install's default LogAscii output is tab-separated, not JSON. Every line silently failed to parse and was skipped -- no error, no warning. This meant rung 2 of the decision ladder (Zeek SNI lookup) permanently reported "no SNI" for every single connection on any default Zeek setup, silently disabling the entire SNI/domain-matching capability with zero indication anything was wrong. Found by live end-to-end testing against a real Zeek 9.0.0 instance (the existing unit tests only ever used hand-written JSON fixtures, which is why this was never caught). Adds TSV parsing: the column order is learned once from the log's own "#fields" header rather than hardcoded, so it survives Zeek versions or configs that add columns. Since the tailer normally starts reading from the current end of the file (past the header, which Zeek writes once at file creation), NewClient now does a separate, small read from the start of the file specifically to learn the header. A "#fields" line seen during normal tailing (the post-rotation case) also updates the learned column map, so a rotated log's fresh header is picked up the same way. JSON parsing is unchanged and still supported. Confirmed fixed live: after this change, a real held-open TLS connection's SNI was correctly matched, and the full decision ladder (rungs 2 through 6, including a live Pi-hole gravity.db check and a live deghost API call) fired end-to-end for the first time. README updated: documents both supported log formats and the Zeek version (9.0.0) this was verified against. --- README.md | 9 ++ internal/zeek/zeek.go | 140 +++++++++++++++++++++++++---- internal/zeek/zeek_test.go | 178 +++++++++++++++++++++++++++++++++++++ 3 files changed, 310 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 6d6dd19..6497d67 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,15 @@ If neither is found, start fails. Use `--db` to specify the path manually. `--enable-zeek` and `--zeek-log` are independent switches for two unrelated Zeek consumers: `--zeek-log` feeds SNI-based domain matching into the main decision ladder, while `--enable-zeek` (plus `--zeek-notice-log`) turns on a separate, standalone DNS-bypass alert path. Either, both, or neither can be set. +### Zeek setup + +hexwall tails `ssl.log` directly, in whichever of Zeek's two output formats it's actually producing — no Zeek-side configuration is required: + +- **TSV** (Zeek's default `LogAscii` output, used by a stock install with no extra config) — the column order is learned from the log file's own `#fields` header the first time it's read, rather than assumed, so it keeps working across Zeek versions or configurations that add extra columns. +- **JSON** (`redef LogAscii::use_json = T;`) — also supported, parsed line by line. + +Tested against Zeek 9.0.0. Point `--zeek-log` at the running instance's `ssl.log`: for a plain `zeek -i ` invocation, that's a file in the working directory Zeek was started from; under `zeekctl`, it's `/current/ssl.log`. + ### Debug logging When `--debug` is enabled, hexwall also logs intermediate, non-actionable outcomes it would otherwise stay quiet about — a domain or IP coming back clean from a reputation check, or a check being skipped because its flag is disabled. Warnings, blocks, and kills are always logged regardless of `--debug`. diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go index 39704f3..e9335d4 100644 --- a/internal/zeek/zeek.go +++ b/internal/zeek/zeek.go @@ -2,6 +2,7 @@ package zeek import ( + "bufio" "bytes" "context" "encoding/json" @@ -29,7 +30,10 @@ type sniEntry struct { observedAt time.Time } -// sslRecord is the subset of fields we extract from a Zeek SSL JSON log line. +// sslRecord is the subset of fields extracted from a Zeek SSL log line when +// that line is JSON (LogAscii::use_json = T). Zeek's default output is +// tab-separated, not JSON — see recordFromLine and parseTSVFieldsLine for +// that path, which is what a stock Zeek install actually produces. type sslRecord struct { OrigH string `json:"id.orig_h"` OrigP any `json:"id.orig_p"` @@ -46,6 +50,12 @@ type Client struct { mu sync.RWMutex cache map[string]sniEntry + // tsvFields maps a Zeek TSV column name (e.g. "id.orig_h") to its column + // index, learned from a "#fields" header line. nil until one has been + // seen for the current file, in which case non-JSON lines can't yet be + // interpreted and are skipped. + tsvFields map[string]int + offset int64 pending []byte // bytes read past the last newline; held until the rest of the line arrives stopCh chan struct{} @@ -53,7 +63,8 @@ type Client struct { } // NewClient opens the ssl.log at the configured path and starts a -// background goroutine that tails new JSON lines. +// background goroutine that tails new lines, in either of Zeek's log +// formats (TSV, the default; or JSON, if LogAscii::use_json is set). // If cfg is nil or LogPath is empty, it returns nil without error. func NewClient(cfg *Config) (*Client, error) { if cfg == nil || strings.TrimSpace(cfg.LogPath) == "" { @@ -80,13 +91,20 @@ func NewClient(cfg *Config) (*Client, error) { logPath: logPath, cache: make(map[string]sniEntry), offset: fi.Size(), - stopCh: make(chan struct{}), - done: make(chan struct{}), + // The tailer starts reading from the current end of the file (above), + // which is always past the "#fields" header Zeek writes once, at the + // very top, when it first creates the log. Without this separate, + // from-the-start read, TSV parsing would never learn the column + // order in the normal case (Zeek already running before hexwall + // starts) and would silently never match anything. + tsvFields: readTSVHeader(logPath), + stopCh: make(chan struct{}), + done: make(chan struct{}), } go c.tailLoop(context.Background()) - slog.Info("zeek log watcher started", "path", logPath, "offset", c.offset) + slog.Info("zeek log watcher started", "path", logPath, "offset", c.offset, "tsv_header_found", c.tsvFields != nil) return c, nil } @@ -122,6 +140,98 @@ func (c *Client) Lookup(localPort, remoteIP, remotePort string) (string, bool) { return entry.domain, true } +// parseTSVFieldsLine parses a Zeek "#fields\t..." header line into a +// column-name -> index map. Returns nil for any other line, including other +// "#"-prefixed header metadata (#separator, #path, #open, ...). +func parseTSVFieldsLine(line string) map[string]int { + if !strings.HasPrefix(line, "#fields\t") { + return nil + } + + cols := strings.Split(line, "\t") + fields := make(map[string]int, len(cols)-1) + for i, name := range cols[1:] { + fields[name] = i + } + + return fields +} + +// readTSVHeader scans from the start of the log file for a "#fields" header +// line. Returns nil if the file isn't in TSV format (e.g. it's JSON, or +// reading fails), in which case the caller falls back to per-line JSON +// detection only. +func readTSVHeader(logPath string) map[string]int { + f, err := os.Open(logPath) + if err != nil { + return nil + } + defer func() { _ = f.Close() }() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if fields := parseTSVFieldsLine(line); fields != nil { + return fields + } + if !strings.HasPrefix(line, "#") { + return nil // reached a data line without finding #fields + } + } + + return nil +} + +// recordFromLine extracts the five ssl.log fields this package needs from a +// single log line, transparently handling both of Zeek's log formats: +// +// - TSV (Zeek's default): a "#fields" header line, seen once per file, +// declares the column order; every following line is read positionally +// against it via c.tsvFields. Any other "#"-prefixed line is header +// metadata and is skipped. A "#fields" line seen here (not just at +// NewClient time) updates c.tsvFields — this is what makes a rotated +// log's fresh header get picked up too. +// - JSON (LogAscii::use_json = T): each line is a self-describing object. +// +// ok is false for header lines, or for a TSV data line seen before its +// header (data missing "-" is normalized to "", matching Zeek's own +// unset-field marker). +func (c *Client) recordFromLine(line string) (origH, origP, respH, respP, serverName string, ok bool) { + switch { + case strings.HasPrefix(line, "#"): + if fields := parseTSVFieldsLine(line); fields != nil { + c.tsvFields = fields + } + return "", "", "", "", "", false + + case strings.HasPrefix(line, "{"): + var rec sslRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return "", "", "", "", "", false + } + return rec.OrigH, fmt.Sprint(rec.OrigP), rec.RespH, fmt.Sprint(rec.RespP), rec.ServerName, true + + default: + if c.tsvFields == nil { + return "", "", "", "", "", false + } + + cols := strings.Split(line, "\t") + get := func(name string) string { + i, ok := c.tsvFields[name] + if !ok || i >= len(cols) { + return "" + } + if cols[i] == "-" { // Zeek's unset-field marker + return "" + } + return cols[i] + } + + return get("id.orig_h"), get("id.orig_p"), get("id.resp_h"), get("id.resp_p"), get("server_name"), true + } +} + func (c *Client) tailLoop(ctx context.Context) { defer close(c.done) @@ -160,7 +270,9 @@ func (c *Client) processNewLines() { } // File was likely rotated — reset to the beginning and drop any - // partial line held over from the old file. + // partial line held over from the old file. The new file's own + // "#fields" header will be re-learned via recordFromLine below as the + // tailer reads through it, same as any other line. if fi.Size() < c.offset { c.offset = 0 c.pending = nil @@ -206,24 +318,18 @@ func (c *Client) processNewLines() { continue } - var rec sslRecord - if err := json.Unmarshal([]byte(line), &rec); err != nil { - continue - } - - if rec.ServerName == "" || rec.OrigH == "" || rec.RespH == "" { + origH, origP, respH, respP, serverName, ok := c.recordFromLine(line) + if !ok || serverName == "" || origH == "" || respH == "" { continue } - origP := fmt.Sprint(rec.OrigP) - respP := fmt.Sprint(rec.RespP) now := time.Now() - entry := sniEntry{domain: rec.ServerName, observedAt: now} + entry := sniEntry{domain: serverName, observedAt: now} // Store both orientations so Lookup works regardless of which // side of the connection is the local endpoint. - c.cache[origP+"|"+rec.RespH+"|"+respP] = entry - c.cache[respP+"|"+rec.OrigH+"|"+origP] = entry + c.cache[origP+"|"+respH+"|"+respP] = entry + c.cache[respP+"|"+origH+"|"+origP] = entry } // Prune expired entries. diff --git a/internal/zeek/zeek_test.go b/internal/zeek/zeek_test.go index 9cb9c85..2cb0976 100644 --- a/internal/zeek/zeek_test.go +++ b/internal/zeek/zeek_test.go @@ -3,6 +3,7 @@ package zeek import ( "os" "path/filepath" + "strings" "testing" "time" ) @@ -131,3 +132,180 @@ func TestClientLookup_Expiry(t *testing.T) { t.Fatal("Lookup() ok = true for an expired entry, want false") } } + +// ---- TSV format (Zeek's actual default output) -------------------------- + +// realZeekTSVHeader and realZeekTSVDataLine are captured verbatim from a +// live Zeek 9.0.0 instance's ssl.log (default config, no LogAscii::use_json). +// This is the format a stock Zeek install actually produces — the JSON +// tests above only exercise the opt-in LogAscii::use_json = T path. +const realZeekTSVHeader = "#separator \\x09\n" + + "#set_separator\t,\n" + + "#empty_field\t(empty)\n" + + "#unset_field\t-\n" + + "#path\tssl\n" + + "#open\t2026-09-23-16-59-25\n" + + "#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tversion\tcipher\tcurve\tserver_name\tresumed\tlast_alert\tnext_protocol\testablished\tssl_history\tcert_chain_fps\tclient_cert_chain_fps\tsni_matches_cert\n" + + "#types\ttime\tstring\taddr\tport\taddr\tport\tstring\tstring\tstring\tstring\tbool\tstring\tstring\tbool\tstring\tvector[string]\tvector[string]\tbool\n" + +const realZeekTSVDataLine = "1790163565.389393\tC5MrwN15Qi19OF5Y2g\t192.168.0.223\t37526\t172.66.147.243\t443\tTLSv13\tTLS_AES_256_GCM_SHA384\tx25519\texample.com\tF\t-\t-\tT\tCsiI\t-\t-\t-\n" + +func TestParseTSVFieldsLine(t *testing.T) { + t.Parallel() + + lines := strings.Split(strings.TrimRight(realZeekTSVHeader, "\n"), "\n") + var fieldsLine string + for _, l := range lines { + if strings.HasPrefix(l, "#fields\t") { + fieldsLine = l + } + } + if fieldsLine == "" { + t.Fatal("test fixture has no #fields line") + } + + fields := parseTSVFieldsLine(fieldsLine) + want := map[string]int{ + "ts": 0, "uid": 1, "id.orig_h": 2, "id.orig_p": 3, "id.resp_h": 4, "id.resp_p": 5, + "server_name": 9, + } + for name, idx := range want { + if fields[name] != idx { + t.Errorf("parseTSVFieldsLine()[%q] = %d, want %d", name, fields[name], idx) + } + } + + if parseTSVFieldsLine("#types\ttime\tstring") != nil { + t.Error("parseTSVFieldsLine(#types line) = non-nil, want nil (only #fields should match)") + } +} + +func TestReadTSVHeader(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine) + + fields := readTSVHeader(path) + if fields == nil { + t.Fatal("readTSVHeader() = nil, want a field map for a real TSV file") + } + if fields["server_name"] != 9 { + t.Fatalf("readTSVHeader()[server_name] = %d, want 9", fields["server_name"]) + } +} + +func TestReadTSVHeaderNilForJSON(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}`+"\n") + + if fields := readTSVHeader(path); fields != nil { + t.Fatalf("readTSVHeader() on a JSON file = %v, want nil", fields) + } +} + +// newTestClientTSV builds a Client the way NewClient would for a TSV file: +// with tsvFields pre-populated from the file's own header, since the real +// tailer always starts reading from the file's current end (past the +// header) and depends on this separate, from-the-start read to learn the +// column order at all. +func newTestClientTSV(t *testing.T, path string) *Client { + t.Helper() + return &Client{ + logPath: path, + cache: make(map[string]sniEntry), + tsvFields: readTSVHeader(path), + } +} + +func TestClientProcessNewLines_TSVRealFixture(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine) + + c := newTestClientTSV(t, path) + c.processNewLines() + + domain, ok := c.Lookup("37526", "172.66.147.243", "443") + if !ok { + t.Fatal("Lookup() ok = false, want true for the real captured TSV line") + } + if domain != "example.com" { + t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") + } + + // Reverse orientation too. + if domain, ok := c.Lookup("443", "192.168.0.223", "37526"); !ok || domain != "example.com" { + t.Fatalf("Lookup() reverse orientation = (%q, %v), want (%q, true)", domain, ok, "example.com") + } +} + +func TestClientProcessNewLines_TSVUnsetServerNameSkipped(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + // server_name column (index 9) is "-": Zeek's unset-field marker, e.g. a + // resumed session where the name wasn't re-sent. + line := "1790163565.389393\tCabc\t192.168.0.223\t40000\t172.66.147.243\t443\tTLSv13\tTLS_AES_256_GCM_SHA384\tx25519\t-\tT\t-\t-\tT\tCsiI\t-\t-\t-\n" + writeFile(t, path, realZeekTSVHeader+line) + + c := newTestClientTSV(t, path) + c.processNewLines() + + if len(c.cache) != 0 { + t.Fatalf("cache = %v, want empty (unset server_name must be skipped)", c.cache) + } +} + +func TestClientProcessNewLines_TSVShortLineSkipped(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + // Far fewer columns than the header declares — must not panic (index + // out of range) and must not produce a cache entry. + writeFile(t, path, realZeekTSVHeader+"1790163565.389393\tCabc\t192.168.0.223\n") + + c := newTestClientTSV(t, path) + c.processNewLines() + + if len(c.cache) != 0 { + t.Fatalf("cache = %v, want empty for a too-short TSV line", c.cache) + } +} + +func TestClientProcessNewLines_TSVDataBeforeHeaderSkipped(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVDataLine) // no header at all + + // Deliberately NOT using newTestClientTSV: this simulates tsvFields + // never having been learned (e.g. the header lived entirely before this + // client's starting offset in some edge case). + c := newTestClient(t, path) + c.processNewLines() + + if len(c.cache) != 0 { + t.Fatalf("cache = %v, want empty when no #fields header has been seen", c.cache) + } +} + +func TestClientProcessNewLines_TSVRotationRelearnsHeader(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine) + + c := newTestClientTSV(t, path) + c.processNewLines() + if _, ok := c.Lookup("37526", "172.66.147.243", "443"); !ok { + t.Fatal("Lookup() ok = false before rotation, want true") + } + + // Rotation: a fresh, smaller file appears at the same path, with its own + // header re-written from scratch (a real Zeek restart does exactly this). + newHeader := "#separator \\x09\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tserver_name\n" + newLine := "1790163999.0\tCnew\t10.0.0.5\t51820\t203.0.113.30\t443\tpost-rotation.example\n" + writeFile(t, path, "") // straddling tick, matches the JSON rotation test above + c.processNewLines() + writeFile(t, path, newHeader+newLine) + c.processNewLines() + + domain, ok := c.Lookup("51820", "203.0.113.30", "443") + if !ok || domain != "post-rotation.example" { + t.Fatalf("Lookup() after rotation = (%q, %v), want (%q, true)", domain, ok, "post-rotation.example") + } +} From 26723920b62af29661fee38f54589c0b1c5d6b7e Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Tue, 29 Sep 2026 00:32:29 +0600 Subject: [PATCH 32/56] docs: README describes ss -K single-connection kill and macOS backend choice --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6497d67..5dfce1d 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ hexwall targets Linux, where prebuilt binaries are published by CI. macOS is bes ### Connection backend -At startup, hexwall picks a connection-listing backend by availability, not by OS: `ss` first if it's on `PATH`, then `somo`, then it exits with an install message if neither is found. This means `ss` is used by default on real Linux boxes (iproute2 ships with virtually every distro) and `somo` only comes into play as a fallback — on macOS, where `ss` doesn't exist, or on a stripped-down Linux system without iproute2 installed. The connection-kill path already tries `ss -K` first and falls back to a process-level signal kill regardless of which lister found the connection, so no behavior changes based on which backend is selected. +At startup, hexwall picks a connection-listing backend and prints which one it chose. On Linux, `ss` is used if it's on `PATH`, otherwise `somo`; hexwall exits with an install message if neither is found. `ss` is therefore the default on real Linux boxes (iproute2 ships with virtually every distro), and `somo` only comes into play on a stripped-down system without iproute2. On macOS, `ss` is never probed: its single-connection kill relies on a Linux kernel feature (`CONFIG_INET_DIAG_DESTROY`), so hexwall goes straight to `somo` and every kill there is process-level. The kill path works the same whichever lister found the connection (see [When a connection is killed](#when-a-connection-is-killed)). ### Install `somo` @@ -164,7 +164,10 @@ Connections are not killed in these cases: - The IP reputation API returns HTTP `403`, treated as clean/private/reserved. - The reputation lookup itself fails — a failed lookup is never treated as a block. -When a kill does happen, hexwall first records the event in the local `killed_connections` audit table, then signals the owning PID: `SIGTERM` first, escalating to `SIGKILL` after a 2-second grace period if the process is still alive. A hardcoded set of protected system processes (`systemd`, `sshd`, `dockerd`, and similar) can never be killed this way, whichever check triggered the match. +When a kill does happen, hexwall first records the event in the local `killed_connections` audit table, then runs its guardrails: it refuses PID 1, its own PID, invalid PIDs, and a hardcoded set of protected system processes (`systemd`, `sshd`, `dockerd`, and similar), whichever check triggered the match. It then closes the connection in one of two ways: + +1. **Just that connection**: `ss -K` asks the kernel to destroy the single socket, leaving every other connection the process holds untouched. This needs `CAP_NET_ADMIN` (e.g. running as root) and a kernel built with `CONFIG_INET_DIAG_DESTROY`. Because `ss -K` can exit successfully even when the kernel refused, hexwall re-lists the socket afterwards and only counts the kill if the connection is actually gone. +2. **The whole process** (fallback): if the connection is still there, or `ss` is unavailable (always the case on macOS), hexwall signals the owning PID: `SIGTERM` first, escalating to `SIGKILL` after a 2-second grace period if the process is still alive. This also drops every other connection that process holds. ### Reputation API cache behavior From 7fd74ba6e48988567ae4aa4443b8048b2b378605 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sat, 10 Oct 2026 16:15:51 +0600 Subject: [PATCH 33/56] fix(pihole): don't trust IPs of domains Pi-hole blocked DomainsSeenSince read every query in the FTL log, including blocked ones. The trust cache then re-resolved those domains through the host's resolver, so with a fallback nameserver a blocked domain's real IP became trusted on the IP-only rung. Only statuses where Pi-hole answered the query now feed the cache; the list is an allow-list so new FTL status codes stay untrusted until reviewed. --- internal/pihole/pihole.go | 32 ++++++++++++++++++++++--- internal/pihole/pihole_test.go | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index d5fd0c2..96f0fba 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -246,16 +246,42 @@ func (c *Checker) PolicyCounts() (map[string]int64, error) { return counts, nil } -// DomainsSeenSince returns the distinct domains Pi-hole recorded since the given Unix timestamp. +// answeredStatuses lists the FTL query status codes that count as "Pi-hole answered this +// domain". Only these feed the trusted-IP cache. A blocked query is still written to the +// query log, and without this filter its domain would be re-resolved by the host's own +// resolver and its real IP trusted -- blocking a domain would end up trusting it. +// Status codes: https://docs.pi-hole.net/database/query-database/#query-status +// +// This is an allow-list on purpose: a status code added by a future FTL release is +// untrusted until it is reviewed here, rather than silently trusted. +var answeredStatuses = []int{ + 2, // forwarded to upstream + 3, // answered from cache + 12, // retried + 13, // retried, but ignored + 14, // already forwarded + 17, // answered from stale cache +} + +// DomainsSeenSince returns the distinct domains Pi-hole answered since the given Unix timestamp. +// Queries Pi-hole blocked are excluded; see answeredStatuses. func (c *Checker) DomainsSeenSince(since int64) ([]string, error) { - rows, err := c.db.Query(` + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(answeredStatuses)), ",") + args := make([]any, 0, len(answeredStatuses)+1) + args = append(args, since) + for _, status := range answeredStatuses { + args = append(args, status) + } + + rows, err := c.db.Query(fmt.Sprintf(` SELECT DISTINCT LOWER(TRIM(domain)) FROM queries WHERE timestamp >= ? AND domain IS NOT NULL AND TRIM(domain) <> '' + AND status IN (%s) ORDER BY LOWER(TRIM(domain)) - `, since) + `, placeholders), args...) if err != nil { return nil, fmt.Errorf("database query failed: %w", err) } diff --git a/internal/pihole/pihole_test.go b/internal/pihole/pihole_test.go index b64decc..499638a 100644 --- a/internal/pihole/pihole_test.go +++ b/internal/pihole/pihole_test.go @@ -2,7 +2,9 @@ package pihole import ( "net/url" + "slices" "testing" + "time" ) func TestSQLiteReadOnlyDSNRelativePath(t *testing.T) { @@ -24,3 +26,44 @@ func TestSQLiteReadOnlyDSNAbsolutePath(t *testing.T) { t.Fatalf("dsn = %q, want %q", dsn, want) } } + +// TestDomainsSeenSinceExcludesBlocked pins the trust-cache input: a domain Pi-hole blocked +// is still in the query log, and must not come back as a domain to resolve and trust. +// Statuses 2, 3 and 17 are the ones a real Pi-hole (FTL v6.7.1) logged for answered queries; +// 1 is what it logged for a gravity-blocked domain. +func TestDomainsSeenSinceExcludesBlocked(t *testing.T) { + t.Parallel() + + checker := newTestChecker(t, "") + now := time.Now().Unix() + rows := []struct { + domain string + status int + }{ + {"forwarded.example", 2}, + {"cached.example", 3}, + {"stale-cache.example", 17}, + {"gravity-blocked.example", 1}, + {"regex-blocked.example", 4}, + {"denylist-blocked.example", 5}, + {"cname-blocked.example", 9}, + } + for _, r := range rows { + if _, err := checker.db.Exec( + `INSERT INTO queries (timestamp, type, domain, status) VALUES (?, 1, ?, ?)`, + now, r.domain, r.status, + ); err != nil { + t.Fatalf("insert %q: %v", r.domain, err) + } + } + + got, err := checker.DomainsSeenSince(now - 60) + if err != nil { + t.Fatalf("DomainsSeenSince() error: %v", err) + } + + want := []string{"cached.example", "forwarded.example", "stale-cache.example"} + if !slices.Equal(got, want) { + t.Fatalf("DomainsSeenSince() = %v, want %v", got, want) + } +} From 6abdb40cb6a00d89bf61f488b7705f160d3b7b89 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 14:28:05 +0600 Subject: [PATCH 34/56] feat(pihole): apply Pi-hole regex allow and deny lists Rungs 3 and 4 only checked the exact lists and gravity; the regex views were counted at startup and never used. A regex-denied domain reached by direct IP was not blocked, and a regex-allowed domain fell through to the reputation API, which could kill it in enforce mode. Patterns are compiled once, matched case-insensitively like FTL, and reloaded on the 30s refresh only when the lists change. FTL options hexwall cannot honour for a connection (querytype, unknown ones) resolve toward not killing: an allow rule still applies, a deny rule is skipped. --- internal/pihole/pihole.go | 21 ++-- internal/pihole/regex.go | 182 ++++++++++++++++++++++++++++++++++ internal/pihole/regex_test.go | 145 +++++++++++++++++++++++++++ main.go | 3 + 4 files changed, 345 insertions(+), 6 deletions(-) create mode 100644 internal/pihole/regex.go create mode 100644 internal/pihole/regex_test.go diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index 96f0fba..d186e54 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "time" // Register the pure-Go SQLite driver used to read the Pi-hole FTL database. @@ -27,6 +28,9 @@ type Config struct { type Checker struct { db *sql.DB gravityDB *sql.DB + + // regex is swapped whole by RefreshRegexLists while scans read it from another goroutine. + regex atomic.Pointer[regexLists] } const domainLookback = time.Hour @@ -83,10 +87,15 @@ func NewChecker(config *Config) (*Checker, error) { "path", gravityPath, "err", err) } - return &Checker{ + checker := &Checker{ db: db, gravityDB: gravityDB, - }, nil + } + if err := checker.RefreshRegexLists(); err != nil { + slog.Warn("pi-hole regex lists unavailable; only exact-match policy checks apply", "err", err) + } + + return checker, nil } // openGravityDB opens the Pi-hole gravity database in read-only mode. @@ -160,7 +169,7 @@ func (c *Checker) IsDomainKnown(domain string) (bool, error) { return true, nil } -// IsAllowedByPolicy reports whether the domain currently appears in Pi-hole's allowlist. +// IsAllowedByPolicy reports whether the domain is on Pi-hole's allowlist or matches an allow regex. func (c *Checker) IsAllowedByPolicy(domain string) (bool, error) { domain = normalizeDomain(domain) if domain == "" { @@ -177,7 +186,7 @@ func (c *Checker) IsAllowedByPolicy(domain string) (bool, error) { LIMIT 1 `, domain).Scan(&found) if errors.Is(err, sql.ErrNoRows) { - return false, nil + return c.matchesRegex(domain, true), nil } if err != nil { return false, fmt.Errorf("allowlist lookup failed: %w", err) @@ -186,7 +195,7 @@ func (c *Checker) IsAllowedByPolicy(domain string) (bool, error) { return true, nil } -// IsBlockedByPolicy reports whether the domain currently matches a Pi-hole denylist or gravity entry. +// IsBlockedByPolicy reports whether the domain is on Pi-hole's denylist or gravity, or matches a deny regex. func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { domain = normalizeDomain(domain) if domain == "" { @@ -215,7 +224,7 @@ func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { LIMIT 1 `, domain).Scan(&found) if errors.Is(err, sql.ErrNoRows) { - return false, nil + return c.matchesRegex(domain, false), nil } if err != nil { return false, fmt.Errorf("gravity lookup failed: %w", err) diff --git a/internal/pihole/regex.go b/internal/pihole/regex.go new file mode 100644 index 0000000..c0001f6 --- /dev/null +++ b/internal/pihole/regex.go @@ -0,0 +1,182 @@ +package pihole + +import ( + "errors" + "fmt" + "log/slog" + "regexp" + "slices" + "strings" +) + +// Pi-hole's regex lists hold rules, not names: one pattern such as (^|\.)tracker\.example$ +// covers every subdomain. FTL matches them as case-insensitive POSIX extended regex; Go's +// regexp accepts nearly all of that syntax, and a pattern it cannot compile is skipped with +// a warning. +// +// A pattern can carry FTL options after a semicolon (";invert", ";querytype=AAAA", +// ";reply=..."). hexwall judges a TLS connection, not a DNS query, so an option it cannot +// honour exactly is resolved toward not killing: an allow rule is applied anyway (too broad +// an allow can only miss a block), while a deny rule is skipped (too broad a deny would kill +// traffic Pi-hole itself lets through). + +const ( + regexAllowView = "vw_regex_allowlist" + regexDenyView = "vw_regex_denylist" +) + +// regexRule is one compiled Pi-hole regex list entry. +type regexRule struct { + re *regexp.Regexp + invert bool +} + +func (r regexRule) matches(domain string) bool { + return r.re.MatchString(domain) != r.invert +} + +// regexLists is an immutable snapshot of both regex lists. The raw patterns are kept so a +// refresh can tell whether anything changed, and skip recompiling (and re-warning) if not. +type regexLists struct { + allowRaw []string + denyRaw []string + allow []regexRule + deny []regexRule +} + +// parseRegexRule compiles one regex list entry. It returns an error when the entry must be +// skipped: the pattern does not compile, or it is a deny rule with an option hexwall cannot +// honour for a connection. +func parseRegexRule(raw string, allow bool) (regexRule, error) { + pattern, options, _ := strings.Cut(raw, ";") + + var rule regexRule + if options != "" { + for _, option := range strings.Split(options, ";") { + switch { + case option == "invert": + rule.invert = true + case strings.HasPrefix(option, "reply="): + // Changes how Pi-hole answers a blocked query, not whether it blocks. + case strings.HasPrefix(option, "querytype="): + if !allow { + return regexRule{}, errors.New("deny rule limited to a DNS query type cannot be applied to a connection") + } + default: + if !allow { + return regexRule{}, fmt.Errorf("unknown option %q", option) + } + } + } + } + + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + return regexRule{}, fmt.Errorf("pattern does not compile in Go: %w", err) + } + rule.re = re + + return rule, nil +} + +func compileRegexRules(raws []string, allow bool) []regexRule { + rules := make([]regexRule, 0, len(raws)) + for _, raw := range raws { + rule, err := parseRegexRule(raw, allow) + if err != nil { + if allow { + slog.Warn("pi-hole regex allow rule skipped; connections it allows may still be killed by reputation checks", + "rule", raw, "err", err) + } else { + slog.Warn("pi-hole regex deny rule skipped", "rule", raw, "err", err) + } + continue + } + rules = append(rules, rule) + } + return rules +} + +// RefreshRegexLists reloads Pi-hole's regex allow and deny lists from gravity.db, so rules +// edited in Pi-hole take effect without restarting hexwall. On error the previously loaded +// rules stay in use: stale rules are closer to the user's intent than no rules. +func (c *Checker) RefreshRegexLists() error { + if c == nil || c.gravityDB == nil { + return nil + } + + allowRaw, err := c.regexPatterns(regexAllowView) + if err != nil { + return err + } + denyRaw, err := c.regexPatterns(regexDenyView) + if err != nil { + return err + } + + if current := c.regex.Load(); current != nil && + slices.Equal(current.allowRaw, allowRaw) && slices.Equal(current.denyRaw, denyRaw) { + return nil + } + + lists := ®exLists{ + allowRaw: allowRaw, + denyRaw: denyRaw, + allow: compileRegexRules(allowRaw, true), + deny: compileRegexRules(denyRaw, false), + } + c.regex.Store(lists) + slog.Info("pi-hole regex lists loaded", + "allow", len(lists.allow), "deny", len(lists.deny), + "skipped", len(allowRaw)+len(denyRaw)-len(lists.allow)-len(lists.deny)) + + return nil +} + +func (c *Checker) regexPatterns(view string) ([]string, error) { + rows, err := c.gravityDB.Query(fmt.Sprintf(` + SELECT DISTINCT TRIM(domain) + FROM %s + WHERE TRIM(domain) <> '' + ORDER BY TRIM(domain) + `, view)) + if err != nil { + return nil, fmt.Errorf("%s lookup failed: %w", view, err) + } + defer func() { + _ = rows.Close() + }() + + var patterns []string + for rows.Next() { + var pattern string + if err := rows.Scan(&pattern); err != nil { + return nil, fmt.Errorf("%s scan failed: %w", view, err) + } + patterns = append(patterns, pattern) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("%s iteration failed: %w", view, err) + } + + return patterns, nil +} + +// matchesRegex reports whether domain matches any loaded allow (or deny) regex rule. +func (c *Checker) matchesRegex(domain string, allow bool) bool { + lists := c.regex.Load() + if lists == nil { + return false + } + + rules := lists.deny + if allow { + rules = lists.allow + } + for _, rule := range rules { + if rule.matches(domain) { + return true + } + } + return false +} diff --git a/internal/pihole/regex_test.go b/internal/pihole/regex_test.go new file mode 100644 index 0000000..e7cf7f1 --- /dev/null +++ b/internal/pihole/regex_test.go @@ -0,0 +1,145 @@ +package pihole + +import "testing" + +// Expected results below were confirmed against Pi-hole's own engine +// (pihole-FTL v6.7.1 regex-test), not assumed. + +func newRegexTestChecker(t *testing.T, rules map[string]int) *Checker { + t.Helper() + gravDB, gravPath := openTestGravityDB(t) + for rule, listType := range rules { + insertDomain(t, gravDB, rule, listType) + } + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + t.Cleanup(func() { _ = checker.Close() }) + if err := checker.RefreshRegexLists(); err != nil { + t.Fatalf("RefreshRegexLists() error = %v", err) + } + return checker +} + +func TestIsBlockedByPolicy_RegexDenylist(t *testing.T) { + t.Parallel() + checker := newRegexTestChecker(t, map[string]int{ + `(^|\.)tracker\.example$`: 3, // type 3 = regex denylist + `^ad[[:digit:]]+\.`: 3, + `(^|\.)Shouty\.Example$`: 3, + `(^|\.)aaaa-only\.example$;querytype=AAAA`: 3, + `(^|\.)reply\.example$;reply=nxdomain`: 3, + `(a)\1`: 3, // backreference: valid for FTL, not for Go's regexp + }) + + tests := []struct { + domain string + want bool + }{ + {"c2.tracker.example", true}, + {"tracker.example", true}, + {"tracker.example.evil.com", false}, // the $ anchor holds + {"C2.TRACKER.EXAMPLE", true}, // FTL matches case-insensitively + {"ad7.example", true}, // POSIX character class + {"api.shouty.example", true}, // uppercase pattern, lowercase domain + {"x.reply.example", true}, // reply= changes the answer, not the verdict + {"x.aaaa-only.example", false}, // a querytype-limited deny rule is skipped + {"unrelated.example", false}, + } + for _, tt := range tests { + blocked, err := checker.IsBlockedByPolicy(tt.domain) + if err != nil { + t.Fatalf("IsBlockedByPolicy(%q) error = %v", tt.domain, err) + } + if blocked != tt.want { + t.Errorf("IsBlockedByPolicy(%q) = %v, want %v", tt.domain, blocked, tt.want) + } + } +} + +func TestIsAllowedByPolicy_RegexAllowlist(t *testing.T) { + t.Parallel() + checker := newRegexTestChecker(t, map[string]int{ + `(^|\.)mycompany\.com$`: 2, // type 2 = regex allowlist + `(^|\.)ipv6-app\.example$;querytype=AAAA`: 2, + }) + + tests := []struct { + domain string + want bool + }{ + {"api.mycompany.com", true}, + {"mycompany.com.evil.net", false}, + {"x.ipv6-app.example", true}, // a querytype-limited allow rule is applied anyway + {"unrelated.example", false}, + } + for _, tt := range tests { + allowed, err := checker.IsAllowedByPolicy(tt.domain) + if err != nil { + t.Fatalf("IsAllowedByPolicy(%q) error = %v", tt.domain, err) + } + if allowed != tt.want { + t.Errorf("IsAllowedByPolicy(%q) = %v, want %v", tt.domain, allowed, tt.want) + } + } +} + +func TestRegexRule_Invert(t *testing.T) { + t.Parallel() + rule, err := parseRegexRule(`(^|\.)tracker\.example$;invert`, false) + if err != nil { + t.Fatalf("parseRegexRule() error = %v", err) + } + if !rule.matches("good.example") { + t.Error("inverted rule should match a domain the pattern does not") + } + if rule.matches("c2.tracker.example") { + t.Error("inverted rule should not match a domain the pattern does") + } +} + +func TestParseRegexRule_UnknownOption(t *testing.T) { + t.Parallel() + if _, err := parseRegexRule(`^x\.;future=1`, false); err == nil { + t.Error("deny rule with an unknown option should be skipped") + } + if _, err := parseRegexRule(`^x\.;future=1`, true); err != nil { + t.Errorf("allow rule with an unknown option should still apply, got %v", err) + } +} + +func TestRefreshRegexLists_PicksUpNewRules(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + t.Cleanup(func() { _ = gravDB.Close() }) + + checker := newTestChecker(t, gravPath) + t.Cleanup(func() { _ = checker.Close() }) + if err := checker.RefreshRegexLists(); err != nil { + t.Fatalf("RefreshRegexLists() error = %v", err) + } + + if blocked, _ := checker.IsBlockedByPolicy("c2.tracker.example"); blocked { + t.Fatal("blocked before any rule exists") + } + + insertDomain(t, gravDB, `(^|\.)tracker\.example$`, 3) + if err := checker.RefreshRegexLists(); err != nil { + t.Fatalf("RefreshRegexLists() error = %v", err) + } + + if blocked, _ := checker.IsBlockedByPolicy("c2.tracker.example"); !blocked { + t.Fatal("rule added in Pi-hole not picked up by refresh") + } +} + +func TestRefreshRegexLists_NilGravityDB(t *testing.T) { + t.Parallel() + checker := &Checker{} + if err := checker.RefreshRegexLists(); err != nil { + t.Fatalf("RefreshRegexLists() error = %v", err) + } + if checker.matchesRegex("anything.example", false) { + t.Fatal("no rules loaded, nothing should match") + } +} diff --git a/main.go b/main.go index 62840ab..0f32a61 100644 --- a/main.go +++ b/main.go @@ -278,6 +278,9 @@ func run() int { return case <-ticker.C: cache.Refresh(ctx) + if err := checker.RefreshRegexLists(); err != nil { + slog.Warn("pi-hole regex list refresh failed; keeping previous rules", "err", err) + } } } }() From 18dfff2f54dd1c0ea174edc14cb4a1786267a90e Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 14:29:58 +0600 Subject: [PATCH 35/56] feat: make deghost reputation checks opt-in Both --enable-deghost-* flags defaulted to true while the code comment called deghost opt-in. Default them to false so no connection data leaves the machine unless asked, and only the user's own Pi-hole rules can kill by default. Without --zeek-log every connection takes the IP-only path, which can't kill with IP deghost off; warn at startup when enforce mode would therefore protect nothing. --- main.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 0f32a61..80c16bc 100644 --- a/main.go +++ b/main.go @@ -57,8 +57,8 @@ func parseFlags() (*runConfig, error) { zeekLogPath := flag.String("zeek-log", "", "path to zeek ssl.log for SNI-based domain verification (empty to skip)") zeekNoticeLog := flag.String("zeek-notice-log", "/opt/zeek/logs/current/notice.log", "path to Zeek notice.log for DNS-bypass alerts") enableZeek := flag.Bool("enable-zeek", false, "enable Zeek-based DNS-bypass detection") - enableDeghostIP := flag.Bool("enable-deghost-ip", true, "enable third-party IP reputation checks on the IP-only fallback path") - enableDeghostDomain := flag.Bool("enable-deghost-domain", true, "enable third-party domain reputation checks at rung 6") + enableDeghostIP := flag.Bool("enable-deghost-ip", false, "send unrecognized IPs to the third-party reputation API (deghostapi.hexbyte.dev) on the IP-only fallback path") + enableDeghostDomain := flag.Bool("enable-deghost-domain", false, "send unrecognized SNI domains to the third-party reputation API (deghostapi.hexbyte.dev) at rung 6") mode := flag.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") debug := flag.Bool("debug", false, "enable debug-level logging, including verbose per-connection scan logging") showVersion := flag.Bool("version", false, "print version and exit") @@ -228,6 +228,12 @@ func run() int { slog.Info("deghost disabled; unrecognized connections will not be escalated to third-party API") } + // Without an SNI source every connection takes the IP-only path, and with IP deghost off + // that path never kills: enforce mode would silently protect nothing. + if cfg.mode == monitor.ModeEnforce && cfg.zeekLogPath == "" && !cfg.enableDeghostIP { + slog.Warn("enforce mode cannot kill any connection: no --zeek-log for Pi-hole rule matching and --enable-deghost-ip is off") + } + if cfg.mode == monitor.ModeEnforce && cfg.enableDeghostIP { slog.Warn("enforce mode with IP deghost enabled: an external reputation verdict can now terminate a local process; shared cloud provider ranges are known to produce false positives on such feeds") } From a538bd1464bd89b307d75e2c4048a9053a4fbbfd Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 16:00:38 +0600 Subject: [PATCH 36/56] fix(zeek): read DNS-bypass notices from real Zeek output The notice.log parser only accepted JSON and pulled the SNI out of msg with a regex. Zeek's default TSV output failed every line, and the JSON output had no SNI, so every bypass alert was dropped without a warning. The detection script now ships embedded in the binary and writes the SNI to notice's sub field. The parser reads TSV (columns from #fields) and JSON, keeps only DNSBypass::PossibleBypass notices, records the source port, and waits for complete lines. Alerts already in the file at startup are skipped so a restart can't act on old ones. Parse errors and a full alert channel are now warned about. Fixtures are real Zeek 9.0.0 output. --- internal/zeek/dns-bypass.zeek | 52 +++++ internal/zeek/notice.go | 290 ++++++++++++++++++--------- internal/zeek/notice_test.go | 236 ++++++++++++++++++++-- internal/zeek/testdata/notice.ndjson | 1 + internal/zeek/testdata/notice.tsv | 10 + 5 files changed, 478 insertions(+), 111 deletions(-) create mode 100644 internal/zeek/dns-bypass.zeek create mode 100644 internal/zeek/testdata/notice.ndjson create mode 100644 internal/zeek/testdata/notice.tsv diff --git a/internal/zeek/dns-bypass.zeek b/internal/zeek/dns-bypass.zeek new file mode 100644 index 0000000..2aa6b75 --- /dev/null +++ b/internal/zeek/dns-bypass.zeek @@ -0,0 +1,52 @@ +##! Detects TLS connections whose SNI was never seen in a prior DNS query +##! from the same client within the correlation window. +##! +##! Installed by `hexwall zeek-setup`, which embeds this file in the hexwall +##! binary. Re-run that command after upgrading hexwall rather than editing +##! this copy: hexwall reads the notices it writes, so the two must match. + +module DNSBypass; + +export { + redef enum Notice::Type += { + PossibleBypass + }; + + ## How far back to look for a matching DNS query. + const correlation_window = 10min &redef; +} + +# client_ip -> set of domains it has queried recently, with last-seen time +global recent_queries: table[addr] of table[string] of time &create_expire=correlation_window; + +event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) + { + local client = c$id$orig_h; + local domain = to_lower(query); + + if ( client !in recent_queries ) + recent_queries[client] = table(); + + recent_queries[client][domain] = network_time(); + } + +event ssl_established(c: connection) + { + if ( ! c?$ssl || ! c$ssl?$server_name ) + return; + + local client = c$id$orig_h; + local sni = to_lower(c$ssl$server_name); + + if ( client in recent_queries && sni in recent_queries[client] ) + return; + + # hexwall reads the SNI from $sub; $msg is for humans reading notice.log. + NOTICE([$note=PossibleBypass, + $msg=fmt("SNI %s from %s with no prior DNS query", sni, client), + $sub=sni, + $src=client, + $dst=c$id$resp_h, + $p=c$id$resp_p, + $conn=c]); + } diff --git a/internal/zeek/notice.go b/internal/zeek/notice.go index 76a2898..a9cdc1e 100644 --- a/internal/zeek/notice.go +++ b/internal/zeek/notice.go @@ -1,82 +1,249 @@ package zeek import ( - "bufio" + "bytes" "context" + _ "embed" "encoding/json" "errors" "fmt" "io" "log/slog" + "math" "os" + "strconv" "strings" "time" ) +// DNSBypassScript is the Zeek script that writes the notices this file reads. It ships +// inside the hexwall binary (installed by `hexwall zeek-setup`) so the script and this +// parser can never drift apart: the script puts the SNI in the notice's sub field, and +// parseLine reads it from there. +// +//go:embed dns-bypass.zeek +var DNSBypassScript []byte + +// bypassNote is the notice type raised by DNSBypassScript. notice.log carries every Zeek +// notice (invalid certificates, scans, ...); only this one is a DNS-bypass alert. +const bypassNote = "DNSBypass::PossibleBypass" + var ( - // ErrNoticeLineEmpty indicates a notice log line was blank. - ErrNoticeLineEmpty = errors.New("empty notice line") - // ErrNoticeLineNoFields indicates a notice log line parsed as JSON but had none of the recognized fields. - ErrNoticeLineNoFields = errors.New("notice line did not contain recognizable fields") + // ErrNoticeHeaderMissing indicates a TSV data line arrived before the "#fields" header that names its columns. + ErrNoticeHeaderMissing = errors.New("tsv notice line before its #fields header") + // ErrNoticeNoSNI indicates a DNS-bypass notice with no SNI in its sub field, e.g. one + // written by an older copy of the Zeek script; re-running `hexwall zeek-setup` fixes it. + ErrNoticeNoSNI = errors.New("dns-bypass notice without an SNI in sub; re-run hexwall zeek-setup") ) // Event is a parsed DNS bypass notice from Zeek. type Event struct { Timestamp time.Time SrcIP string + SrcPort string DstIP string DstPort string SNI string } -// ParseNoticeLine parses a single Zeek notice JSON line into an Event. -func ParseNoticeLine(line string) (Event, error) { - line = strings.TrimSpace(line) - if line == "" { - return Event{}, ErrNoticeLineEmpty +// noticeParser turns notice.log lines into Events, in either of Zeek's log formats: TSV +// (the default), read against the column order of the file's "#fields" header, or JSON +// (LogAscii::use_json = T). It remembers the header between calls, so it must be fed one +// file's lines in order. +type noticeParser struct { + tsvFields map[string]int +} + +// parseLine returns ok=false with a nil error for lines that are not DNS-bypass alerts: +// blank lines, header lines, and other notice types. +func (p *noticeParser) parseLine(line string) (event Event, ok bool, err error) { + line = strings.TrimSuffix(line, "\r") + switch { + case strings.TrimSpace(line) == "": + return Event{}, false, nil + case strings.HasPrefix(line, "#"): + if fields := parseTSVFieldsLine(line); fields != nil { + p.tsvFields = fields + } + return Event{}, false, nil } - var payload map[string]any - if err := json.Unmarshal([]byte(line), &payload); err != nil { - return Event{}, fmt.Errorf("parse notice line: %w", err) + fields, err := p.fields(line) + if err != nil { + return Event{}, false, err + } + if fields["note"] != bypassNote { + return Event{}, false, nil } - event := Event{ - Timestamp: time.Now(), + if fields["sub"] == "" { + return Event{}, false, ErrNoticeNoSNI } - if ts, ok := payload["ts"].(string); ok { - if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil { - event.Timestamp = parsed + return Event{ + Timestamp: parseZeekTime(fields["ts"]), + SrcIP: firstNonEmpty(fields["src"], fields["id.orig_h"]), + SrcPort: fields["id.orig_p"], + DstIP: firstNonEmpty(fields["dst"], fields["id.resp_h"]), + DstPort: firstNonEmpty(fields["p"], fields["id.resp_p"]), + SNI: fields["sub"], + }, true, nil +} + +// fields maps column name to value for one data line. Unset values are left out. +func (p *noticeParser) fields(line string) (map[string]string, error) { + if strings.HasPrefix(line, "{") { + decoder := json.NewDecoder(strings.NewReader(line)) + // Keeps ts as "1398362902.726337" rather than a float64 printed as 1.398362902726337e+09. + decoder.UseNumber() + + var payload map[string]any + if err := decoder.Decode(&payload); err != nil { + return nil, fmt.Errorf("parse json notice line: %w", err) + } + + fields := make(map[string]string, len(payload)) + for name, value := range payload { + fields[name] = fmt.Sprint(value) } + return fields, nil + } + + if p.tsvFields == nil { + return nil, ErrNoticeHeaderMissing } - if ts, ok := payload["timestamp"].(string); ok { - if parsed, err := time.Parse(time.RFC3339Nano, ts); err == nil { - event.Timestamp = parsed + cols := strings.Split(line, "\t") + fields := make(map[string]string, len(p.tsvFields)) + for name, i := range p.tsvFields { + // "-" is Zeek's unset-field marker, "(empty)" its empty-set marker. + if i < len(cols) && cols[i] != "-" && cols[i] != "(empty)" { + fields[name] = cols[i] } } + return fields, nil +} - event.SrcIP = stringValue(payload["src_ip"], payload["src"], payload["id.orig_h"]) - event.DstIP = stringValue(payload["dst_ip"], payload["dst"], payload["id.resp_h"]) - event.DstPort = stringValue(payload["dst_port"], payload["p"], payload["id.resp_p"]) - event.SNI = stringValue(payload["sni"], payload["server_name"]) +// parseZeekTime reads Zeek's default epoch-seconds timestamp, or an RFC 3339 one (the +// LogAscii::json_timestamps option). It falls back to now so an event is never undated. +func parseZeekTime(value string) time.Time { + if seconds, err := strconv.ParseFloat(value, 64); err == nil { + whole, frac := math.Modf(seconds) + return time.Unix(int64(whole), int64(frac*1e9)) + } + if parsed, err := time.Parse(time.RFC3339Nano, value); err == nil { + return parsed + } + return time.Now() +} - if event.SrcIP == "" && event.DstIP == "" && event.SNI == "" { - return Event{}, ErrNoticeLineNoFields +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } } + return "" +} + +// noticeTailer reads notice.log from where it last stopped. +type noticeTailer struct { + path string + offset int64 + parser noticeParser + warnedBadLine bool +} + +// skipExisting reads the file as it is now without sending anything: the parser still +// learns the TSV "#fields" header at the top, but alerts already in the file are not +// replayed. In enforce mode an old alert would otherwise trigger a kill on every restart. +func (t *noticeTailer) skipExisting() { + t.read(nil) +} - return event, nil +// poll reads every complete line added since the last poll and sends its DNS-bypass alerts. +func (t *noticeTailer) poll(events chan<- Event) { + t.read(events) } -// WatchNoticeLog tails a Zeek notice log and emits parsed Events. +// read consumes the complete lines past t.offset. With a nil events channel it only +// advances the offset and the parser's header. +func (t *noticeTailer) read(events chan<- Event) { + file, err := os.Open(t.path) + if err != nil { + slog.Debug("zeek: cannot open notice log", "path", t.path, "err", err) + return + } + defer func() { _ = file.Close() }() + + fi, err := file.Stat() + if err != nil { + slog.Debug("zeek: cannot stat notice log", "path", t.path, "err", err) + return + } + + // Smaller than what was already read: the log was rotated. The new file starts with + // its own header, so forget the old one too. + if fi.Size() < t.offset { + t.offset = 0 + t.parser = noticeParser{} + } + if fi.Size() == t.offset { + return + } + + if _, err := file.Seek(t.offset, io.SeekStart); err != nil { + slog.Debug("zeek: seek notice log failed", "path", t.path, "err", err) + return + } + data, err := io.ReadAll(file) + if err != nil { + slog.Debug("zeek: read notice log failed", "path", t.path, "err", err) + return + } + + // Zeek may still be writing the last line. Stop at the last newline; the unfinished + // line is read again, whole, on a later poll. + end := bytes.LastIndexByte(data, '\n') + if end == -1 { + return + } + t.offset += int64(end + 1) + + for _, line := range strings.Split(string(data[:end]), "\n") { + event, ok, err := t.parser.parseLine(line) + if err != nil { + // Warn once: a format mismatch fails every line, and one warning is enough + // to show that alerts are being missed. + if !t.warnedBadLine { + slog.Warn("zeek: notice log line could not be parsed; DNS-bypass alerts may be missed", "path", t.path, "err", err) + t.warnedBadLine = true + } else { + slog.Debug("zeek: notice log line could not be parsed", "path", t.path, "err", err) + } + continue + } + if !ok || events == nil { + continue + } + + select { + case events <- event: + default: + slog.Warn("zeek: DNS-bypass alert dropped, alert handler is behind", "sni", event.SNI, "dst_ip", event.DstIP) + } + } +} + +// WatchNoticeLog tails a Zeek notice log and emits parsed DNS-bypass Events. func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error { logPath := strings.TrimSpace(path) if ctx == nil || logPath == "" { return nil } - var offset int64 + tailer := ¬iceTailer{path: logPath} + tailer.skipExisting() poller := time.NewTicker(2 * time.Second) defer poller.Stop() @@ -85,68 +252,7 @@ func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error case <-ctx.Done(): return nil case <-poller.C: - file, err := os.Open(logPath) - if err != nil { - slog.Debug("zeek: cannot open notice log", "path", logPath, "err", err) - continue - } - - fi, err := file.Stat() - if err != nil { - _ = file.Close() - slog.Debug("zeek: cannot stat notice log", "path", logPath, "err", err) - continue - } - - if fi.Size() < offset { - offset = 0 - } - if fi.Size() > offset { - if _, err := file.Seek(offset, io.SeekStart); err != nil { - _ = file.Close() - slog.Debug("zeek: seek notice log failed", "path", logPath, "err", err) - continue - } - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - - event, err := ParseNoticeLine(line) - if err != nil { - continue - } - - select { - case events <- event: - default: - } - } - if err := scanner.Err(); err != nil { - slog.Debug("zeek: notice log scan failed", "path", logPath, "err", err) - } - } - _ = file.Close() - offset = fi.Size() + tailer.poll(events) } } } - -func stringValue(values ...any) string { - for _, value := range values { - switch v := value.(type) { - case string: - return strings.TrimSpace(v) - case fmt.Stringer: - return strings.TrimSpace(v.String()) - case nil: - continue - default: - return fmt.Sprint(v) - } - } - return "" -} diff --git a/internal/zeek/notice_test.go b/internal/zeek/notice_test.go index 0aa6769..b8046b4 100644 --- a/internal/zeek/notice_test.go +++ b/internal/zeek/notice_test.go @@ -1,41 +1,239 @@ package zeek import ( + "errors" + "os" + "path/filepath" + "strings" "testing" "time" ) -func TestParseNoticeLine(t *testing.T) { - line := `{"note":"DNSBypass::PossibleBypass","src_ip":"192.0.2.10","dst_ip":"203.0.113.20","dst_port":"443","sni":"evil.example","msg":"SNI evil.example from 192.0.2.10"}` +// The testdata notice logs are real Zeek 9.0.0 output: this package's dns-bypass.zeek run +// over Zeek's own chrome-34-google.pcap, once with default settings (TSV) and once with +// LogAscii::use_json=T. Regenerate them whenever the script changes: +// +// zeek -C -r /opt/zeek/share/btest/data/pcaps/tls/chrome-34-google.pcap dns-bypass.zeek [LogAscii::use_json=T] - event, err := ParseNoticeLine(line) +func readFixtureLines(t *testing.T, name string) []string { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", name)) if err != nil { - t.Fatalf("ParseNoticeLine() error = %v", err) + t.Fatalf("read fixture %s: %v", name, err) } + return strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") +} + +func parseAll(t *testing.T, lines []string) []Event { + t.Helper() + var parser noticeParser + var events []Event + for _, line := range lines { + event, ok, err := parser.parseLine(line) + if err != nil { + t.Fatalf("parseLine(%q) error = %v", line, err) + } + if ok { + events = append(events, event) + } + } + return events +} - if event.SrcIP != "192.0.2.10" { - t.Fatalf("ParseNoticeLine().SrcIP = %q, want %q", event.SrcIP, "192.0.2.10") +func assertGoogleBypassEvent(t *testing.T, events []Event) { + t.Helper() + if len(events) != 1 { + t.Fatalf("got %d events, want 1: %+v", len(events), events) + } + event := events[0] + + if event.SNI != "google.de" { + t.Errorf("SNI = %q, want %q", event.SNI, "google.de") + } + if event.SrcIP != "192.168.4.149" { + t.Errorf("SrcIP = %q, want %q", event.SrcIP, "192.168.4.149") + } + if event.SrcPort != "55881" { + t.Errorf("SrcPort = %q, want %q", event.SrcPort, "55881") } - if event.DstIP != "203.0.113.20" { - t.Fatalf("ParseNoticeLine().DstIP = %q, want %q", event.DstIP, "203.0.113.20") + if event.DstIP != "74.125.239.152" { + t.Errorf("DstIP = %q, want %q", event.DstIP, "74.125.239.152") } if event.DstPort != "443" { - t.Fatalf("ParseNoticeLine().DstPort = %q, want %q", event.DstPort, "443") + t.Errorf("DstPort = %q, want %q", event.DstPort, "443") + } + want := time.Unix(1398362902, 726337000) + if diff := event.Timestamp.Sub(want); diff < -time.Microsecond || diff > time.Microsecond { + t.Errorf("Timestamp = %v, want %v (Zeek's ts, not the parse time)", event.Timestamp, want) + } +} + +func TestNoticeParser_RealTSV(t *testing.T) { + assertGoogleBypassEvent(t, parseAll(t, readFixtureLines(t, "notice.tsv"))) +} + +func TestNoticeParser_RealJSON(t *testing.T) { + assertGoogleBypassEvent(t, parseAll(t, readFixtureLines(t, "notice.ndjson"))) +} + +func TestNoticeParser_TSVLineBeforeHeader(t *testing.T) { + var dataLine string + for _, line := range readFixtureLines(t, "notice.tsv") { + if !strings.HasPrefix(line, "#") { + dataLine = line + } + } + + var parser noticeParser + if _, _, err := parser.parseLine(dataLine); !errors.Is(err, ErrNoticeHeaderMissing) { + t.Fatalf("parseLine() error = %v, want ErrNoticeHeaderMissing", err) + } +} + +func TestNoticeParser_IgnoresOtherNoticeTypes(t *testing.T) { + line := readFixtureLines(t, "notice.ndjson")[0] + line = strings.Replace(line, bypassNote, "SSL::Invalid_Server_Cert", 1) + + var parser noticeParser + _, ok, err := parser.parseLine(line) + if err != nil || ok { + t.Fatalf("parseLine() = ok %v, err %v; want a silent skip", ok, err) + } +} + +// A notice from an older copy of the script (SNI only in msg, no sub) must fail loudly, +// not be skipped: the warning tells the user to re-run zeek-setup. +func TestNoticeParser_BypassWithoutSub(t *testing.T) { + line := readFixtureLines(t, "notice.ndjson")[0] + line = strings.Replace(line, `"sub":"google.de",`, "", 1) + + var parser noticeParser + if _, _, err := parser.parseLine(line); !errors.Is(err, ErrNoticeNoSNI) { + t.Fatalf("parseLine() error = %v, want ErrNoticeNoSNI", err) + } +} + +func TestNoticeParser_InvalidJSON(t *testing.T) { + var parser noticeParser + if _, _, err := parser.parseLine(`{"note":`); err == nil { + t.Fatal("parseLine() error = nil, want error") + } +} + +// Zeek can be mid-write when hexwall polls. A half-written line must not be parsed (its +// SNI would be cut short); it must be read whole on the next poll. +func TestNoticeTailer_WaitsForCompleteLine(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", "notice.tsv")) + if err != nil { + t.Fatal(err) + } + cut := strings.Index(string(data), "google.de") + len("goo") + + path := filepath.Join(t.TempDir(), "notice.log") + if err := os.WriteFile(path, data[:cut], 0o600); err != nil { + t.Fatal(err) + } + + events := make(chan Event, 10) + tailer := ¬iceTailer{path: path} + + tailer.poll(events) + if len(events) != 0 { + t.Fatalf("got %d events from a half-written line, want 0", len(events)) + } + + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + tailer.poll(events) + + close(events) + var got []Event + for event := range events { + got = append(got, event) + } + assertGoogleBypassEvent(t, got) +} + +func TestNoticeTailer_RelearnsHeaderAfterRotation(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", "notice.tsv")) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "notice.log") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + + events := make(chan Event, 10) + tailer := ¬iceTailer{path: path} + tailer.poll(events) + <-events + + // Rotation: a fresh, smaller file with its own header. + var header strings.Builder + var dataLine string + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "#fields") || strings.HasPrefix(line, "#separator") { + header.WriteString(line + "\n") + } else if line != "" && !strings.HasPrefix(line, "#") { + dataLine = line + } + } + if err := os.WriteFile(path, []byte(header.String()+dataLine+"\n"), 0o600); err != nil { + t.Fatal(err) + } + tailer.poll(events) + + if len(events) != 1 { + t.Fatalf("got %d events after rotation, want 1", len(events)) + } +} + +// Alerts already in notice.log when hexwall starts must not be replayed: in enforce mode +// each one would trigger a kill attempt on every restart. Alerts written afterwards must +// still be read, which needs the header learned during the skip. +func TestNoticeTailer_SkipsExistingAlerts(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", "notice.tsv")) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "notice.log") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + + events := make(chan Event, 10) + tailer := ¬iceTailer{path: path} + tailer.skipExisting() + tailer.poll(events) + if len(events) != 0 { + t.Fatalf("got %d replayed events, want 0", len(events)) } - if event.SNI != "evil.example" { - t.Fatalf("ParseNoticeLine().SNI = %q, want %q", event.SNI, "evil.example") + + var dataLine string + for _, line := range strings.Split(string(data), "\n") { + if line != "" && !strings.HasPrefix(line, "#") { + dataLine = line + } + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) } - if event.Timestamp.IsZero() { - t.Fatal("ParseNoticeLine().Timestamp = zero time, want non-zero") + if _, err := file.WriteString(dataLine + "\n"); err != nil { + t.Fatal(err) } - if event.Timestamp.After(time.Now().Add(time.Second)) { - t.Fatalf("ParseNoticeLine().Timestamp too far in the future: %v", event.Timestamp) + _ = file.Close() + + tailer.poll(events) + if len(events) != 1 { + t.Fatalf("got %d events for a new alert, want 1", len(events)) } } -func TestParseNoticeLineRejectsInvalidJSON(t *testing.T) { - _, err := ParseNoticeLine(`{"note":`) - if err == nil { - t.Fatal("ParseNoticeLine() error = nil, want error") +func TestDNSBypassScriptEmbedded(t *testing.T) { + if !strings.Contains(string(DNSBypassScript), "$sub=sni") { + t.Fatal("embedded script does not write the SNI to sub, which parseLine reads") } } diff --git a/internal/zeek/testdata/notice.ndjson b/internal/zeek/testdata/notice.ndjson new file mode 100644 index 0000000..ac83674 --- /dev/null +++ b/internal/zeek/testdata/notice.ndjson @@ -0,0 +1 @@ +{"ts":1398362902.726337,"uid":"CCLHJAgTLjzZMS085","id.orig_h":"192.168.4.149","id.orig_p":55881,"id.resp_h":"74.125.239.152","id.resp_p":443,"proto":"tcp","note":"DNSBypass::PossibleBypass","msg":"SNI google.de from 192.168.4.149 with no prior DNS query","sub":"google.de","src":"192.168.4.149","dst":"74.125.239.152","p":443,"actions":["Notice::ACTION_LOG"],"email_dest":[],"suppress_for":3600.0} diff --git a/internal/zeek/testdata/notice.tsv b/internal/zeek/testdata/notice.tsv new file mode 100644 index 0000000..e3bd261 --- /dev/null +++ b/internal/zeek/testdata/notice.tsv @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path notice +#open 2026-10-11-14-47-56 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions email_dest suppress_for remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] set[string] interval string string string double double +1398362902.726337 CNVr1M3g8O7l7ULm2 192.168.4.149 55881 74.125.239.152 443 - - - tcp DNSBypass::PossibleBypass SNI google.de from 192.168.4.149 with no prior DNS query google.de 192.168.4.149 74.125.239.152 443 - - Notice::ACTION_LOG (empty) 3600.000000 - - - - - +#close 2026-10-11-14-47-56 From b1c7ab0e9b0eddd293fbe42ff41d380d4cb001d9 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 16:00:39 +0600 Subject: [PATCH 37/56] feat(monitor): kill connections behind blocked Zeek bypass alerts A bypass alert for a Pi-hole-blocked domain was only logged, even in enforce mode. The alert's 4-tuple is now looked up among this host's sockets and, when found, goes through applyBlockedOutcome, the same guarded kill path the decision ladder uses. Alerts about other devices can't be killed here; they are recorded as not_on_this_host with a warning. --- internal/monitor/monitor.go | 52 ++++++++++++++++++++-- internal/monitor/monitor_test.go | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 76bcb63..175b4ae 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -89,7 +89,12 @@ func remoteIP(address string) (net.IP, error) { return ip, nil } -// HandleZeekEvent evaluates a Zeek notice event against the current policy and persists it. +// HandleZeekEvent evaluates a Zeek DNS-bypass notice against Pi-hole policy and persists it. +// +// A notice whose SNI is policy-blocked is handled like rung 4: the connection is killed in +// enforce mode and reported in watch mode. hexwall can only close sockets on the machine +// it runs on, so the connection is first looked up in this machine's socket list; a notice +// about another device (Zeek watching a mirrored port, say) is recorded but not acted on. func (s *Scanner) HandleZeekEvent(event zeek.Event) { if s.Checker == nil || s.Store == nil { return @@ -107,21 +112,62 @@ func (s *Scanner) HandleZeekEvent(event zeek.Event) { confidence := "medium" actionTaken := "logged" + var conn connlist.Connection + found := false if blocked { confidence = "high" slog.Warn("zeek bypass alert", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI, "mode", selectedMode) + + conn, found = s.findLocalConnection(event) + switch { + case !found: + actionTaken = "not_on_this_host" + slog.Warn("zeek bypass alert: connection is not on this machine (another device, or already closed); hexwall can only kill its own machine's connections", + "src_ip", event.SrcIP, "src_port", event.SrcPort, "dst_ip", event.DstIP, "sni", event.SNI) + case selectedMode == ModeEnforce: + actionTaken = "kill_attempted" + } } else { slog.Info("zeek notice was not policy-blocked", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI) } + // Persist before acting, so the alert is on record even if the kill goes wrong. if err := s.Store.LogZeekAlert(event.SrcIP, event.DstIP, event.DstPort, event.SNI, blocked, confidence, actionTaken); err != nil { slog.Error("failed to persist zeek alert", "src_ip", event.SrcIP, "sni", event.SNI, "err", err) } - if selectedMode == ModeEnforce && blocked { - slog.Info("enforce mode: zeek alert recorded without direct connection kill", "src_ip", event.SrcIP, "sni", event.SNI) + if found { + if ip, err := remoteIP(conn.RAddress); err == nil { + s.applyBlockedOutcome(ip.String(), conn) + } + } +} + +// findLocalConnection looks a Zeek notice's connection up in this machine's socket list. +// Zeek names a connection from the client's side (orig port -> resp IP:port); for a +// connection this machine opened, that is its local port and remote address. +func (s *Scanner) findLocalConnection(event zeek.Event) (connlist.Connection, bool) { + wantIP := net.ParseIP(event.DstIP) + if s.ConnectionLister == nil || event.SrcPort == "" || wantIP == nil { + return connlist.Connection{}, false + } + + connections, err := s.ConnectionLister() + if err != nil { + slog.Warn("zeek bypass alert: could not list connections", "err", err) + return connlist.Connection{}, false + } + + for _, conn := range connections { + if conn.LPort != event.SrcPort || conn.RPort != event.DstPort { + continue + } + if ip, err := remoteIP(conn.RAddress); err == nil && ip.Equal(wantIP) { + return conn, true + } } + return connlist.Connection{}, false } // Run inspects established connections and applies the decision ladder. diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index 9322d98..e77782f 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -326,6 +326,81 @@ func TestHandleZeekEventNoopCases(t *testing.T) { (&Scanner{Checker: checker, Store: hexwallStore, Mode: ModeWatch}).HandleZeekEvent(zeek.Event{SrcIP: "10.0.0.1", SNI: ""}) } +// A blocked DNS-bypass alert for a connection this machine owns is killed in enforce mode. +// The connection's PID is 1, which the enforcer's guardrails always refuse, so no real +// process is signalled; the audit row proves the kill path was taken. +func TestHandleZeekEventEnforceKillsLocalConnection(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "evil.example.com", 1) // denylist + }) + hexwallStore, dbPath := newTestStore(t) + + local := connlist.Connection{LPort: "55881", RAddress: "203.0.113.20:443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{local}, nil }, + } + event := zeek.Event{SrcIP: "192.168.0.223", SrcPort: "55881", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"} + scanner.HandleZeekEvent(event) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.20"); got != 1 { + t.Fatalf("killed_connections count = %d, want 1", got) + } + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil || alert == nil { + t.Fatalf("RecentZeekAlert() = %v, %v; want persisted alert", alert, err) + } + if alert.ActionTaken != "kill_attempted" { + t.Fatalf("alert.ActionTaken = %q, want %q", alert.ActionTaken, "kill_attempted") + } +} + +// A connection that isn't in this machine's socket list belongs to another device (or has +// closed). hexwall cannot close a socket on another machine, so it records and warns. +func TestHandleZeekEventEnforceOtherDeviceNotKilled(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "evil.example.com", 1) + }) + hexwallStore, dbPath := newTestStore(t) + + unrelated := connlist.Connection{LPort: "40000", RAddress: "203.0.113.20:443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{unrelated}, nil }, + } + event := zeek.Event{SrcIP: "192.168.0.50", SrcPort: "55881", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"} + scanner.HandleZeekEvent(event) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0", got) + } + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil || alert == nil { + t.Fatalf("RecentZeekAlert() = %v, %v; want persisted alert", alert, err) + } + if alert.ActionTaken != "not_on_this_host" { + t.Fatalf("alert.ActionTaken = %q, want %q", alert.ActionTaken, "not_on_this_host") + } +} + +func TestHandleZeekEventWatchModeDoesNotKill(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "evil.example.com", 1) + }) + hexwallStore, dbPath := newTestStore(t) + + local := connlist.Connection{LPort: "55881", RAddress: "203.0.113.20:443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeWatch, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{local}, nil }, + } + scanner.HandleZeekEvent(zeek.Event{SrcIP: "192.168.0.223", SrcPort: "55881", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"}) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0 in watch mode", got) + } +} + // ---- applyBlockedOutcome --------------------------------------------------- func TestApplyBlockedOutcomeWatchModeNeverLogsKill(t *testing.T) { From bb6c92863fcc938fff6799fd52d244e7b2dca0b6 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 16:00:39 +0600 Subject: [PATCH 38/56] feat: add hexwall zeek-setup Installing the DNS-bypass script was a manual step documented nowhere users would look. `sudo hexwall zeek-setup` writes the embedded script into Zeek's site directory, loads it from local.zeek once, checks that node.cfg's interfaces exist, and prints the zeekctl deploy step and the hexwall flags for the configured log directory. It does not restart Zeek or edit node.cfg. --- internal/zeek/setup.go | 209 ++++++++++++++++++++++++++++++++++++ internal/zeek/setup_test.go | 110 +++++++++++++++++++ main.go | 10 ++ 3 files changed, 329 insertions(+) create mode 100644 internal/zeek/setup.go create mode 100644 internal/zeek/setup_test.go diff --git a/internal/zeek/setup.go b/internal/zeek/setup.go new file mode 100644 index 0000000..61f6ca1 --- /dev/null +++ b/internal/zeek/setup.go @@ -0,0 +1,209 @@ +package zeek + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "net" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + scriptName = "dns-bypass.zeek" + loadLine = "@load dns-bypass" + // Where Zeek's binary packages install; under sudo, /opt/zeek/bin is usually not on PATH. + defaultZeekConfig = "/opt/zeek/bin/zeek-config" +) + +// Setup installs DNSBypassScript into a local Zeek, loads it from local.zeek, and checks +// that the interfaces Zeek is told to watch exist. It prints what it did, what is still +// wrong, and the hexwall flags that read the resulting logs. It does not restart Zeek. +func Setup(out io.Writer) error { + zeekConfig, err := findZeekConfig() + if err != nil { + return err + } + siteDir, err := zeekConfigValue(zeekConfig, "--site_dir") + if err != nil { + return err + } + prefix, err := zeekConfigValue(zeekConfig, "--prefix") + if err != nil { + return err + } + + changed, err := installScript(siteDir, DNSBypassScript) + if err != nil { + return err + } + if changed { + _, _ = fmt.Fprintf(out, "✓ installed %s\n", filepath.Join(siteDir, scriptName)) + } else { + _, _ = fmt.Fprintf(out, "✓ %s already up to date\n", filepath.Join(siteDir, scriptName)) + } + + localZeek := filepath.Join(siteDir, "local.zeek") + added, err := ensureLoaded(localZeek) + if err != nil { + return err + } + if added { + _, _ = fmt.Fprintf(out, "✓ added %q to %s\n", loadLine, localZeek) + } else { + _, _ = fmt.Fprintf(out, "✓ %s already loads the script\n", localZeek) + } + + nodeCfg := filepath.Join(prefix, "etc", "node.cfg") + missing, err := missingInterfaces(nodeCfg, interfaceExists) + if err != nil { + _, _ = fmt.Fprintf(out, "! could not check %s: %v\n", nodeCfg, err) + } + for _, name := range missing { + _, _ = fmt.Fprintf(out, "✗ %s tells Zeek to watch %q, which does not exist on this machine (available: %s)\n", + nodeCfg, name, strings.Join(interfaceNames(), ", ")) + } + + logDir := zeekLogDir(prefix) + _, _ = fmt.Fprintf(out, ` +Next steps: + 1. sudo %s deploy (restarts Zeek with the script loaded) + 2. run hexwall with: + --zeek-log %s --enable-zeek --zeek-notice-log %s +`, + filepath.Join(prefix, "bin", "zeekctl"), + filepath.Join(logDir, "current", "ssl.log"), + filepath.Join(logDir, "current", "notice.log")) + + if len(missing) > 0 { + return fmt.Errorf("fix the interface in %s before deploying: Zeek cannot start on an interface that does not exist", nodeCfg) + } + return nil +} + +func findZeekConfig() (string, error) { + if path, err := exec.LookPath("zeek-config"); err == nil { + return path, nil + } + if _, err := os.Stat(defaultZeekConfig); err == nil { + return defaultZeekConfig, nil + } + return "", errors.New("zeek-config not found on PATH or at " + defaultZeekConfig + "; is Zeek installed?") +} + +func zeekConfigValue(zeekConfig, flag string) (string, error) { + out, err := exec.Command(zeekConfig, flag).Output() + if err != nil { + return "", fmt.Errorf("%s %s: %w", zeekConfig, flag, err) + } + return strings.TrimSpace(string(out)), nil +} + +// installScript writes script into siteDir unless an identical copy is already there. +func installScript(siteDir string, script []byte) (changed bool, err error) { + path := filepath.Join(siteDir, scriptName) + if current, err := os.ReadFile(path); err == nil && bytes.Equal(current, script) { + return false, nil + } + if err := os.WriteFile(path, script, 0o644); err != nil { + return false, permissionHint(fmt.Errorf("write %s: %w", path, err)) + } + return true, nil +} + +// ensureLoaded appends the @load line to local.zeek unless it is already there. +func ensureLoaded(localZeek string) (added bool, err error) { + data, err := os.ReadFile(localZeek) + if err != nil { + return false, permissionHint(fmt.Errorf("read %s: %w", localZeek, err)) + } + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) == loadLine { + return false, nil + } + } + + addition := "\n# Added by hexwall zeek-setup: DNS-bypass notices that hexwall reads.\n" + loadLine + "\n" + if len(data) > 0 && !bytes.HasSuffix(data, []byte("\n")) { + addition = "\n" + addition + } + file, err := os.OpenFile(localZeek, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + return false, permissionHint(fmt.Errorf("open %s: %w", localZeek, err)) + } + if _, err := file.WriteString(addition); err != nil { + _ = file.Close() + return false, fmt.Errorf("append to %s: %w", localZeek, err) + } + return true, file.Close() +} + +// missingInterfaces returns the interfaces named in node.cfg that exists reports as absent. +// Newer zeekctl accepts a packet-source prefix ("af_packet::eth0"); only the name is checked. +func missingInterfaces(nodeCfg string, exists func(string) bool) ([]string, error) { + file, err := os.Open(nodeCfg) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + + var missing []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + key, value, ok := strings.Cut(line, "=") + if !ok || strings.HasPrefix(line, "#") || strings.TrimSpace(key) != "interface" { + continue + } + name := strings.TrimSpace(value) + if i := strings.LastIndex(name, "::"); i >= 0 { + name = name[i+2:] + } + if name != "" && !exists(name) { + missing = append(missing, name) + } + } + return missing, scanner.Err() +} + +func interfaceExists(name string) bool { + _, err := net.InterfaceByName(name) + return err == nil +} + +func interfaceNames() []string { + interfaces, err := net.Interfaces() + if err != nil { + return nil + } + names := make([]string, 0, len(interfaces)) + for _, iface := range interfaces { + names = append(names, iface.Name) + } + return names +} + +// zeekLogDir reads LogDir from zeekctl.cfg, falling back to zeekctl's default. +func zeekLogDir(prefix string) string { + if data, err := os.ReadFile(filepath.Join(prefix, "etc", "zeekctl.cfg")); err == nil { + for _, line := range strings.Split(string(data), "\n") { + key, value, ok := strings.Cut(line, "=") + if ok && strings.EqualFold(strings.TrimSpace(key), "LogDir") { + return strings.TrimSpace(value) + } + } + } + return filepath.Join(prefix, "logs") +} + +func permissionHint(err error) error { + if errors.Is(err, fs.ErrPermission) { + return fmt.Errorf("%w (run with sudo)", err) + } + return err +} diff --git a/internal/zeek/setup_test.go b/internal/zeek/setup_test.go new file mode 100644 index 0000000..b61033b --- /dev/null +++ b/internal/zeek/setup_test.go @@ -0,0 +1,110 @@ +package zeek + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestInstallScript_WritesThenSkipsIdentical(t *testing.T) { + siteDir := t.TempDir() + + changed, err := installScript(siteDir, DNSBypassScript) + if err != nil || !changed { + t.Fatalf("first installScript() = %v, %v; want true, nil", changed, err) + } + got, err := os.ReadFile(filepath.Join(siteDir, scriptName)) + if err != nil || string(got) != string(DNSBypassScript) { + t.Fatalf("installed script differs from the embedded one (err %v)", err) + } + + changed, err = installScript(siteDir, DNSBypassScript) + if err != nil || changed { + t.Fatalf("second installScript() = %v, %v; want false, nil", changed, err) + } +} + +// An older copy (e.g. one without $sub) must be replaced, or hexwall can't read its notices. +func TestInstallScript_ReplacesOutdatedCopy(t *testing.T) { + siteDir := t.TempDir() + if err := os.WriteFile(filepath.Join(siteDir, scriptName), []byte("# old version\n"), 0o644); err != nil { + t.Fatal(err) + } + + changed, err := installScript(siteDir, DNSBypassScript) + if err != nil || !changed { + t.Fatalf("installScript() = %v, %v; want true, nil", changed, err) + } +} + +func TestEnsureLoaded_AppendsOnce(t *testing.T) { + localZeek := filepath.Join(t.TempDir(), "local.zeek") + original := "@load misc/detect-traceroute\n@load tuning/defaults" // no trailing newline + if err := os.WriteFile(localZeek, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + for i, want := range []bool{true, false} { + added, err := ensureLoaded(localZeek) + if err != nil || added != want { + t.Fatalf("ensureLoaded() call %d = %v, %v; want %v, nil", i+1, added, err, want) + } + } + + data, _ := os.ReadFile(localZeek) + lines := strings.Split(string(data), "\n") + if !slices.Contains(lines, "@load tuning/defaults") { + t.Fatalf("last existing line was mangled:\n%s", data) + } + if strings.Count(string(data), loadLine) != 1 { + t.Fatalf("want exactly one %q line:\n%s", loadLine, data) + } +} + +func TestMissingInterfaces(t *testing.T) { + nodeCfg := filepath.Join(t.TempDir(), "node.cfg") + // Shape of zeekctl's default standalone node.cfg, plus a worker using a packet-source prefix. + content := `# Example ZeekControl node configuration. +[zeek] +type=standalone +host=localhost +interface=eth0 + +[worker-1] +type=worker +interface=af_packet::wlx001ea6c37e44 +#interface=commented0 +` + if err := os.WriteFile(nodeCfg, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + exists := func(name string) bool { return name == "wlx001ea6c37e44" } + missing, err := missingInterfaces(nodeCfg, exists) + if err != nil { + t.Fatalf("missingInterfaces() error = %v", err) + } + if !slices.Equal(missing, []string{"eth0"}) { + t.Fatalf("missingInterfaces() = %v, want [eth0]", missing) + } +} + +func TestZeekLogDir(t *testing.T) { + prefix := t.TempDir() + if got, want := zeekLogDir(prefix), filepath.Join(prefix, "logs"); got != want { + t.Fatalf("zeekLogDir() without zeekctl.cfg = %q, want %q", got, want) + } + + if err := os.MkdirAll(filepath.Join(prefix, "etc"), 0o755); err != nil { + t.Fatal(err) + } + cfg := "# Log rotation\nLogRotationInterval = 3600\nLogDir = /var/log/zeek\n" + if err := os.WriteFile(filepath.Join(prefix, "etc", "zeekctl.cfg"), []byte(cfg), 0o644); err != nil { + t.Fatal(err) + } + if got := zeekLogDir(prefix); got != "/var/log/zeek" { + t.Fatalf("zeekLogDir() = %q, want %q", got, "/var/log/zeek") + } +} diff --git a/main.go b/main.go index 80c16bc..266308c 100644 --- a/main.go +++ b/main.go @@ -95,6 +95,16 @@ func parseFlags() (*runConfig, error) { } func run() int { + // `hexwall zeek-setup` installs the Zeek script whose notices hexwall reads, so the + // script always matches this binary's parser. + if len(os.Args) > 1 && os.Args[1] == "zeek-setup" { + if err := zeek.Setup(os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "zeek-setup:", err) + return 1 + } + return 0 + } + cfg, err := parseFlags() if err != nil { slog.Error(err.Error()) From 6d1ed05177250551f459021feeaa75ecf977891c Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 16:37:10 +0600 Subject: [PATCH 39/56] feat: add hexwall install-service Nothing restarted hexwall or Zeek after a reboot, so protection stopped silently. `sudo hexwall install-service [flags]` validates the flags, then writes hexwall.service (this binary's absolute path, state in /var/lib/hexwall, restart on failure) and, when the flags use Zeek, a zeekctl-driven zeek.service that hexwall is ordered after. A zeek.service hexwall didn't write is left alone. The unit runs in watch mode unless --mode is given explicitly. go run binaries are refused because they are deleted on exit. parseFlags now takes its arguments so install-service can validate the service's flags without running hexwall. --- internal/service/service.go | 210 +++++++++++++++++++++++++++++++ internal/service/service_test.go | 132 +++++++++++++++++++ internal/zeek/setup.go | 13 ++ main.go | 81 ++++++++++-- 4 files changed, 423 insertions(+), 13 deletions(-) create mode 100644 internal/service/service.go create mode 100644 internal/service/service_test.go diff --git a/internal/service/service.go b/internal/service/service.go new file mode 100644 index 0000000..3d53367 --- /dev/null +++ b/internal/service/service.go @@ -0,0 +1,210 @@ +// Package service installs systemd units so hexwall, and the Zeek it reads, survive a reboot. +package service + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + unitDir = "/etc/systemd/system" + hexwallUnit = "hexwall.service" + zeekUnit = "zeek.service" + // The first line of every unit hexwall writes. A zeek.service without it belongs to + // someone else and is never overwritten. + marker = "# Installed by hexwall install-service. Re-run it to update; edits here are overwritten." + // hexwall's state directory; systemd creates it (StateDirectory=hexwall) and starts + // hexwall there, so the default ./hexwall.db lands in it instead of in /. + stateDir = "/var/lib/hexwall" +) + +// Packaged Zeek units live here; one of them means Zeek already has a unit of its own. +var vendorUnitDirs = []string{"/usr/lib/systemd/system", "/lib/systemd/system"} + +// Options describes the units to install. +type Options struct { + Binary string // absolute path of the hexwall binary the unit runs + Args []string // validated hexwall flags, already passed through WithSafeMode + // Zeekctl is the zeekctl to run from zeek.service; empty when hexwall doesn't use Zeek. + Zeekctl string +} + +// Install writes the units, reloads systemd, enables them for boot, and (re)starts hexwall. +func Install(out io.Writer, opts Options) error { + if os.Geteuid() != 0 { + return errors.New("writing systemd units needs root (run with sudo)") + } + if _, err := exec.LookPath("systemctl"); err != nil { + return errors.New("systemctl not found: install-service only supports systemd") + } + return install(out, opts, unitDir, vendorUnitDirs, runSystemctl) +} + +func install(out io.Writer, opts Options, dir string, vendorDirs []string, systemctl func(args ...string) error) error { + withZeek := false + if opts.Zeekctl != "" { + switch owned, err := zeekUnitOwner(dir, vendorDirs); { + case err != nil: + return err + case owned: + changed, err := writeUnit(filepath.Join(dir, zeekUnit), renderZeekUnit(opts.Zeekctl)) + if err != nil { + return err + } + report(out, zeekUnit, dir, changed) + withZeek = true + default: + _, _ = fmt.Fprintf(out, "✓ %s already exists and isn't hexwall's; left as is\n", zeekUnit) + withZeek = true + } + } + + changed, err := writeUnit(filepath.Join(dir, hexwallUnit), renderHexwallUnit(opts.Binary, opts.Args, withZeek)) + if err != nil { + return err + } + report(out, hexwallUnit, dir, changed) + + steps := [][]string{{"daemon-reload"}} + if withZeek { + steps = append(steps, []string{"enable", "--now", zeekUnit}) + } + steps = append(steps, []string{"enable", hexwallUnit}, []string{"restart", hexwallUnit}) + for _, step := range steps { + if err := systemctl(step...); err != nil { + return fmt.Errorf("systemctl %s: %w", strings.Join(step, " "), err) + } + } + + _, _ = fmt.Fprintf(out, ` +hexwall now starts at boot and restarts if it crashes. + status: systemctl status %[1]s + logs: journalctl -u %[1]s -f + remove: sudo systemctl disable --now %[1]s +`, hexwallUnit) + return nil +} + +// zeekUnitOwner reports whether zeek.service is free for hexwall to write: either absent, +// or a unit hexwall wrote earlier. +func zeekUnitOwner(dir string, vendorDirs []string) (bool, error) { + for _, vendor := range vendorDirs { + if _, err := os.Stat(filepath.Join(vendor, zeekUnit)); err == nil { + return false, nil + } + } + data, err := os.ReadFile(filepath.Join(dir, zeekUnit)) + if errors.Is(err, os.ErrNotExist) { + return true, nil + } + if err != nil { + return false, err + } + return bytes.HasPrefix(data, []byte(marker)), nil +} + +func writeUnit(path, content string) (changed bool, err error) { + if current, err := os.ReadFile(path); err == nil && string(current) == content { + return false, nil + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + return false, fmt.Errorf("write %s: %w", path, err) + } + return true, nil +} + +func report(out io.Writer, unit, dir string, changed bool) { + if changed { + _, _ = fmt.Fprintf(out, "✓ wrote %s\n", filepath.Join(dir, unit)) + } else { + _, _ = fmt.Fprintf(out, "✓ %s already up to date\n", filepath.Join(dir, unit)) + } +} + +func runSystemctl(args ...string) error { + cmd := exec.Command("systemctl", args...) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + return cmd.Run() +} + +func renderHexwallUnit(binary string, args []string, withZeek bool) string { + after, wants := "network-online.target docker.service", "network-online.target" + if withZeek { + // Wants starts Zeek with hexwall; After makes hexwall wait for it, so ssl.log exists. + after += " " + zeekUnit + wants += " " + zeekUnit + } + execStart := make([]string, 0, len(args)+1) + for _, arg := range append([]string{binary}, args...) { + execStart = append(execStart, quoteExecArg(arg)) + } + return marker + ` +[Unit] +Description=hexwall outbound connection monitor +After=` + after + ` +Wants=` + wants + ` + +[Service] +ExecStart=` + strings.Join(execStart, " ") + ` +StateDirectory=hexwall +WorkingDirectory=` + stateDir + ` +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +` +} + +func renderZeekUnit(zeekctl string) string { + zeekctl = quoteExecArg(zeekctl) + // zeekctl starts Zeek in the background and exits, so the unit is a oneshot that stays + // "active" until stopped. + return marker + ` +[Unit] +Description=Zeek network monitor (via zeekctl) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=` + zeekctl + ` deploy +ExecStop=` + zeekctl + ` stop + +[Install] +WantedBy=multi-user.target +` +} + +// quoteExecArg makes arg safe in an ExecStart= line, where systemd splits on whitespace, +// treats % as a specifier and expands $ as an environment variable. +func quoteExecArg(arg string) string { + arg = strings.ReplaceAll(arg, "%", "%%") + arg = strings.ReplaceAll(arg, "$", "$$") + if arg != "" && !strings.ContainsAny(arg, " \t\"'\\") { + return arg + } + arg = strings.ReplaceAll(arg, `\`, `\\`) + arg = strings.ReplaceAll(arg, `"`, `\"`) + return `"` + arg + `"` +} + +// WithSafeMode returns the flags the unit runs hexwall with. A service starts at every +// boot, before anyone is watching, so it runs in watch mode unless the user asked for a +// mode explicitly. +func WithSafeMode(args []string) []string { + for _, arg := range args { + // Go's flag package accepts one or two dashes, with the value attached by = or not. + if arg == "--mode" || arg == "-mode" || strings.HasPrefix(arg, "--mode=") || strings.HasPrefix(arg, "-mode=") { + return args + } + } + return append([]string{"--mode", "watch"}, args...) +} diff --git a/internal/service/service_test.go b/internal/service/service_test.go new file mode 100644 index 0000000..a5c7ea4 --- /dev/null +++ b/internal/service/service_test.go @@ -0,0 +1,132 @@ +package service + +import ( + "bytes" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestWithSafeMode(t *testing.T) { + tests := []struct { + name string + args []string + want []string + }{ + {"no flags", nil, []string{"--mode", "watch"}}, + {"other flags only", []string{"--debug"}, []string{"--mode", "watch", "--debug"}}, + {"explicit enforce", []string{"--mode", "enforce"}, []string{"--mode", "enforce"}}, + {"equals form", []string{"--mode=enforce", "--debug"}, []string{"--mode=enforce", "--debug"}}, + {"single dash", []string{"-mode", "enforce"}, []string{"-mode", "enforce"}}, + {"single dash equals", []string{"-mode=watch"}, []string{"-mode=watch"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := WithSafeMode(tt.args); !slices.Equal(got, tt.want) { + t.Fatalf("WithSafeMode(%q) = %q, want %q", tt.args, got, tt.want) + } + }) + } +} + +func TestQuoteExecArg(t *testing.T) { + tests := map[string]string{ + "/usr/local/bin/hexwall": "/usr/local/bin/hexwall", + "/path with space/x": `"/path with space/x"`, + "100%": "100%%", + "$HOME": "$$HOME", + `a"b`: `"a\"b"`, + "": `""`, + } + for in, want := range tests { + if got := quoteExecArg(in); got != want { + t.Errorf("quoteExecArg(%q) = %q, want %q", in, got, want) + } + } +} + +func TestRenderHexwallUnit(t *testing.T) { + unit := renderHexwallUnit("/usr/local/bin/hexwall", []string{"--mode", "watch", "--zeek-log", "/opt/zeek/logs/current/ssl.log"}, true) + for _, want := range []string{ + marker + "\n", + "ExecStart=/usr/local/bin/hexwall --mode watch --zeek-log /opt/zeek/logs/current/ssl.log\n", + "After=network-online.target docker.service zeek.service\n", + "Wants=network-online.target zeek.service\n", + "WorkingDirectory=/var/lib/hexwall\n", + "Restart=on-failure\n", + } { + if !strings.Contains(unit, want) { + t.Errorf("unit is missing %q:\n%s", want, unit) + } + } + if strings.Contains(renderHexwallUnit("/x", nil, false), "zeek") { + t.Error("unit without Zeek still references zeek.service") + } +} + +type fakeSystemctl struct{ calls []string } + +func (f *fakeSystemctl) run(args ...string) error { + f.calls = append(f.calls, strings.Join(args, " ")) + return nil +} + +func TestInstall_WritesUnitsAndStartsHexwall(t *testing.T) { + dir := t.TempDir() + systemctl := &fakeSystemctl{} + opts := Options{Binary: "/usr/local/bin/hexwall", Args: []string{"--mode", "watch"}, Zeekctl: "/opt/zeek/bin/zeekctl"} + + var out bytes.Buffer + if err := install(&out, opts, dir, nil, systemctl.run); err != nil { + t.Fatalf("install() error = %v", err) + } + + for _, unit := range []string{hexwallUnit, zeekUnit} { + if _, err := os.Stat(filepath.Join(dir, unit)); err != nil { + t.Errorf("%s not written: %v", unit, err) + } + } + want := []string{"daemon-reload", "enable --now zeek.service", "enable hexwall.service", "restart hexwall.service"} + if !slices.Equal(systemctl.calls, want) { + t.Fatalf("systemctl calls = %q, want %q", systemctl.calls, want) + } +} + +// A zeek.service someone else wrote (a distro package, or by hand) must survive untouched. +func TestInstall_KeepsForeignZeekUnit(t *testing.T) { + dir := t.TempDir() + foreign := "[Service]\nExecStart=/usr/bin/zeek -i eth1\n" + if err := os.WriteFile(filepath.Join(dir, zeekUnit), []byte(foreign), 0o644); err != nil { + t.Fatal(err) + } + + opts := Options{Binary: "/x", Zeekctl: "/opt/zeek/bin/zeekctl"} + if err := install(&bytes.Buffer{}, opts, dir, nil, (&fakeSystemctl{}).run); err != nil { + t.Fatalf("install() error = %v", err) + } + + got, _ := os.ReadFile(filepath.Join(dir, zeekUnit)) + if string(got) != foreign { + t.Fatalf("foreign zeek.service was overwritten:\n%s", got) + } + hexwall, _ := os.ReadFile(filepath.Join(dir, hexwallUnit)) + if !strings.Contains(string(hexwall), "After=network-online.target docker.service zeek.service") { + t.Fatalf("hexwall.service doesn't order itself after the existing zeek.service:\n%s", hexwall) + } +} + +func TestInstall_WithoutZeek(t *testing.T) { + dir := t.TempDir() + systemctl := &fakeSystemctl{} + if err := install(&bytes.Buffer{}, Options{Binary: "/x"}, dir, nil, systemctl.run); err != nil { + t.Fatalf("install() error = %v", err) + } + if _, err := os.Stat(filepath.Join(dir, zeekUnit)); !os.IsNotExist(err) { + t.Fatalf("zeek.service written although hexwall doesn't use Zeek (err %v)", err) + } + if slices.Contains(systemctl.calls, "enable --now zeek.service") { + t.Fatalf("zeek.service enabled although hexwall doesn't use Zeek: %q", systemctl.calls) + } +} diff --git a/internal/zeek/setup.go b/internal/zeek/setup.go index 61f6ca1..f3d80bc 100644 --- a/internal/zeek/setup.go +++ b/internal/zeek/setup.go @@ -86,6 +86,19 @@ Next steps: return nil } +// Zeekctl returns the path of the local Zeek's zeekctl, located through zeek-config. +func Zeekctl() (string, error) { + zeekConfig, err := findZeekConfig() + if err != nil { + return "", err + } + prefix, err := zeekConfigValue(zeekConfig, "--prefix") + if err != nil { + return "", err + } + return filepath.Join(prefix, "bin", "zeekctl"), nil +} + func findZeekConfig() (string, error) { if path, err := exec.LookPath("zeek-config"); err == nil { return path, nil diff --git a/main.go b/main.go index 266308c..6c29b66 100644 --- a/main.go +++ b/main.go @@ -3,11 +3,13 @@ package main import ( "context" + "errors" "flag" "fmt" "log/slog" "os" "os/signal" + "path/filepath" "runtime" "strings" "syscall" @@ -18,6 +20,7 @@ import ( "github.com/hexbytedev/hexwall/internal/detector" "github.com/hexbytedev/hexwall/internal/monitor" "github.com/hexbytedev/hexwall/internal/pihole" + "github.com/hexbytedev/hexwall/internal/service" "github.com/hexbytedev/hexwall/internal/store" "github.com/hexbytedev/hexwall/internal/zeek" ) @@ -51,18 +54,22 @@ func main() { os.Exit(run()) } -func parseFlags() (*runConfig, error) { - dbPath := flag.String("db", "", "path to pihole-FTL.db (auto-detected if not set)") - hexwallDB := flag.String("hexwall-db", "./hexwall.db", "path to local hexwall database") - zeekLogPath := flag.String("zeek-log", "", "path to zeek ssl.log for SNI-based domain verification (empty to skip)") - zeekNoticeLog := flag.String("zeek-notice-log", "/opt/zeek/logs/current/notice.log", "path to Zeek notice.log for DNS-bypass alerts") - enableZeek := flag.Bool("enable-zeek", false, "enable Zeek-based DNS-bypass detection") - enableDeghostIP := flag.Bool("enable-deghost-ip", false, "send unrecognized IPs to the third-party reputation API (deghostapi.hexbyte.dev) on the IP-only fallback path") - enableDeghostDomain := flag.Bool("enable-deghost-domain", false, "send unrecognized SNI domains to the third-party reputation API (deghostapi.hexbyte.dev) at rung 6") - mode := flag.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") - debug := flag.Bool("debug", false, "enable debug-level logging, including verbose per-connection scan logging") - showVersion := flag.Bool("version", false, "print version and exit") - flag.Parse() +// parseFlags parses hexwall's run flags from args (os.Args[1:] in normal use). +func parseFlags(args []string) (*runConfig, error) { + flags := flag.NewFlagSet("hexwall", flag.ContinueOnError) + dbPath := flags.String("db", "", "path to pihole-FTL.db (auto-detected if not set)") + hexwallDB := flags.String("hexwall-db", "./hexwall.db", "path to local hexwall database") + zeekLogPath := flags.String("zeek-log", "", "path to zeek ssl.log for SNI-based domain verification (empty to skip)") + zeekNoticeLog := flags.String("zeek-notice-log", "/opt/zeek/logs/current/notice.log", "path to Zeek notice.log for DNS-bypass alerts") + enableZeek := flags.Bool("enable-zeek", false, "enable Zeek-based DNS-bypass detection") + enableDeghostIP := flags.Bool("enable-deghost-ip", false, "send unrecognized IPs to the third-party reputation API (deghostapi.hexbyte.dev) on the IP-only fallback path") + enableDeghostDomain := flags.Bool("enable-deghost-domain", false, "send unrecognized SNI domains to the third-party reputation API (deghostapi.hexbyte.dev) at rung 6") + mode := flags.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") + debug := flags.Bool("debug", false, "enable debug-level logging, including verbose per-connection scan logging") + showVersion := flags.Bool("version", false, "print version and exit") + if err := flags.Parse(args); err != nil { + return nil, err + } cfg := &runConfig{ dbPath: strings.TrimSpace(*dbPath), @@ -104,8 +111,18 @@ func run() int { } return 0 } + if len(os.Args) > 1 && os.Args[1] == "install-service" { + if err := installService(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, "install-service:", err) + return 1 + } + return 0 + } - cfg, err := parseFlags() + cfg, err := parseFlags(os.Args[1:]) + if errors.Is(err, flag.ErrHelp) { + return 0 + } if err != nil { slog.Error(err.Error()) return 1 @@ -345,3 +362,41 @@ func run() int { } } } + +// installService installs the systemd units that run hexwall with args at every boot. +func installService(args []string) error { + if runtime.GOOS != "linux" { + return errors.New("only Linux with systemd is supported") + } + args = service.WithSafeMode(args) + // Reject bad flags now: in a unit they would crash-loop at every boot instead. + cfg, err := parseFlags(args) + if err != nil { + return err + } + + binary, err := os.Executable() + if err != nil { + return err + } + if binary, err = filepath.EvalSymlinks(binary); err != nil { + return err + } + if strings.Contains(binary, "go-build") { + return errors.New("this binary is temporary (go run); build hexwall and run the built binary instead") + } + + opts := service.Options{Binary: binary, Args: args} + if cfg.enableZeek || cfg.zeekLogPath != "" { + if opts.Zeekctl, err = zeek.Zeekctl(); err != nil { + return err + } + } + if err := service.Install(os.Stdout, opts); err != nil { + return err + } + if cfg.mode == monitor.ModeWatch { + fmt.Println("\nRunning in watch mode: nothing is killed. Re-run with --mode enforce once the logs look right.") + } + return nil +} From a8382a6994083ac0975fdeaf6e3a1e1463f82417 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 16:37:46 +0600 Subject: [PATCH 40/56] docs: README covers zeek-setup, install-service, bypass-alert kills and regex lists Also corrects two claims the code had proved wrong: notice.log does need Zeek-side setup, and blocked domains are excluded from trust up front rather than assumed not to resolve. --- README.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5dfce1d..c9d207b 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ You can download them from the project's GitHub Releases page: For production Linux servers, download the matching binary from Releases, place it where your deployment expects `./app/hexwall`, and use the included demo Docker Compose file at [`docker-compose.yml`](./docker-compose.yml) to launch it on the server. +To run hexwall directly on the host instead, so that it starts at boot and restarts if it crashes, use [`hexwall install-service`](#run-as-a-systemd-service). + hexwall targets Linux, where prebuilt binaries are published by CI. macOS is best-effort: clone the repository and build the binary manually, and install `somo` as a fallback since `ss` (Linux's iproute2) isn't available there. Windows is not supported. ### Connection backend @@ -114,8 +116,8 @@ If neither is found, start fails. Use `--db` to specify the path manually. | `--enable-zeek` | `false` | Enable the Zeek `notice.log` DNS-bypass alert path (independent of `--zeek-log`) | | `--zeek-log` | _(unset — SNI matching skipped)_ | Path to Zeek's `ssl.log`, used to match a connection's TLS SNI against Pi-hole's allow/deny lists | | `--zeek-notice-log` | `/opt/zeek/logs/current/notice.log` | Path to Zeek's `notice.log`; only read when `--enable-zeek` is set | -| `--enable-deghost-ip` | `true` | Enable third-party IP reputation checks for unrecognized connections with no SNI | -| `--enable-deghost-domain` | `true` | Enable third-party domain reputation checks for unrecognized SNI domains | +| `--enable-deghost-ip` | `false` | Send unrecognized IPs (connections with no SNI) to the third-party reputation API | +| `--enable-deghost-domain` | `false` | Send unrecognized SNI domains to the third-party reputation API | | `--version` | `false` | Print version and exit | `--mode` accepts `watch` or `enforce` (case-insensitive). Any other value exits with an error. @@ -124,12 +126,40 @@ If neither is found, start fails. Use `--db` to specify the path manually. ### Zeek setup -hexwall tails `ssl.log` directly, in whichever of Zeek's two output formats it's actually producing — no Zeek-side configuration is required: +hexwall reads two Zeek logs, in whichever of Zeek's two output formats Zeek is producing: -- **TSV** (Zeek's default `LogAscii` output, used by a stock install with no extra config) — the column order is learned from the log file's own `#fields` header the first time it's read, rather than assumed, so it keeps working across Zeek versions or configurations that add extra columns. +- **TSV** (Zeek's default `LogAscii` output, used by a stock install with no extra config) — the column order is learned from the log file's own `#fields` header rather than assumed, so it keeps working across Zeek versions or configurations that add extra columns. - **JSON** (`redef LogAscii::use_json = T;`) — also supported, parsed line by line. -Tested against Zeek 9.0.0. Point `--zeek-log` at the running instance's `ssl.log`: for a plain `zeek -i ` invocation, that's a file in the working directory Zeek was started from; under `zeekctl`, it's `/current/ssl.log`. +Tested against Zeek 9.0.0. + +`ssl.log` (for `--zeek-log`) is written by every Zeek install; nothing needs configuring. `notice.log` alerts (for `--enable-zeek`) come from a DNS-bypass detection script that ships inside the hexwall binary. Install it with: + +```sh +sudo hexwall zeek-setup +``` + +`zeek-setup` finds Zeek through `zeek-config` (on `PATH`, or `/opt/zeek/bin/zeek-config`), then: + +1. writes `dns-bypass.zeek` into Zeek's site directory, replacing an older copy so the script always matches the hexwall binary that reads its alerts; +2. adds `@load dns-bypass` to `local.zeek` once, leaving the rest of the file alone; +3. checks that every `interface=` in `node.cfg` exists on this machine, and lists the real interfaces if one doesn't (Zeek can't start on a missing interface; the stock `node.cfg` says `eth0`); +4. prints the next steps: `sudo zeekctl deploy` to restart Zeek with the script, and the exact `--zeek-log`/`--zeek-notice-log` flags for the configured log directory. + +It does not edit `node.cfg` or restart Zeek itself. Re-running it is safe. + +Under `zeekctl`, both logs are in `/current/` (`/opt/zeek/logs/current/` by default). For a plain `zeek -i ` invocation, they're written to the directory Zeek was started from. + +Both logs are read from the moment hexwall starts: alerts already in `notice.log` are skipped, so a restart can't act on old ones, and SNI is only known for TLS handshakes made after startup. + +### DNS-bypass alerts + +The script raises a `DNSBypass::PossibleBypass` notice when a TLS connection's SNI names a domain that the client never looked up through DNS, a sign that it bypassed Pi-hole. For each alert hexwall checks the domain against Pi-hole's lists and records it in the `zeek_alerts` table. If the domain is blocked: + +- in `--mode watch`, the alert is logged; +- in `--mode enforce`, hexwall looks for the alerted connection among **this machine's** sockets and kills it through the same guarded path as every other kill (see [When a connection is killed](#when-a-connection-is-killed)). + +Zeek sees the whole network it listens on, but a socket can only be closed on the machine that owns it. An alert about another device is recorded as `not_on_this_host` with a warning; to block that device's connection, run hexwall on it too. ### Debug logging @@ -137,22 +167,52 @@ When `--debug` is enabled, hexwall also logs intermediate, non-actionable outcom Every scan tick prints a one-line summary regardless of `--debug` — connection count, and how many were trusted, blocked, unknown, or errored — and an empty scan (the connection-listing tool returning zero connections) is always logged too. +### Run as a systemd service + +```sh +sudo hexwall install-service [hexwall flags...] +# e.g. +sudo /usr/local/bin/hexwall install-service --zeek-log /opt/zeek/logs/current/ssl.log --enable-zeek +``` + +This makes hexwall start at boot and restart if it crashes. The flags after `install-service` are the ones the service runs with, and they're checked first, so a typo fails here instead of making the service crash at every boot. Then it: + +1. writes `/etc/systemd/system/hexwall.service`, running this binary (by its absolute path) with those flags from `/var/lib/hexwall`, where the default `./hexwall.db` is created; +2. if the flags use Zeek (`--zeek-log` or `--enable-zeek`), writes a `zeek.service` that runs `zeekctl deploy` at boot and `zeekctl stop` on shutdown, and orders hexwall after it. A `zeek.service` that hexwall didn't write (from a package, or written by hand) is left untouched and used as is; +3. runs `systemctl daemon-reload`, enables both units, and (re)starts hexwall. + +**The service runs in `--mode watch` unless you pass `--mode` yourself.** A service starts before anyone is watching, so killing has to be switched on deliberately: check the logs first, then re-run `install-service` with `--mode enforce`. + +Re-running `install-service` updates the units. Build hexwall before you run it: the binary that `go run` makes is deleted when it exits, so it's refused. + +```sh +systemctl status hexwall # is it running? +journalctl -u hexwall -f # follow its logs +sudo systemctl disable --now hexwall # stop it and remove it from boot +``` + +Linux with systemd only. + --- ## When a connection is killed -A connection is only ever killed in `--mode enforce`, for a connection the active connection-listing tool (`ss` or `somo`) reports as currently established. Three independent checks can trigger it: +A connection is only ever killed in `--mode enforce`, for a connection the active connection-listing tool (`ss` or `somo`) reports as currently established. Four independent checks can trigger it: + +1. **Pi-hole domain denylist match** — Zeek captured the connection's TLS SNI (via `--zeek-log`) and that domain is on Pi-hole's own denylist, a regex denylist entry, or the gravity blocklist. Requires only Pi-hole; no third-party service is involved, and this check runs regardless of the `--enable-deghost-*` flags. +2. **Third-party domain reputation match** — an SNI was captured, the domain is on neither Pi-hole list and isn't already known-trusted for this IP, and the reputation API's verdict (fetched live or reused from a 6-hour cache) says to block it. Requires `--enable-deghost-domain` (default `false`). +3. **Third-party IP reputation match** — no SNI was available at all (non-TLS traffic, ECH, or a connection predating startup), the IP is not trusted, and the reputation API's verdict on the bare IP (live or 6-hour cached) marks it `is_abuser`, `is_attacker`, or `is_threat`. Requires `--enable-deghost-ip` (default `false`). + +4. **Zeek DNS-bypass alert for a blocked domain** — Zeek raised a bypass alert (`--enable-zeek`), the domain is blocked by Pi-hole's lists, and the alerted connection is on this machine. See [DNS-bypass alerts](#dns-bypass-alerts). -1. **Pi-hole domain denylist match** — Zeek captured the connection's TLS SNI (via `--zeek-log`) and that domain is on Pi-hole's own denylist or gravity blocklist. Requires only Pi-hole; no third-party service is involved, and this check runs regardless of the `--enable-deghost-*` flags. -2. **Third-party domain reputation match** — an SNI was captured, the domain is on neither Pi-hole list and isn't already known-trusted for this IP, and the reputation API's verdict (fetched live or reused from a 6-hour cache) says to block it. Requires `--enable-deghost-domain` (default `true`). -3. **Third-party IP reputation match** — no SNI was available at all (non-TLS traffic, ECH, or a connection predating startup), the IP is not trusted, and the reputation API's verdict on the bare IP (live or 6-hour cached) marks it `is_abuser`, `is_attacker`, or `is_threat`. Requires `--enable-deghost-ip` (default `true`). +Both `--enable-deghost-*` flags default to `false`, because they send connection data off the machine. With the defaults, only the Pi-hole checks (1 and 4) can trigger a kill, and both need Zeek. So `--mode enforce` without `--zeek-log` kills nothing unless you opt in to `--enable-deghost-ip`; hexwall warns about this at startup. -With both `--enable-deghost-ip` and `--enable-deghost-domain` set to `false`, only the first check — Pi-hole's own denylist — can ever trigger a kill. +Pi-hole regex entries are matched the way Pi-hole matches them: case-insensitive, with `;invert` honoured. Options hexwall can't evaluate for a connection (`;querytype=`, `;reply=`, unknown ones) resolve toward not killing: an allow rule with them still applies, and a deny rule with them is skipped. Regex lists are reloaded on the 30-second refresh, so edits in Pi-hole take effect without a restart. A connection is treated as trusted, and skipped entirely, when any of these are true: - Its IP matches the built-in CIDR allowlist. -- Its domain is explicitly on Pi-hole's allowlist. +- Its domain is on Pi-hole's allowlist (exact or regex). - Its IP was refreshed from Pi-hole query history within the last hour, or was already seen as an established connection within the last 60 seconds. - Its domain has already been seen for this exact IP in hexwall's own history. @@ -188,10 +248,10 @@ This cache lives in the local hexwall database as Unix timestamps (`INTEGER` / i Pi-hole trust is built from recent DNS history, not by directly trusting every current connection. The refresh flow is: 1. Read the `queries` table from `pihole-FTL.db`. -2. Select distinct non-empty domains seen within the last hour. +2. Select distinct non-empty domains seen within the last hour **that Pi-hole answered**. Queries Pi-hole blocked are skipped, so a blocked domain's IP never becomes trusted, even if the host's resolver could look it up through another nameserver. Only the FTL statuses known to mean "answered" (forwarded, cached, retried, and their variants) count; a status hexwall doesn't recognise counts as not answered. 3. Normalize them to lowercase and trim whitespace. 4. Resolve each domain through the system DNS resolver, with bounded concurrency and a 1 second lookup timeout per domain. -5. Ignore domains that do not resolve. This is normal for blocked domains, expired records, and similar cases. +5. Ignore domains that do not resolve, such as expired records. 6. Deduplicate resolved IPs. If multiple domains resolve to the same IP, the first domain in sorted order is stored for that IP. 7. Upsert each resolved IP into the local `allowed_ips` table, setting or refreshing `last_refreshed`. From aaed2190e34f4667b26935e1a5f5b4d5d7b1601c Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 17:05:38 +0600 Subject: [PATCH 41/56] feat: add hexwall uninstall Undoing install-service and zeek-setup took seven hand-typed commands and knowledge of every file hexwall touched. `sudo hexwall uninstall` stops, disables and deletes hexwall's units, removes the @load lines it added to local.zeek, and deletes the script. Only files carrying hexwall's signature are touched, so units and scripts someone else wrote survive. The database (kill and alert history) and the binary are kept, with the command to delete the data printed. --- README.md | 10 ++++ internal/service/service.go | 49 ++++++++++++++++ internal/service/service_test.go | 37 +++++++++++++ internal/zeek/setup.go | 95 +++++++++++++++++++++++++++++++- internal/zeek/setup_test.go | 52 +++++++++++++++++ main.go | 30 ++++++++++ 6 files changed, 270 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c9d207b..3a319e9 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,16 @@ sudo systemctl disable --now hexwall # stop it and remove it from boot Linux with systemd only. +### Uninstall + +```sh +sudo hexwall uninstall +``` + +Undoes `install-service` and `zeek-setup`: it stops, disables and deletes the units hexwall wrote, removes the `@load dns-bypass` lines from `local.zeek`, and deletes the script. Only files carrying hexwall's signature are touched, so a `zeek.service` or `dns-bypass.zeek` someone else wrote is left in place. Changes you made yourself, such as the interface in `node.cfg`, stay. + +Two things are kept on purpose: `/var/lib/hexwall` (the database with the kill and alert history; delete it with `sudo rm -r /var/lib/hexwall`) and the hexwall binary, which you installed. + --- ## When a connection is killed diff --git a/internal/service/service.go b/internal/service/service.go index 3d53367..b2cd561 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -91,6 +91,55 @@ hexwall now starts at boot and restarts if it crashes. return nil } +// Uninstall stops, disables and deletes the units hexwall installed. A zeek.service hexwall +// didn't write keeps running untouched. +func Uninstall(out io.Writer) error { + if os.Geteuid() != 0 { + return errors.New("removing systemd units needs root (run with sudo)") + } + if _, err := exec.LookPath("systemctl"); err != nil { + _, _ = fmt.Fprintln(out, "✓ no systemd here; no units to remove") + return nil + } + return uninstall(out, unitDir, runSystemctl) +} + +func uninstall(out io.Writer, dir string, systemctl func(args ...string) error) error { + var removed []string + // hexwall first: it is ordered after Zeek, so it stops before Zeek does. + for _, unit := range []string{hexwallUnit, zeekUnit} { + path := filepath.Join(dir, unit) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return err + } + if !bytes.HasPrefix(data, []byte(marker)) { + _, _ = fmt.Fprintf(out, "! %s wasn't written by hexwall; left in place\n", path) + continue + } + if err := systemctl("disable", "--now", unit); err != nil { + return fmt.Errorf("systemctl disable --now %s: %w", unit, err) + } + if err := os.Remove(path); err != nil { + return err + } + _, _ = fmt.Fprintf(out, "✓ stopped and removed %s\n", path) + removed = append(removed, unit) + } + + if len(removed) == 0 { + _, _ = fmt.Fprintln(out, "✓ no hexwall systemd units installed") + return nil + } + if err := systemctl("daemon-reload"); err != nil { + return fmt.Errorf("systemctl daemon-reload: %w", err) + } + return nil +} + // zeekUnitOwner reports whether zeek.service is free for hexwall to write: either absent, // or a unit hexwall wrote earlier. func zeekUnitOwner(dir string, vendorDirs []string) (bool, error) { diff --git a/internal/service/service_test.go b/internal/service/service_test.go index a5c7ea4..b3976c7 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -130,3 +130,40 @@ func TestInstall_WithoutZeek(t *testing.T) { t.Fatalf("zeek.service enabled although hexwall doesn't use Zeek: %q", systemctl.calls) } } + +func TestUninstall_RemovesOnlyHexwallsUnits(t *testing.T) { + dir := t.TempDir() + if err := install(&bytes.Buffer{}, Options{Binary: "/x"}, dir, nil, (&fakeSystemctl{}).run); err != nil { + t.Fatal(err) + } + foreign := "[Service]\nExecStart=/usr/bin/zeek -i eth1\n" + if err := os.WriteFile(filepath.Join(dir, zeekUnit), []byte(foreign), 0o644); err != nil { + t.Fatal(err) + } + + systemctl := &fakeSystemctl{} + if err := uninstall(&bytes.Buffer{}, dir, systemctl.run); err != nil { + t.Fatalf("uninstall() error = %v", err) + } + + if _, err := os.Stat(filepath.Join(dir, hexwallUnit)); !os.IsNotExist(err) { + t.Fatalf("hexwall.service still there (err %v)", err) + } + if got, _ := os.ReadFile(filepath.Join(dir, zeekUnit)); string(got) != foreign { + t.Fatalf("foreign zeek.service changed:\n%s", got) + } + want := []string{"disable --now hexwall.service", "daemon-reload"} + if !slices.Equal(systemctl.calls, want) { + t.Fatalf("systemctl calls = %q, want %q", systemctl.calls, want) + } +} + +func TestUninstall_NothingInstalled(t *testing.T) { + systemctl := &fakeSystemctl{} + if err := uninstall(&bytes.Buffer{}, t.TempDir(), systemctl.run); err != nil { + t.Fatalf("uninstall() error = %v", err) + } + if len(systemctl.calls) != 0 { + t.Fatalf("systemctl called with nothing installed: %q", systemctl.calls) + } +} diff --git a/internal/zeek/setup.go b/internal/zeek/setup.go index f3d80bc..c7bbd63 100644 --- a/internal/zeek/setup.go +++ b/internal/zeek/setup.go @@ -15,8 +15,11 @@ import ( ) const ( - scriptName = "dns-bypass.zeek" - loadLine = "@load dns-bypass" + scriptName = "dns-bypass.zeek" + loadLine = "@load dns-bypass" + loadComment = "# Added by hexwall zeek-setup: DNS-bypass notices that hexwall reads." + // Every version of the script carries this phrase; Teardown only deletes a file with it. + scriptSignature = "hexwall zeek-setup" // Where Zeek's binary packages install; under sudo, /opt/zeek/bin is usually not on PATH. defaultZeekConfig = "/opt/zeek/bin/zeek-config" ) @@ -86,6 +89,92 @@ Next steps: return nil } +// Teardown undoes Setup: it removes the @load lines from local.zeek and deletes the script. +// A dns-bypass.zeek that hexwall didn't write is left in place. A machine without Zeek has +// nothing to undo. +func Teardown(out io.Writer) error { + zeekConfig, err := findZeekConfig() + if err != nil { + _, _ = fmt.Fprintln(out, "✓ Zeek not found; nothing to remove there") + return nil + } + siteDir, err := zeekConfigValue(zeekConfig, "--site_dir") + if err != nil { + return err + } + + localZeek := filepath.Join(siteDir, "local.zeek") + removed, err := removeLoad(localZeek) + if err != nil { + return err + } + if removed { + _, _ = fmt.Fprintf(out, "✓ removed %q from %s\n", loadLine, localZeek) + } + + script := filepath.Join(siteDir, scriptName) + deleted, err := removeScript(script) + if err != nil { + return err + } + if deleted { + _, _ = fmt.Fprintf(out, "✓ deleted %s\n", script) + } else if _, statErr := os.Stat(script); statErr == nil { + _, _ = fmt.Fprintf(out, "! %s wasn't written by hexwall; left in place\n", script) + } + return nil +} + +// removeLoad deletes the lines ensureLoaded added: the @load line, its comment, and the +// blank line before the comment. +func removeLoad(localZeek string) (removed bool, err error) { + data, err := os.ReadFile(localZeek) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + if err != nil { + return false, permissionHint(fmt.Errorf("read %s: %w", localZeek, err)) + } + + lines := strings.Split(string(data), "\n") + kept := make([]string, 0, len(lines)) + for i, line := range lines { + switch trimmed := strings.TrimSpace(line); { + case trimmed == loadLine || trimmed == loadComment: + removed = true + case trimmed == "" && i+1 < len(lines) && strings.TrimSpace(lines[i+1]) == loadComment: + // The blank line ensureLoaded put before its comment. + default: + kept = append(kept, line) + } + } + if !removed { + return false, nil + } + if err := os.WriteFile(localZeek, []byte(strings.Join(kept, "\n")), 0o644); err != nil { + return false, permissionHint(fmt.Errorf("write %s: %w", localZeek, err)) + } + return true, nil +} + +// removeScript deletes the script if hexwall wrote it (any version), and reports whether it did. +func removeScript(path string) (deleted bool, err error) { + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + if err != nil { + return false, permissionHint(fmt.Errorf("read %s: %w", path, err)) + } + if !bytes.Contains(data, []byte(scriptSignature)) { + return false, nil + } + if err := os.Remove(path); err != nil { + return false, permissionHint(fmt.Errorf("delete %s: %w", path, err)) + } + return true, nil +} + // Zeekctl returns the path of the local Zeek's zeekctl, located through zeek-config. func Zeekctl() (string, error) { zeekConfig, err := findZeekConfig() @@ -141,7 +230,7 @@ func ensureLoaded(localZeek string) (added bool, err error) { } } - addition := "\n# Added by hexwall zeek-setup: DNS-bypass notices that hexwall reads.\n" + loadLine + "\n" + addition := "\n" + loadComment + "\n" + loadLine + "\n" if len(data) > 0 && !bytes.HasSuffix(data, []byte("\n")) { addition = "\n" + addition } diff --git a/internal/zeek/setup_test.go b/internal/zeek/setup_test.go index b61033b..46d49f4 100644 --- a/internal/zeek/setup_test.go +++ b/internal/zeek/setup_test.go @@ -108,3 +108,55 @@ func TestZeekLogDir(t *testing.T) { t.Fatalf("zeekLogDir() = %q, want %q", got, "/var/log/zeek") } } + +// Setup then Teardown must give local.zeek back exactly as it was. +func TestRemoveLoad_UndoesEnsureLoaded(t *testing.T) { + localZeek := filepath.Join(t.TempDir(), "local.zeek") + original := "@load misc/detect-traceroute\n\n@load tuning/defaults\n" + if err := os.WriteFile(localZeek, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + if _, err := ensureLoaded(localZeek); err != nil { + t.Fatal(err) + } + removed, err := removeLoad(localZeek) + if err != nil || !removed { + t.Fatalf("removeLoad() = %v, %v; want true, nil", removed, err) + } + got, _ := os.ReadFile(localZeek) + if string(got) != original { + t.Fatalf("local.zeek after setup+teardown:\n%q\nwant:\n%q", got, original) + } + + if removed, err := removeLoad(localZeek); err != nil || removed { + t.Fatalf("second removeLoad() = %v, %v; want false, nil", removed, err) + } +} + +func TestRemoveScript_OnlyDeletesHexwallsOwn(t *testing.T) { + dir := t.TempDir() + + ours := filepath.Join(dir, "ours.zeek") + if err := os.WriteFile(ours, DNSBypassScript, 0o644); err != nil { + t.Fatal(err) + } + if deleted, err := removeScript(ours); err != nil || !deleted { + t.Fatalf("removeScript(hexwall's script) = %v, %v; want true, nil", deleted, err) + } + + foreign := filepath.Join(dir, "foreign.zeek") + if err := os.WriteFile(foreign, []byte("# someone's own dns-bypass\n"), 0o644); err != nil { + t.Fatal(err) + } + if deleted, err := removeScript(foreign); err != nil || deleted { + t.Fatalf("removeScript(foreign) = %v, %v; want false, nil", deleted, err) + } + if _, err := os.Stat(foreign); err != nil { + t.Fatalf("foreign script was deleted: %v", err) + } + + if deleted, err := removeScript(filepath.Join(dir, "missing.zeek")); err != nil || deleted { + t.Fatalf("removeScript(missing) = %v, %v; want false, nil", deleted, err) + } +} diff --git a/main.go b/main.go index 6c29b66..3ea3fba 100644 --- a/main.go +++ b/main.go @@ -119,6 +119,14 @@ func run() int { return 0 } + if len(os.Args) > 1 && os.Args[1] == "uninstall" { + if err := uninstall(); err != nil { + fmt.Fprintln(os.Stderr, "uninstall:", err) + return 1 + } + return 0 + } + cfg, err := parseFlags(os.Args[1:]) if errors.Is(err, flag.ErrHelp) { return 0 @@ -400,3 +408,25 @@ func installService(args []string) error { } return nil } + +// uninstall removes what install-service and zeek-setup added. The database (the kill and +// alert audit trail) and the binary are kept: hexwall didn't install the binary, and +// deleting evidence should be a separate, deliberate step. +func uninstall() error { + if runtime.GOOS != "linux" { + return errors.New("only Linux with systemd is supported") + } + // Services first, so Zeek stops before its script is removed. + if err := service.Uninstall(os.Stdout); err != nil { + return err + } + if err := zeek.Teardown(os.Stdout); err != nil { + return err + } + fmt.Println(` +Kept on purpose: + /var/lib/hexwall the service's database, with its kill and alert history + (delete it with: sudo rm -r /var/lib/hexwall) + this binary delete it yourself if you installed it (e.g. /usr/local/bin/hexwall)`) + return nil +} From ac2199da24f00975348ac4bf18baf8429e96cb90 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Sun, 11 Oct 2026 17:27:25 +0600 Subject: [PATCH 42/56] fix(pihole): stop the trust refresh from keeping itself alive Every 30s the refresh re-resolved every domain seen in the last hour. On a host whose resolver is Pi-hole, each lookup was logged as a new query, so every domain stayed "seen in the last hour" and its IPs were trusted forever; Pi-hole logged ~530 queries a minute of hexwall's own. Queries are now read once, by id, and hexwall remembers when it looked up each domain so the matching logged query is recognised as its own echo. Trust lasts an hour from a client's last real query. A flushed query log (ids restarting) is detected and the last hour re-read. Measured live: Pi-hole's query rate went from ~530 to ~7 a minute. --- README.md | 20 +++--- internal/pihole/cache.go | 99 ++++++++++++++++++++++++---- internal/pihole/cache_test.go | 116 +++++++++++++++++++++++++++++++-- internal/pihole/pihole.go | 51 ++++++++++----- internal/pihole/pihole_test.go | 25 +++++-- 5 files changed, 261 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 3a319e9..0897ea9 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ In short: `Pi-hole + hexwall` gives you a practical containment layer for post-c Pi-hole operates at the DNS layer. It can block a domain, but it has no visibility into connections that bypass DNS entirely — processes that dial a hard-coded IP address directly. Malware and compromised packages commonly use this technique to phone home without triggering any DNS-based block. -hexwall works from the inverse assumption: **if a connection is not trusted, it is suspicious by default.** Trust is granted when the IP matches the built-in CIDR allowlist, was refreshed from Pi-hole query history within the last hour, or was already observed as an established connection within the last 60 seconds. On startup it immediately refreshes trusted IPs from Pi-hole, then refreshes them every 30 seconds while scanning connections every 10 seconds. +hexwall works from the inverse assumption: **if a connection is not trusted, it is suspicious by default.** Trust is granted when the IP matches the built-in CIDR allowlist, was resolved from a domain a client asked Pi-hole for within the last hour, or was already observed as an established connection within the last 60 seconds. On startup it immediately reads the last hour of Pi-hole history, then picks up new queries every 30 seconds while scanning connections every 10 seconds. When Zeek is enabled (`--zeek-log`), hexwall also captures the TLS SNI for each connection and checks that exact domain against Pi-hole's own allow/deny lists before ever falling back to third-party reputation — this is what lets it tell apart a blocked site from a harmless one even when both happen to share the same IP, as is common behind a CDN. Anything that still falls outside every trust check is optionally escalated to an external reputation API, with results cached locally for 6 hours. Confirmed threats are either logged (watch mode) or killed (enforce mode). @@ -223,7 +223,7 @@ A connection is treated as trusted, and skipped entirely, when any of these are - Its IP matches the built-in CIDR allowlist. - Its domain is on Pi-hole's allowlist (exact or regex). -- Its IP was refreshed from Pi-hole query history within the last hour, or was already seen as an established connection within the last 60 seconds. +- Its IP was resolved from a domain a client asked Pi-hole for within the last hour, or was already seen as an established connection within the last 60 seconds. - Its domain has already been seen for this exact IP in hexwall's own history. Connections are not killed in these cases: @@ -257,29 +257,31 @@ This cache lives in the local hexwall database as Unix timestamps (`INTEGER` / i Pi-hole trust is built from recent DNS history, not by directly trusting every current connection. The refresh flow is: -1. Read the `queries` table from `pihole-FTL.db`. -2. Select distinct non-empty domains seen within the last hour **that Pi-hole answered**. Queries Pi-hole blocked are skipped, so a blocked domain's IP never becomes trusted, even if the host's resolver could look it up through another nameserver. Only the FTL statuses known to mean "answered" (forwarded, cached, retried, and their variants) count; a status hexwall doesn't recognise counts as not answered. +1. Read the queries logged in `pihole-FTL.db` since the previous refresh (the last hour, on the first refresh). Each query is read once, tracked by its id. +2. Keep the non-empty domains **that Pi-hole answered**. Queries Pi-hole blocked are skipped, so a blocked domain's IP never becomes trusted, even if the host's resolver could look it up through another nameserver. Only the FTL statuses known to mean "answered" (forwarded, cached, retried, and their variants) count; a status hexwall doesn't recognise counts as not answered. 3. Normalize them to lowercase and trim whitespace. 4. Resolve each domain through the system DNS resolver, with bounded concurrency and a 1 second lookup timeout per domain. 5. Ignore domains that do not resolve, such as expired records. + If the host's resolver is Pi-hole itself, these lookups are logged as queries too. hexwall remembers when it looked up each domain and skips the logged query that matches, so its own lookups never count as a client asking again. Without that, every domain would stay "recently seen", and trusted, forever. 6. Deduplicate resolved IPs. If multiple domains resolve to the same IP, the first domain in sorted order is stored for that IP. 7. Upsert each resolved IP into the local `allowed_ips` table, setting or refreshing `last_refreshed`. Important details: - Startup performs this refresh immediately before the first monitoring scan, so normal traffic seen in recent Pi-hole history is trusted before enforcement begins. -- The refresh repeats every 30 seconds. -- Trust from Pi-hole refresh lasts for 1 hour from the most recent successful upsert. +- The refresh repeats every 30 seconds, and only resolves domains clients asked for since the previous one. +- Trust from Pi-hole lasts for 1 hour from the most recent successful upsert, which is shortly after a client last asked for one of the IP's domains. +- If Pi-hole's query log is flushed or recreated (its ids restart), hexwall notices and re-reads the last hour. - Existing rows keep their original `first_approved` and `last_established` values when refreshed; only the stored domain and `last_refreshed` timestamp are updated. - The monitor also extends trust for long-lived allowed connections by updating `last_established` whenever an already-trusted connection is seen still established. This means an IP becomes trusted from Pi-hole history only after all of these happen: -1. Pi-hole recorded a domain query for it within the last hour. -2. That domain still resolves during a refresh cycle. +1. A client asked Pi-hole for one of its domains, and Pi-hole answered. +2. That domain still resolves at the next refresh. 3. One of the resolved IPs is written into the local `allowed_ips` cache. -If a domain was seen in Pi-hole history but no longer resolves during refresh, no IP is added for it, so there is nothing to trust from that domain alone. +If a domain was seen in Pi-hole history but doesn't resolve at the refresh, no IP is added for it, so there is nothing to trust from that domain alone. --- diff --git a/internal/pihole/cache.go b/internal/pihole/cache.go index 6e479af..ab2d0c3 100644 --- a/internal/pihole/cache.go +++ b/internal/pihole/cache.go @@ -15,16 +15,35 @@ const ( refreshLookback = time.Hour lookupTimeout = time.Second maxConcurrentLookups = 32 + + // hexwall's own lookups go through the host resolver, which may be Pi-hole, so they land + // in the query log hexwall reads. A logged query for a domain hexwall looked up within + // this margin of its lookup is treated as that lookup's echo. + ownLookupSlack = 2 * time.Second + // How long to remember a lookup. FTL writes its log to disk about once a minute, so an + // echo can show up well after the lookup itself. + ownLookupMemory = 10 * time.Minute ) -// IPCache resolves Pi-hole domains to IPs and writes them to the store. +// IPCache resolves the domains Pi-hole answered to IPs and writes them to the store. // The store owns the trusted-IP set, and this type only drives the refresh cycle. +// +// Each logged query is read once, by id, so a domain's IPs are trusted for refreshLookback +// after a client last asked for it. Re-resolving every recent domain on each refresh +// instead would, through a host that uses Pi-hole, log a new query per domain every +// refresh and keep every domain "recent" forever. type IPCache struct { checker *Checker store *store.Store - domainsSeenSince func(int64) ([]string, error) - lookupHost func(context.Context, string) ([]string, error) + answeredQueriesAfter func(afterID, since int64) ([]AnsweredQuery, error) + maxQueryID func() (int64, error) + lookupHost func(context.Context, string) ([]string, error) + now func() time.Time + + mu sync.Mutex + lastID int64 // highest query id already read + ownLookups map[string]time.Time // domain -> when hexwall last resolved it } // NewIPCache creates an IPCache backed by the given Checker and Store. @@ -32,32 +51,32 @@ func NewIPCache(checker *Checker, store *store.Store) *IPCache { return &IPCache{ checker: checker, store: store, - domainsSeenSince: checker.DomainsSeenSince, - lookupHost: net.DefaultResolver.LookupHost, + answeredQueriesAfter: checker.AnsweredQueriesAfter, + maxQueryID: checker.MaxQueryID, + lookupHost: net.DefaultResolver.LookupHost, + now: time.Now, } } -// Refresh queries Pi-hole for recent domains, resolves them with bounded concurrency, -// and upserts the resulting unique IPs into the store. +// Refresh reads the queries Pi-hole answered since the last refresh, resolves their +// domains with bounded concurrency, and upserts the resulting unique IPs into the store. func (c *IPCache) Refresh(ctx context.Context) { if ctx.Err() != nil { return } + c.mu.Lock() + defer c.mu.Unlock() - since := time.Now().Add(-refreshLookback).Unix() - - domains, err := c.domainsSeenSince(since) + domains, err := c.newClientDomains() if err != nil { slog.Error("cache refresh: failed to query pi-hole domains", "err", err) return } if len(domains) == 0 { - slog.Info("cache refreshed", "domains", 0, "ips", 0) + slog.Debug("cache refreshed", "domains", 0, "ips", 0) return } - slices.Sort(domains) - sem := make(chan struct{}, maxConcurrentLookups) // Workers write straight into resolved under mu. Using a shared map instead of a @@ -85,6 +104,9 @@ func (c *IPCache) Refresh(ctx context.Context) { rctx, cancel := context.WithTimeout(ctx, lookupTimeout) defer cancel() + mu.Lock() + c.ownLookups[d] = c.now() + mu.Unlock() ips, err := c.lookupHost(rctx, d) if err != nil { // DNS failure is normal for blocked domains, expired records, and similar cases. @@ -145,6 +167,57 @@ func (c *IPCache) Refresh(ctx context.Context) { slog.Info("cache refreshed", "domains", len(domains), "ips", totalIPs, "pairs", totalPairs) } +// newClientDomains returns, sorted and without duplicates, the domains in queries logged +// since the last call, leaving out the echoes of hexwall's own lookups. +func (c *IPCache) newClientDomains() ([]string, error) { + now := c.now() + if c.ownLookups == nil { + c.ownLookups = make(map[string]time.Time) + } + for domain, at := range c.ownLookups { + if now.Sub(at) > ownLookupMemory { + delete(c.ownLookups, domain) + } + } + + maxID, err := c.maxQueryID() + if err != nil { + return nil, err + } + if maxID < c.lastID { + slog.Info("cache refresh: pi-hole query log was reset; re-reading the last hour") + c.lastID = 0 + } + + queries, err := c.answeredQueriesAfter(c.lastID, now.Add(-refreshLookback).Unix()) + if err != nil { + return nil, err + } + + var domains []string + for _, q := range queries { + c.lastID = max(c.lastID, q.ID) + if c.isOwnLookup(q) { + continue + } + domains = append(domains, q.Domain) + } + slices.Sort(domains) + return slices.Compact(domains), nil +} + +// isOwnLookup reports whether q was logged when hexwall itself resolved q.Domain. A client +// that asked for the same domain in that same moment is skipped too, which costs nothing: +// hexwall's lookup has just refreshed that domain's IPs. +func (c *IPCache) isOwnLookup(q AnsweredQuery) bool { + at, ok := c.ownLookups[q.Domain] + if !ok { + return false + } + return !q.Timestamp.Before(at.Add(-ownLookupSlack)) && + !q.Timestamp.After(at.Add(lookupTimeout+ownLookupSlack)) +} + // RunRefresh calls Refresh on the given interval until ctx is cancelled. func (c *IPCache) RunRefresh(ctx context.Context, interval time.Duration) { if interval <= 0 { diff --git a/internal/pihole/cache_test.go b/internal/pihole/cache_test.go index 44e3881..41b81f5 100644 --- a/internal/pihole/cache_test.go +++ b/internal/pihole/cache_test.go @@ -4,7 +4,9 @@ import ( "context" "path/filepath" "reflect" + "slices" "testing" + "time" "github.com/hexbytedev/hexwall/internal/store" ) @@ -21,11 +23,14 @@ func TestIPCacheRefreshWritesTrustedIPAndDomains(t *testing.T) { } }() + log := &fakeQueryLog{now: time.Now()} + log.clientAsks("example.com") + log.clientAsks("example.org") cache := &IPCache{ - store: hexwallStore, - domainsSeenSince: func(int64) ([]string, error) { - return []string{"example.com", "example.org"}, nil - }, + store: hexwallStore, + answeredQueriesAfter: log.answeredQueriesAfter, + maxQueryID: log.maxQueryID, + now: func() time.Time { return log.now }, lookupHost: func(_ context.Context, host string) ([]string, error) { switch host { case "example.com": @@ -57,3 +62,106 @@ func TestIPCacheRefreshWritesTrustedIPAndDomains(t *testing.T) { t.Fatalf("DomainsForIP() = %v, want %v", domains, want) } } + +// fakeQueryLog stands in for Pi-hole's query log. When hexwall resolves through Pi-hole, +// its own lookups are logged too; lookupHost records them like Pi-hole would. +type fakeQueryLog struct { + now time.Time + queries []AnsweredQuery + lookups []string // domains hexwall resolved, in order +} + +func (l *fakeQueryLog) clientAsks(domain string) { + l.queries = append(l.queries, AnsweredQuery{ID: int64(len(l.queries) + 1), Domain: domain, Timestamp: l.now}) +} + +func (l *fakeQueryLog) answeredQueriesAfter(afterID, since int64) ([]AnsweredQuery, error) { + var out []AnsweredQuery + for _, q := range l.queries { + if q.ID > afterID && q.Timestamp.Unix() >= since { + out = append(out, q) + } + } + return out, nil +} + +func (l *fakeQueryLog) maxQueryID() (int64, error) { + if len(l.queries) == 0 { + return 0, nil + } + return l.queries[len(l.queries)-1].ID, nil +} + +// lookupThroughPihole resolves like a host whose resolver is Pi-hole: the lookup is logged. +func (l *fakeQueryLog) lookupThroughPihole(_ context.Context, host string) ([]string, error) { + l.lookups = append(l.lookups, host) + l.clientAsks(host) + return []string{"192.0.2.10"}, nil +} + +func newFakeCache(t *testing.T, log *fakeQueryLog) *IPCache { + t.Helper() + hexwallStore, err := store.NewStore(filepath.Join(t.TempDir(), "hexwall.db")) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + t.Cleanup(func() { _ = hexwallStore.Close() }) + return &IPCache{ + store: hexwallStore, + answeredQueriesAfter: log.answeredQueriesAfter, + maxQueryID: log.maxQueryID, + lookupHost: log.lookupThroughPihole, + now: func() time.Time { return log.now }, + } +} + +// The feedback loop: hexwall's lookup is logged by Pi-hole, and must not count as a client +// asking again, or every domain stays trusted forever. +func TestIPCacheRefreshIgnoresOwnLookups(t *testing.T) { + log := &fakeQueryLog{now: time.Unix(1_800_000_000, 0)} + cache := newFakeCache(t, log) + + log.clientAsks("example.com") + for range 5 { + cache.Refresh(context.Background()) + log.now = log.now.Add(30 * time.Second) + } + + if want := []string{"example.com"}; !slices.Equal(log.lookups, want) { + t.Fatalf("lookups = %q, want %q (one per client query)", log.lookups, want) + } +} + +func TestIPCacheRefreshResolvesRepeatClientQueries(t *testing.T) { + log := &fakeQueryLog{now: time.Unix(1_800_000_000, 0)} + cache := newFakeCache(t, log) + + log.clientAsks("example.com") + cache.Refresh(context.Background()) + log.now = log.now.Add(time.Minute) + log.clientAsks("example.com") + cache.Refresh(context.Background()) + + if want := []string{"example.com", "example.com"}; !slices.Equal(log.lookups, want) { + t.Fatalf("lookups = %q, want %q", log.lookups, want) + } +} + +// "pihole flush" empties the log and restarts ids; queries after it must still be read. +func TestIPCacheRefreshAfterQueryLogReset(t *testing.T) { + log := &fakeQueryLog{now: time.Unix(1_800_000_000, 0)} + cache := newFakeCache(t, log) + + log.clientAsks("a.example") + log.clientAsks("b.example") + cache.Refresh(context.Background()) + + log.queries = nil + log.now = log.now.Add(time.Minute) + log.clientAsks("c.example") + cache.Refresh(context.Background()) + + if !slices.Contains(log.lookups, "c.example") { + t.Fatalf("lookups = %q, want c.example resolved after the reset", log.lookups) + } +} diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index d186e54..1ad5b1f 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -272,24 +272,33 @@ var answeredStatuses = []int{ 17, // answered from stale cache } -// DomainsSeenSince returns the distinct domains Pi-hole answered since the given Unix timestamp. -// Queries Pi-hole blocked are excluded; see answeredStatuses. -func (c *Checker) DomainsSeenSince(since int64) ([]string, error) { +// AnsweredQuery is one query from Pi-hole's log that Pi-hole answered. +type AnsweredQuery struct { + ID int64 + Domain string + Timestamp time.Time +} + +// AnsweredQueriesAfter returns the queries Pi-hole answered with an id above afterID and a +// timestamp at or after since (Unix seconds), in id order. Queries Pi-hole blocked are +// excluded; see answeredStatuses. +func (c *Checker) AnsweredQueriesAfter(afterID, since int64) ([]AnsweredQuery, error) { placeholders := strings.TrimSuffix(strings.Repeat("?,", len(answeredStatuses)), ",") - args := make([]any, 0, len(answeredStatuses)+1) - args = append(args, since) + args := make([]any, 0, len(answeredStatuses)+2) + args = append(args, afterID, since) for _, status := range answeredStatuses { args = append(args, status) } rows, err := c.db.Query(fmt.Sprintf(` - SELECT DISTINCT LOWER(TRIM(domain)) + SELECT id, LOWER(TRIM(domain)), timestamp FROM queries - WHERE timestamp >= ? + WHERE id > ? + AND timestamp >= ? AND domain IS NOT NULL AND TRIM(domain) <> '' AND status IN (%s) - ORDER BY LOWER(TRIM(domain)) + ORDER BY id `, placeholders), args...) if err != nil { return nil, fmt.Errorf("database query failed: %w", err) @@ -298,24 +307,30 @@ func (c *Checker) DomainsSeenSince(since int64) ([]string, error) { _ = rows.Close() }() - var domains []string + var queries []AnsweredQuery for rows.Next() { - var domain string - if err := rows.Scan(&domain); err != nil { + var q AnsweredQuery + var timestamp float64 // FTL v6 stores fractional seconds + if err := rows.Scan(&q.ID, &q.Domain, ×tamp); err != nil { return nil, fmt.Errorf("scan failed: %w", err) } - - if domain == "" { - continue - } - domains = append(domains, domain) + q.Timestamp = time.Unix(0, int64(timestamp*float64(time.Second))) + queries = append(queries, q) } - if err := rows.Err(); err != nil { return nil, fmt.Errorf("rows iteration failed: %w", err) } + return queries, nil +} - return domains, nil +// MaxQueryID returns the highest id in Pi-hole's query log, or 0 when it is empty. It drops +// below an id seen earlier only when the log was flushed or recreated. +func (c *Checker) MaxQueryID() (int64, error) { + var maxID sql.NullInt64 + if err := c.db.QueryRow(`SELECT MAX(id) FROM queries`).Scan(&maxID); err != nil { + return 0, fmt.Errorf("database query failed: %w", err) + } + return maxID.Int64, nil } // DomainMatches reports whether an observed SNI domain corresponds to a known domain, diff --git a/internal/pihole/pihole_test.go b/internal/pihole/pihole_test.go index 499638a..8c3600f 100644 --- a/internal/pihole/pihole_test.go +++ b/internal/pihole/pihole_test.go @@ -27,11 +27,11 @@ func TestSQLiteReadOnlyDSNAbsolutePath(t *testing.T) { } } -// TestDomainsSeenSinceExcludesBlocked pins the trust-cache input: a domain Pi-hole blocked +// TestAnsweredQueriesAfterExcludesBlocked pins the trust-cache input: a domain Pi-hole blocked // is still in the query log, and must not come back as a domain to resolve and trust. // Statuses 2, 3 and 17 are the ones a real Pi-hole (FTL v6.7.1) logged for answered queries; // 1 is what it logged for a gravity-blocked domain. -func TestDomainsSeenSinceExcludesBlocked(t *testing.T) { +func TestAnsweredQueriesAfterExcludesBlocked(t *testing.T) { t.Parallel() checker := newTestChecker(t, "") @@ -57,13 +57,26 @@ func TestDomainsSeenSinceExcludesBlocked(t *testing.T) { } } - got, err := checker.DomainsSeenSince(now - 60) + queries, err := checker.AnsweredQueriesAfter(0, now-60) if err != nil { - t.Fatalf("DomainsSeenSince() error: %v", err) + t.Fatalf("AnsweredQueriesAfter() error: %v", err) + } + var got []string + for _, q := range queries { + got = append(got, q.Domain) } - want := []string{"cached.example", "forwarded.example", "stale-cache.example"} + want := []string{"forwarded.example", "cached.example", "stale-cache.example"} if !slices.Equal(got, want) { - t.Fatalf("DomainsSeenSince() = %v, want %v", got, want) + t.Fatalf("AnsweredQueriesAfter() domains = %v, want %v", got, want) + } + + // The cursor: nothing after the last id read. + maxID, err := checker.MaxQueryID() + if err != nil { + t.Fatalf("MaxQueryID() error: %v", err) + } + if rest, err := checker.AnsweredQueriesAfter(maxID, now-60); err != nil || len(rest) != 0 { + t.Fatalf("AnsweredQueriesAfter(MaxQueryID()) = %v, %v; want none", rest, err) } } From 0f2cbb8b77c3822e1cb2ab2c7b465fb44a44113b Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 13:37:44 +0600 Subject: [PATCH 43/56] feat: refuse enforce mode without root, warn in watch mode Without root, ss can't name other users' programs, ss -K can't close sockets and other users' processes can't be signalled, so enforce mode started and then failed every kill quietly. It now refuses to start; watch mode, which still sees connections, starts with a warning. A hexwall.db owned by another user (often root, from an earlier sudo run) failed with "attempt to write a readonly database"; the error now says what that means and what to do. --- main.go | 25 +++++++++++++++++++++++++ main_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 main_test.go diff --git a/main.go b/main.go index 3ea3fba..6c83911 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "errors" "flag" "fmt" + "io/fs" "log/slog" "os" "os/signal" @@ -159,6 +160,13 @@ func run() int { } slog.Info("detected OS", "os", runtime.GOOS) + if fatal, msg := privilegeCheck(cfg.mode, os.Geteuid()); fatal { + slog.Error(msg) + return 1 + } else if msg != "" { + slog.Warn(msg) + } + // Cancel background work cleanly on Ctrl+C. ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -216,6 +224,9 @@ func run() int { hexwallStore, err := store.NewStore(cfg.hexwallDBPath) if err != nil { slog.Error("failed to open hexwall database", "path", cfg.hexwallDBPath, "err", err) + if strings.Contains(err.Error(), "readonly database") || errors.Is(err, fs.ErrPermission) { + slog.Error("the database file belongs to another user (often root, from an earlier sudo run): run with sudo, or pick another --hexwall-db") + } return 1 } defer func() { @@ -430,3 +441,17 @@ Kept on purpose: this binary delete it yourself if you installed it (e.g. /usr/local/bin/hexwall)`) return nil } + +// privilegeCheck says what running as euid means for mode. Without root, ss can't name +// other users' programs, ss -K can't close sockets (it needs CAP_NET_ADMIN), and other +// users' processes can't be signalled. Enforce mode would then report kills it can't +// make, so it refuses to start; watch mode still sees connections and only warns. +func privilegeCheck(mode string, euid int) (fatal bool, msg string) { + if euid == 0 { + return false, "" + } + if mode == monitor.ModeEnforce { + return true, "enforce mode needs root: without it hexwall can't close connections or stop other users' programs (run with sudo)" + } + return false, "not running as root: other users' programs show without a name, and Zeek's logs may be unreadable (run with sudo for the full picture)" +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..9b23676 --- /dev/null +++ b/main_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" + + "github.com/hexbytedev/hexwall/internal/monitor" +) + +func TestPrivilegeCheck(t *testing.T) { + tests := []struct { + name string + mode string + euid int + wantFatal bool + wantMsg bool + }{ + {"root enforce", monitor.ModeEnforce, 0, false, false}, + {"root watch", monitor.ModeWatch, 0, false, false}, + {"user enforce refuses", monitor.ModeEnforce, 1000, true, true}, + {"user watch warns", monitor.ModeWatch, 1000, false, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fatal, msg := privilegeCheck(tt.mode, tt.euid) + if fatal != tt.wantFatal || (msg != "") != tt.wantMsg { + t.Fatalf("privilegeCheck(%q, %d) = %v, %q; want fatal %v, message %v", tt.mode, tt.euid, fatal, msg, tt.wantFatal, tt.wantMsg) + } + }) + } +} From 5e26818e90811601992d9f76a6f510f13f7dec52 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 13:37:44 +0600 Subject: [PATCH 44/56] style(pihole): gofmt IPCache constructor --- internal/pihole/cache.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pihole/cache.go b/internal/pihole/cache.go index ab2d0c3..256d7e4 100644 --- a/internal/pihole/cache.go +++ b/internal/pihole/cache.go @@ -49,8 +49,8 @@ type IPCache struct { // NewIPCache creates an IPCache backed by the given Checker and Store. func NewIPCache(checker *Checker, store *store.Store) *IPCache { return &IPCache{ - checker: checker, - store: store, + checker: checker, + store: store, answeredQueriesAfter: checker.AnsweredQueriesAfter, maxQueryID: checker.MaxQueryID, lookupHost: net.DefaultResolver.LookupHost, From 9b73cfc6b7e7c1fb300eb3e5c9fb0084e2ec17f2 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 14:34:26 +0600 Subject: [PATCH 45/56] fix(zeek): say once that notice.log is missing, and when it appears The notice watcher polls every 2s and logged "cannot open notice log" on every poll until Zeek wrote its first alert. The tailer now remembers what it last reported about the file and logs only when that changes: once while waiting (normal until Zeek's first alert), once when the file is found, and once as a warning when it exists but can't be opened. A file that appears after going missing is read from the start, so no alert in it is lost. --- internal/zeek/notice.go | 37 ++++++++++++++++++++++++++++++- internal/zeek/notice_test.go | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/internal/zeek/notice.go b/internal/zeek/notice.go index a9cdc1e..21e4a94 100644 --- a/internal/zeek/notice.go +++ b/internal/zeek/notice.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "io/fs" "log/slog" "math" "os" @@ -152,8 +153,19 @@ type noticeTailer struct { offset int64 parser noticeParser warnedBadLine bool + state fileState // what was last reported about the file, so each change is logged once } +// fileState is what the tailer last found at its path. +type fileState int + +const ( + fileUnchecked fileState = iota + fileMissing // Zeek writes notice.log at its first alert, so this is normal for a while + fileUnreadable // exists but can't be opened, e.g. permissions + fileOpen +) + // skipExisting reads the file as it is now without sending anything: the parser still // learns the TSV "#fields" header at the top, but alerts already in the file are not // replayed. In enforce mode an old alert would otherwise trigger a kill on every restart. @@ -171,10 +183,14 @@ func (t *noticeTailer) poll(events chan<- Event) { func (t *noticeTailer) read(events chan<- Event) { file, err := os.Open(t.path) if err != nil { - slog.Debug("zeek: cannot open notice log", "path", t.path, "err", err) + t.reportOpenError(err) return } defer func() { _ = file.Close() }() + if t.state != fileOpen { + slog.Info("zeek: notice log found, watching for DNS-bypass alerts", "path", t.path) + t.state = fileOpen + } fi, err := file.Stat() if err != nil { @@ -235,6 +251,25 @@ func (t *noticeTailer) read(events chan<- Event) { } } +// reportOpenError logs why notice.log can't be read, once per change of reason, so a +// 2-second poll doesn't repeat it. +func (t *noticeTailer) reportOpenError(err error) { + if errors.Is(err, fs.ErrNotExist) { + if t.state != fileMissing { + slog.Info("zeek: waiting for notice log; Zeek creates it at its first alert", "path", t.path) + t.state = fileMissing + } + // Whatever appears at this path next is a new file: read it from the start. + t.offset = 0 + t.parser = noticeParser{} + return + } + if t.state != fileUnreadable { + slog.Warn("zeek: cannot open notice log; DNS-bypass alerts are not being read", "path", t.path, "err", err) + t.state = fileUnreadable + } +} + // WatchNoticeLog tails a Zeek notice log and emits parsed DNS-bypass Events. func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error { logPath := strings.TrimSpace(path) diff --git a/internal/zeek/notice_test.go b/internal/zeek/notice_test.go index b8046b4..01d8aca 100644 --- a/internal/zeek/notice_test.go +++ b/internal/zeek/notice_test.go @@ -1,7 +1,9 @@ package zeek import ( + "bytes" "errors" + "log/slog" "os" "path/filepath" "strings" @@ -237,3 +239,43 @@ func TestDNSBypassScriptEmbedded(t *testing.T) { t.Fatal("embedded script does not write the SNI to sub, which parseLine reads") } } + +// The tailer polls every 2 s. A missing notice.log is normal until Zeek's first alert, so +// "waiting" must be logged once, not on every poll, and the file appearing must be logged. +// Not parallel: it swaps the default logger. +func TestNoticeTailer_LogsFileStateOnce(t *testing.T) { + var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + path := filepath.Join(t.TempDir(), "notice.log") + tailer := ¬iceTailer{path: path} + events := make(chan Event, 10) + + tailer.skipExisting() + for range 5 { + tailer.poll(events) + } + if got := strings.Count(logs.String(), "waiting for notice log"); got != 1 { + t.Fatalf("logged 'waiting' %d times over 6 polls, want 1:\n%s", got, logs.String()) + } + + data, err := os.ReadFile(filepath.Join("testdata", "notice.tsv")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + for range 3 { + tailer.poll(events) + } + if got := strings.Count(logs.String(), "notice log found"); got != 1 { + t.Fatalf("logged 'found' %d times, want 1:\n%s", got, logs.String()) + } + // Everything in a file that appears after startup is new, so its alert is delivered. + if len(events) != 1 { + t.Fatalf("got %d events from the newly created file, want 1", len(events)) + } +} From 869ab046898afc461c85cf46517e775c8c467959 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 14:34:26 +0600 Subject: [PATCH 46/56] fix(monitor): Zeek alerts respect the allowlist and local network HandleZeekEvent decided "blocked" from rung 4 alone. A domain on both gravity and the Pi-hole allowlist (the allowlist's main use: undoing a blocklist's mistake) is answered by Pi-hole and spared by the scan, but a bypass alert about it killed the connection in enforce mode. A connection to the local network could be killed the same way. Alerts now go through rungs 1, 3 and 4 in the scan's order. An allowlist lookup error is reported instead of read as "not allowed", so it can't lead to a kill. --- internal/monitor/monitor.go | 20 ++++++++++++- internal/monitor/monitor_test.go | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 175b4ae..b68fc1d 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -104,7 +104,7 @@ func (s *Scanner) HandleZeekEvent(event zeek.Event) { } selectedMode := normalizeMode(s.Mode) - blocked, err := s.Checker.IsBlockedByPolicy(event.SNI) + blocked, err := s.alertBlockedByPolicy(event) if err != nil { slog.Warn("zeek policy lookup failed", "sni", event.SNI, "err", err) return @@ -144,6 +144,24 @@ func (s *Scanner) HandleZeekEvent(event zeek.Event) { } } +// alertBlockedByPolicy applies the ladder's policy rungs to a Zeek alert, in Run's order: +// rung 1 (local network), rung 3 (Pi-hole allowlist, which overrides gravity, as it does +// in Pi-hole itself) and only then rung 4 (denylist, gravity, regex). An allowlist lookup +// error is returned rather than read as "not allowed", so it can never lead to a kill. +func (s *Scanner) alertBlockedByPolicy(event zeek.Event) (bool, error) { + if ip := net.ParseIP(event.DstIP); ip != nil && allowlist.Contains(ip) { + return false, nil + } + allowed, err := s.Checker.IsAllowedByPolicy(event.SNI) + if err != nil { + return false, fmt.Errorf("allowlist lookup: %w", err) + } + if allowed { + return false, nil + } + return s.Checker.IsBlockedByPolicy(event.SNI) +} + // findLocalConnection looks a Zeek notice's connection up in this machine's socket list. // Zeek names a connection from the client's side (orig port -> resp IP:port); for a // connection this machine opened, that is its local port and remote address. diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index e77782f..2be8ef8 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -355,6 +355,57 @@ func TestHandleZeekEventEnforceKillsLocalConnection(t *testing.T) { } } +// The allowlist exists to override gravity: allowlisting a domain a blocklist wrongly +// caught is its main use. Pi-hole answers such a domain, and the scan spares it at rung 3; +// a Zeek alert about it must not kill it either. +func TestHandleZeekEventAllowlistedDomainNotKilled(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + if _, err := db.Exec(`INSERT INTO gravity (domain) VALUES ('cdn.example.com')`); err != nil { + t.Fatal(err) + } + insertDomain(t, db, "cdn.example.com", 0) // allowlist + }) + hexwallStore, dbPath := newTestStore(t) + + local := connlist.Connection{LPort: "55881", RAddress: "203.0.113.20:443", RPort: "443", PID: "1", Program: "firefox"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{local}, nil }, + } + event := zeek.Event{SrcIP: "192.168.0.223", SrcPort: "55881", DstIP: "203.0.113.20", DstPort: "443", SNI: "cdn.example.com"} + scanner.HandleZeekEvent(event) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0: the domain is allowlisted", got) + } + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil || alert == nil { + t.Fatalf("RecentZeekAlert() = %v, %v; want the alert recorded", alert, err) + } + if alert.Blocked { + t.Fatal("alert recorded as blocked; Pi-hole allows this domain") + } +} + +// Rung 1 applies to Zeek alerts too: a connection to the local network is never killed. +func TestHandleZeekEventLocalNetworkNotKilled(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "nas.example.com", 1) // denylist + }) + hexwallStore, dbPath := newTestStore(t) + + local := connlist.Connection{LPort: "55881", RAddress: "192.168.0.10:443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{local}, nil }, + } + scanner.HandleZeekEvent(zeek.Event{SrcIP: "192.168.0.223", SrcPort: "55881", DstIP: "192.168.0.10", DstPort: "443", SNI: "nas.example.com"}) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0: the destination is on the local network", got) + } +} + // A connection that isn't in this machine's socket list belongs to another device (or has // closed). hexwall cannot close a socket on another machine, so it records and warns. func TestHandleZeekEventEnforceOtherDeviceNotKilled(t *testing.T) { From 545eee313bc31cf816d6cc356b6e1d5f79307c7e Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 14:47:01 +0600 Subject: [PATCH 47/56] fix: record what happened to each kill, not just that one was tried applyBlockedOutcome wrote the killed_connections row before the guardrails ran, so a kill refused for PID 1, hexwall itself or sshd, or one that failed, read as a kill in the audit log. Writing first is kept so the row survives a crash mid-kill, but it now starts as "attempting" and is updated to killed, refused or failed. Guardrail refusals wrap enforcer.ErrRefused so callers can tell them from failures. Existing databases gain the outcome column on startup; their old rows say "unknown". --- README.md | 2 +- internal/enforcer/enforcer.go | 15 ++++--- internal/enforcer/enforcer_test.go | 16 ++++++++ internal/monitor/monitor.go | 30 ++++++++++++-- internal/monitor/monitor_test.go | 17 ++++---- internal/store/store.go | 66 ++++++++++++++++++++++++++---- internal/store/store_test.go | 61 +++++++++++++++++++++++++++ 7 files changed, 182 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 0897ea9..8aa4ab3 100644 --- a/README.md +++ b/README.md @@ -234,7 +234,7 @@ Connections are not killed in these cases: - The IP reputation API returns HTTP `403`, treated as clean/private/reserved. - The reputation lookup itself fails — a failed lookup is never treated as a block. -When a kill does happen, hexwall first records the event in the local `killed_connections` audit table, then runs its guardrails: it refuses PID 1, its own PID, invalid PIDs, and a hardcoded set of protected system processes (`systemd`, `sshd`, `dockerd`, and similar), whichever check triggered the match. It then closes the connection in one of two ways: +When a kill does happen, hexwall first records it in the local `killed_connections` audit table with the outcome `attempting`, so the record exists even if hexwall dies mid-kill. It then runs its guardrails: it refuses PID 1, its own PID, invalid PIDs, and a hardcoded set of protected system processes (`systemd`, `sshd`, `dockerd`, and similar), whichever check triggered the match. Once the attempt ends, the row's `outcome` says what happened: `killed`, `refused` (a guardrail declined it), or `failed` (no usable PID, or the kill itself errored). Rows written before this column existed say `unknown`. A kill the guardrails allow closes the connection in one of two ways: 1. **Just that connection**: `ss -K` asks the kernel to destroy the single socket, leaving every other connection the process holds untouched. This needs `CAP_NET_ADMIN` (e.g. running as root) and a kernel built with `CONFIG_INET_DIAG_DESTROY`. Because `ss -K` can exit successfully even when the kernel refused, hexwall re-lists the socket afterwards and only counts the kill if the connection is actually gone. 2. **The whole process** (fallback): if the connection is still there, or `ss` is unavailable (always the case on macOS), hexwall signals the owning PID: `SIGTERM` first, escalating to `SIGKILL` after a 2-second grace period if the process is still alive. This also drops every other connection that process holds. diff --git a/internal/enforcer/enforcer.go b/internal/enforcer/enforcer.go index 3fc0838..e8879ab 100644 --- a/internal/enforcer/enforcer.go +++ b/internal/enforcer/enforcer.go @@ -10,6 +10,7 @@ package enforcer import ( + "errors" "fmt" "log/slog" "os" @@ -48,27 +49,31 @@ func isProtected(name string) bool { return protectedPrograms[lower] } +// ErrRefused marks a kill the guardrails declined on purpose, as opposed to one that was +// attempted and failed. Test for it with errors.Is. +var ErrRefused = errors.New("kill refused by guardrail") + // checkGuardrails applies the safety checks shared by both kill paths below, -// before either one is attempted: +// before either one is attempted. Every refusal wraps ErrRefused: // - Refuses any PID <= 0. // - Refuses PID 1 (init). // - Refuses hexwall's own PID. // - Refuses protected programs (case-insensitive match). func checkGuardrails(pid int, program string) error { if pid <= 0 { - return fmt.Errorf("invalid PID %d", pid) + return fmt.Errorf("%w: invalid PID %d", ErrRefused, pid) } if pid == 1 { - return fmt.Errorf("refusing to signal init (PID 1)") + return fmt.Errorf("%w: refusing to signal init (PID 1)", ErrRefused) } if pid == os.Getpid() { - return fmt.Errorf("refusing to signal own process (PID %d)", pid) + return fmt.Errorf("%w: refusing to signal own process (PID %d)", ErrRefused, pid) } if isProtected(program) { - return fmt.Errorf("refusing to kill protected program %q (PID %d)", program, pid) + return fmt.Errorf("%w: refusing to kill protected program %q (PID %d)", ErrRefused, program, pid) } return nil diff --git a/internal/enforcer/enforcer_test.go b/internal/enforcer/enforcer_test.go index a6ba22b..89c1e30 100644 --- a/internal/enforcer/enforcer_test.go +++ b/internal/enforcer/enforcer_test.go @@ -1,6 +1,7 @@ package enforcer import ( + "errors" "os" "strings" "testing" @@ -202,3 +203,18 @@ func TestIsProtectedMixedCaseAllEntries(t *testing.T) { } } } + +// Callers tell "declined on purpose" from "tried and failed" with errors.Is(err, ErrRefused). +func TestGuardrailRefusalsWrapErrRefused(t *testing.T) { + for _, tc := range []struct { + pid int + program string + }{{0, "curl"}, {1, "curl"}, {os.Getpid(), "hexwall"}, {4242, "sshd"}} { + if err := checkGuardrails(tc.pid, tc.program); !errors.Is(err, ErrRefused) { + t.Errorf("checkGuardrails(%d, %q) = %v, want an ErrRefused", tc.pid, tc.program, err) + } + } + if err := checkGuardrails(4242, "curl"); err != nil { + t.Errorf("checkGuardrails(4242, curl) = %v, want nil", err) + } +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index b68fc1d..fae9e34 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -4,6 +4,7 @@ package monitor import ( "context" + "errors" "fmt" "log/slog" "net" @@ -494,23 +495,44 @@ func (s *Scanner) applyBlockedOutcome(ipStr string, conn connlist.Connection) { return } - if err := s.Store.LogKill(ipStr, conn.PID, conn.Program); err != nil { + // The audit row is written before the attempt, so it exists even if the kill never + // returns, and then updated with what actually happened. + rowID, err := s.Store.LogKill(ipStr, conn.PID, conn.Program) + if err != nil { slog.Error("failed to log kill", "address", conn.RAddress, "err", err) } + record := func(outcome store.KillOutcome) { + if rowID == 0 { + return + } + + if err := s.Store.SetKillOutcome(rowID, outcome); err != nil { + slog.Error("failed to record kill outcome", "address", conn.RAddress, "outcome", outcome, "err", err) + } + } + pidInt, err := enforcer.ParsePID(conn.PID) if err != nil { slog.Warn("invalid PID, skipping kill", "pid", conn.PID, "address", conn.RAddress, "err", err) + record(store.KillFailed) return } // Prefers a connection-level kill (ss -K) so any other connection the same // process holds is left untouched; falls back to killing the whole process // when that isn't possible. See enforcer.KillConnection. - if err := enforcer.KillConnection(pidInt, conn.Program, conn.LPort, ipStr, conn.RPort); err != nil { - slog.Warn("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) - } else { + err = enforcer.KillConnection(pidInt, conn.Program, conn.LPort, ipStr, conn.RPort) + switch { + case err == nil: slog.Info("killed connection", "address", conn.RAddress) + record(store.KillKilled) + case errors.Is(err, enforcer.ErrRefused): + slog.Warn("kill refused by guardrail", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "err", err) + record(store.KillRefused) + default: + slog.Warn("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) + record(store.KillFailed) } } diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index 2be8ef8..ef03d1d 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -468,13 +468,16 @@ func TestApplyBlockedOutcomeWatchModeNeverLogsKill(t *testing.T) { func TestApplyBlockedOutcomeEnforceModeLogsKillBeforeAttemptingIt(t *testing.T) { hexwallStore, dbPath := newTestStore(t) - // PID 1 (init) is guaranteed to be refused by enforcer.KillProcess, but the - // audit row must still be written first ("audit survives the kill"). + // PID 1 (init) is always refused by the guardrails. The audit row is still written + // first ("audit survives the kill"), but must then say the kill was refused, not done. conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} (&Scanner{Store: hexwallStore, Mode: ModeEnforce}).applyBlockedOutcome("203.0.113.10", conn) - if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.10"); got != 1 { - t.Fatalf("killed_connections count = %d, want 1 even though the kill itself was refused", got) + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ? AND outcome = 'refused'`, "203.0.113.10"); got != 1 { + t.Fatalf("refused rows = %d, want 1: a refused kill must be recorded as refused", got) + } + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE outcome = 'killed'`); got != 0 { + t.Fatalf("killed rows = %d, want 0: nothing was killed", got) } } @@ -484,9 +487,9 @@ func TestApplyBlockedOutcomeInvalidPIDLogsWarningAndReturns(t *testing.T) { conn := connlist.Connection{PID: "not-a-pid", Program: "curl", RAddress: "203.0.113.11:443"} (&Scanner{Store: hexwallStore, Mode: ModeEnforce}).applyBlockedOutcome("203.0.113.11", conn) - // LogKill happens before ParsePID, so the audit row is still written. - if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.11"); got != 1 { - t.Fatalf("killed_connections count = %d, want 1 (audit before parse failure)", got) + // LogKill happens before ParsePID, so the audit row exists, marked failed. + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ? AND outcome = 'failed'`, "203.0.113.11"); got != 1 { + t.Fatalf("failed rows = %d, want 1 (audit before parse failure, then marked failed)", got) } } diff --git a/internal/store/store.go b/internal/store/store.go index 658777a..eaa0afc 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -51,7 +51,8 @@ CREATE TABLE IF NOT EXISTS killed_connections ( ip TEXT NOT NULL, pid TEXT NOT NULL, program TEXT NOT NULL, - killed_at INTEGER NOT NULL + killed_at INTEGER NOT NULL, + outcome TEXT NOT NULL DEFAULT 'unknown' ); CREATE TABLE IF NOT EXISTS fraud_checks ( @@ -249,6 +250,23 @@ func migrateSNITable(db *sql.DB) error { return nil } +// migrateKillOutcome adds the outcome column to a killed_connections table created before +// it existed. Rows written before then are left as 'unknown': at the time, a refused or +// failed kill was recorded the same way as a real one. +func migrateKillOutcome(db *sql.DB) error { + var has int + if err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info('killed_connections') WHERE name = 'outcome'`).Scan(&has); err != nil { + return fmt.Errorf("query table info: %w", err) + } + if has > 0 { + return nil + } + if _, err := db.Exec(`ALTER TABLE killed_connections ADD COLUMN outcome TEXT NOT NULL DEFAULT 'unknown'`); err != nil { + return fmt.Errorf("add outcome column: %w", err) + } + return nil +} + // NewStore opens or creates the hexwall database at dbPath and applies the schema. func NewStore(dbPath string) (*Store, error) { readWrite, err := sql.Open("sqlite", sqliteDSN(dbPath, "rwc")) @@ -281,6 +299,11 @@ func NewStore(dbPath string) (*Store, error) { return nil, fmt.Errorf("failed to re-apply schema after migration: %w", err) } + if err := migrateKillOutcome(readWrite); err != nil { + _ = readWrite.Close() + return nil, fmt.Errorf("failed to migrate killed_connections: %w", err) + } + if _, err := readWrite.Exec(backfillAllowedIPDomains); err != nil { _ = readWrite.Close() return nil, fmt.Errorf("failed to backfill allowed ip domains: %w", err) @@ -580,17 +603,44 @@ func (s *Store) UpsertDomainCheck(domain string, shouldBlock bool) error { return nil } -// LogKill records a killed connection in the audit log. -func (s *Store) LogKill(ip, pid, program string) error { - _, err := s.readWrite.Exec(` - INSERT INTO killed_connections (ip, pid, program, killed_at) - VALUES (?, ?, ?, ?) - `, ip, pid, program, time.Now().Unix()) +// LogKill records a kill about to be attempted, with outcome KillAttempting, and returns +// the row id for SetKillOutcome. Writing first keeps the record even if the kill never +// returns; SetKillOutcome then says whether it happened. +func (s *Store) LogKill(ip, pid, program string) (int64, error) { + result, err := s.readWrite.Exec(` + INSERT INTO killed_connections (ip, pid, program, killed_at, outcome) + VALUES (?, ?, ?, ?, ?) + `, ip, pid, program, time.Now().Unix(), KillAttempting) + if err != nil { + return 0, fmt.Errorf("log kill %s: %w", ip, err) + } + id, err := result.LastInsertId() if err != nil { - return fmt.Errorf("log kill %s: %w", ip, err) + return 0, fmt.Errorf("log kill %s: read row id: %w", ip, err) } + return id, nil +} +// KillOutcome is what happened to a kill recorded by LogKill. +type KillOutcome string + +const ( + // KillAttempting is written before the kill, so the row survives if hexwall dies mid-kill. + KillAttempting KillOutcome = "attempting" + // KillKilled means the connection, or its process, is gone. + KillKilled KillOutcome = "killed" + // KillRefused means a guardrail declined it on purpose (PID 1, hexwall itself, sshd...). + KillRefused KillOutcome = "refused" + // KillFailed means it couldn't be done: no usable PID, or the kill itself errored. + KillFailed KillOutcome = "failed" +) + +// SetKillOutcome records how the kill logged as row id ended. +func (s *Store) SetKillOutcome(id int64, outcome KillOutcome) error { + if _, err := s.readWrite.Exec(`UPDATE killed_connections SET outcome = ? WHERE id = ?`, outcome, id); err != nil { + return fmt.Errorf("set kill outcome %d: %w", id, err) + } return nil } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 3f6eac5..36b0b6a 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1,6 +1,7 @@ package store import ( + "database/sql" "path/filepath" "testing" "time" @@ -1036,3 +1037,63 @@ func TestPruneRemovesStaleIPObservations(t *testing.T) { t.Fatalf("ip_observations count = %d, want 1 fresh row remaining", count) } } + +// A database from before the outcome column existed must gain it on open, and its old +// rows must say 'unknown': back then a refused kill was recorded like a real one. +func TestNewStoreAddsKillOutcomeToOldDatabase(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + old, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := old.Exec(` + CREATE TABLE killed_connections (id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, pid TEXT NOT NULL, program TEXT NOT NULL, killed_at INTEGER NOT NULL); + INSERT INTO killed_connections (ip, pid, program, killed_at) VALUES ('203.0.113.1', '1', 'curl', 1); + `); err != nil { + t.Fatal(err) + } + _ = old.Close() + + s, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() on an old database error = %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + + var outcome string + if err := s.readWrite.QueryRow(`SELECT outcome FROM killed_connections WHERE ip = '203.0.113.1'`).Scan(&outcome); err != nil { + t.Fatalf("read old row: %v", err) + } + if outcome != "unknown" { + t.Fatalf("old row outcome = %q, want %q", outcome, "unknown") + } +} + +func TestLogKillThenSetOutcome(t *testing.T) { + s, err := NewStore(filepath.Join(t.TempDir(), "hexwall.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + + id, err := s.LogKill("203.0.113.2", "4242", "curl") + if err != nil || id == 0 { + t.Fatalf("LogKill() = %d, %v; want a row id", id, err) + } + read := func() string { + var outcome string + if err := s.readWrite.QueryRow(`SELECT outcome FROM killed_connections WHERE id = ?`, id).Scan(&outcome); err != nil { + t.Fatal(err) + } + return outcome + } + if got := read(); got != string(KillAttempting) { + t.Fatalf("outcome before the kill = %q, want %q", got, KillAttempting) + } + if err := s.SetKillOutcome(id, KillKilled); err != nil { + t.Fatal(err) + } + if got := read(); got != string(KillKilled) { + t.Fatalf("outcome after SetKillOutcome = %q, want %q", got, KillKilled) + } +} From 6f5dfac8fdc17fee9e23c994de22b9f0afc933b3 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 15:33:50 +0600 Subject: [PATCH 48/56] test: cover the decision ladder, the kill, IP reputation and zeek-setup Run's SNI rungs (2-6) had no tests: the scanner took a concrete *zeek.Client, whose SNI cache only a real ssl.log and the 5s tail loop can fill. Scanner.ZeekClient is now an SNISource interface (one Lookup method); production passes the Zeek client, tests a fixed map. Client.Lookup answers "no SNI" on a nil receiver, since a nil client in the interface is not a nil interface. New tests walk every rung (allowlist over gravity; denylist, gravity and regex kills; known-for-IP trust; unknown never killed with deghost off; deghost verdicts cached; no SNI goes to the IP path), KillConnection (ss -K targets the exact tuple, process fallback, guardrails before ss), deghost CheckIP/ShouldKill, and zeek-setup/uninstall end to end against a fake zeek-config. No test signals a real process. Coverage: monitor 58->78%, enforcer 54->78%, deghost 42->87%, zeek 55->69%. --- internal/deghost/deghost_test.go | 80 +++++++++ internal/enforcer/enforcer_socket_test.go | 65 ++++++++ internal/monitor/ladder_test.go | 190 ++++++++++++++++++++++ internal/monitor/monitor.go | 13 +- internal/zeek/setup_test.go | 92 +++++++++++ internal/zeek/zeek.go | 5 + internal/zeek/zeek_test.go | 9 + 7 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 internal/monitor/ladder_test.go diff --git a/internal/deghost/deghost_test.go b/internal/deghost/deghost_test.go index 7f2153f..d3852bf 100644 --- a/internal/deghost/deghost_test.go +++ b/internal/deghost/deghost_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" ) func TestCheckDomainOK(t *testing.T) { @@ -282,3 +283,82 @@ func TestShouldBlockDomainSpamAloneDoesNotBlock(t *testing.T) { t.Error("ShouldBlockDomain(spam alone) = true, want false") } } + +func TestCheckIPOK(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _, _ = w.Write([]byte(`{"ip":"203.0.113.9","security":{"is_attacker":true}}`)) + })) + t.Cleanup(srv.Close) + + report, err := NewClient(srv.URL, time.Second).CheckIP(context.Background(), " 203.0.113.9 ") + if err != nil { + t.Fatalf("CheckIP() error = %v", err) + } + if gotPath != "/ip/203.0.113.9" { + t.Fatalf("request path = %q, want /ip/203.0.113.9", gotPath) + } + if report == nil || !report.Security.IsAttacker { + t.Fatalf("CheckIP() report = %+v, want is_attacker decoded", report) + } +} + +// The API answers 403 for private and reserved IPs; that is a clean verdict, not an error. +func TestCheckIPForbiddenIsCleanNotError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + report, err := NewClient(srv.URL, time.Second).CheckIP(context.Background(), "10.0.0.1") + if err != nil || report != nil { + t.Fatalf("CheckIP() = %+v, %v; want nil, nil", report, err) + } +} + +// Any other failure is an error, never a verdict: a failed lookup must not become a kill. +func TestCheckIPErrors(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ip/203.0.113.1" { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte(`{not json`)) + })) + t.Cleanup(srv.Close) + client := NewClient(srv.URL, time.Second) + + for _, ip := range []string{"203.0.113.1", "203.0.113.2", "not-an-ip", ""} { + if report, err := client.CheckIP(context.Background(), ip); err == nil || report != nil { + t.Errorf("CheckIP(%q) = %+v, %v; want nil report and an error", ip, report, err) + } + } + var nilClient *Client + if _, err := nilClient.CheckIP(context.Background(), "203.0.113.1"); err == nil { + t.Error("nil client CheckIP() error = nil, want error") + } +} + +func TestShouldKill(t *testing.T) { + tests := []struct { + name string + security SecurityReport + want bool + }{ + {"abuser", SecurityReport{IsAbuser: true}, true}, + {"attacker", SecurityReport{IsAttacker: true}, true}, + {"threat", SecurityReport{IsThreat: true}, true}, + // Anonymity alone is too noisy to kill on: VPN, Tor and cloud users are not attackers. + {"vpn, tor, proxy, cloud", SecurityReport{IsVPN: true, IsTor: true, IsProxy: true, IsCloudProvider: true}, false}, + {"clean", SecurityReport{}, false}, + } + for _, tt := range tests { + if got := ShouldKill(&IPReport{Security: tt.security}); got != tt.want { + t.Errorf("ShouldKill(%s) = %v, want %v", tt.name, got, tt.want) + } + } + if ShouldKill(nil) { + t.Error("ShouldKill(nil) = true, want false") + } +} diff --git a/internal/enforcer/enforcer_socket_test.go b/internal/enforcer/enforcer_socket_test.go index d5d3a86..23fad1e 100644 --- a/internal/enforcer/enforcer_socket_test.go +++ b/internal/enforcer/enforcer_socket_test.go @@ -1,6 +1,7 @@ package enforcer import ( + "errors" "net" "os" "os/exec" @@ -164,3 +165,67 @@ func TestSocketStillEstablishedNoMatch(t *testing.T) { t.Fatal("socketStillEstablished() = true, want false for a tuple that was never connected") } } + +// ---- KillConnection: guardrails, then ss -K, then the process fallback ---- + +// noSuchPID is above the kernel's pid_max (at most 2^22), so no process can have it: if a +// test ever reaches the process fallback, the signal fails with ESRCH instead of hitting +// a real process. +const noSuchPID = 1 << 30 + +// installRecordingSS installs a fake ss that appends each invocation to a log file and +// answers the recheck with stillThere's connection or with the header alone. +func installRecordingSS(t *testing.T, stillThere bool) (callLog string) { + t.Helper() + callLog = filepath.Join(t.TempDir(), "ss-calls") + answer := `echo "Recv-Q Send-Q Local Address:Port Peer Address:Port"` + if stillThere { + answer = `printf "Recv-Q Send-Q Local Address:Port Peer Address:Port\n0 0 10.0.0.5:51820 203.0.113.10:443\n"` + } + installFakeSS(t, "#!/bin/sh\n"+ + "echo \"$@\" >> '"+callLog+"'\n"+ + "case \"$1\" in\n"+ + " -K) exit 0 ;;\n"+ + " -tn) "+answer+" ;;\n"+ + "esac\n") + return callLog +} + +func TestKillConnectionClosesJustTheSocket(t *testing.T) { + callLog := installRecordingSS(t, false) + + if err := KillConnection(noSuchPID, "curl", "51820", "203.0.113.10", "443"); err != nil { + t.Fatalf("KillConnection() error = %v, want nil: ss -K closed the socket, so no process signal was needed", err) + } + calls, _ := os.ReadFile(callLog) + if !strings.Contains(string(calls), "-K dst 203.0.113.10 dport 443 sport 51820") { + t.Fatalf("ss calls = %q, want an ss -K for exactly this connection", calls) + } +} + +// When ss -K can't close the socket, KillConnection falls back to signalling the process. +// noSuchPID makes that signal fail harmlessly; the error proves the fallback ran and is +// a failure, not a guardrail refusal. +func TestKillConnectionFallsBackToProcessKill(t *testing.T) { + installRecordingSS(t, true) + + err := KillConnection(noSuchPID, "curl", "51820", "203.0.113.10", "443") + if err == nil || !strings.Contains(err.Error(), "SIGTERM") { + t.Fatalf("KillConnection() error = %v, want the SIGTERM fallback's error", err) + } + if errors.Is(err, ErrRefused) { + t.Fatalf("KillConnection() error = %v is ErrRefused, want a plain failure", err) + } +} + +// Guardrails run before anything else: a refused kill never even calls ss. +func TestKillConnectionRefusedBeforeTouchingSS(t *testing.T) { + callLog := installRecordingSS(t, false) + + if err := KillConnection(1, "curl", "51820", "203.0.113.10", "443"); !errors.Is(err, ErrRefused) { + t.Fatalf("KillConnection(PID 1) error = %v, want ErrRefused", err) + } + if _, err := os.Stat(callLog); !os.IsNotExist(err) { + t.Fatal("ss was called for a refused kill; guardrails must run first") + } +} diff --git a/internal/monitor/ladder_test.go b/internal/monitor/ladder_test.go new file mode 100644 index 0000000..e45a56d --- /dev/null +++ b/internal/monitor/ladder_test.go @@ -0,0 +1,190 @@ +package monitor + +import ( + "context" + "database/sql" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/hexbytedev/hexwall/internal/connlist" + "github.com/hexbytedev/hexwall/internal/deghost" + "github.com/hexbytedev/hexwall/internal/store" +) + +// These tests walk Run's decision ladder rung by rung for a connection Zeek saw an SNI +// for. Each rung leaves its verdict in sni_observations.outcome; a kill leaves a row in +// killed_connections (PID 1, so the guardrails refuse it and no real process is touched). + +// fakeSNI is an SNISource with fixed answers, keyed like zeek.Client: "lport|ip|rport". +type fakeSNI map[string]string + +func (f fakeSNI) Lookup(localPort, remoteIP, remotePort string) (string, bool) { + sni, ok := f[localPort+"|"+remoteIP+"|"+remotePort] + return sni, ok +} + +const ladderIP = "203.0.113.50" + +// ladderScan runs one enforce-mode scan of a single connection to ladderIP whose SNI is sni. +func ladderScan(t *testing.T, scanner *Scanner, sni string) { + t.Helper() + conn := connlist.Connection{LPort: "40000", RAddress: ladderIP + ":443", RPort: "443", PID: "1", Program: "curl"} + scanner.Mode = ModeEnforce + scanner.ConnectionLister = func() ([]connlist.Connection, error) { return []connlist.Connection{conn}, nil } + scanner.ZeekClient = fakeSNI{"40000|" + ladderIP + "|443": sni} + scanner.Run(context.Background()) +} + +func sniOutcome(t *testing.T, dbPath, domain string) string { + t.Helper() + db, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + var outcome string + if err := db.QueryRow(`SELECT outcome FROM sni_observations WHERE ip = ? AND domain = ?`, ladderIP, domain).Scan(&outcome); err != nil { + t.Fatalf("no sni_observations row for %s: %v", domain, err) + } + return outcome +} + +func killRows(t *testing.T, dbPath string) int { + t.Helper() + return queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, ladderIP) +} + +// Rung 3 outranks rung 4: an allowlisted domain is trusted even though gravity has it. +func TestRunAllowlistedSNIIsTrusted(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + if _, err := db.Exec(`INSERT INTO gravity (domain) VALUES ('ads.example.com')`); err != nil { + t.Fatal(err) + } + insertDomain(t, db, "ads.example.com", 0) + }) + hexwallStore, dbPath := newTestStore(t) + + ladderScan(t, &Scanner{Checker: checker, Store: hexwallStore}, "ads.example.com") + + if got := sniOutcome(t, dbPath, "ads.example.com"); got != string(store.OutcomePolicyAllowed) { + t.Fatalf("outcome = %q, want %q", got, store.OutcomePolicyAllowed) + } + if got := killRows(t, dbPath); got != 0 { + t.Fatalf("kill rows = %d, want 0", got) + } +} + +// Rung 4, for each way Pi-hole can block a domain: exact denylist, gravity, regex. +func TestRunBlockedSNIIsKilled(t *testing.T) { + for _, tc := range []struct { + name string + sni string + setup func(t *testing.T, db *sql.DB) + }{ + {"denylist", "evil.example.com", func(t *testing.T, db *sql.DB) { insertDomain(t, db, "evil.example.com", 1) }}, + {"gravity", "ads.example.com", func(t *testing.T, db *sql.DB) { + if _, err := db.Exec(`INSERT INTO gravity (domain) VALUES ('ads.example.com')`); err != nil { + t.Fatal(err) + } + }}, + {"regex", "track.example.com", func(t *testing.T, db *sql.DB) { insertDomain(t, db, `^track\.`, 3) }}, + } { + t.Run(tc.name, func(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { tc.setup(t, db) }) + hexwallStore, dbPath := newTestStore(t) + + ladderScan(t, &Scanner{Checker: checker, Store: hexwallStore}, tc.sni) + + if got := sniOutcome(t, dbPath, tc.sni); got != string(store.OutcomePolicyBlocked) { + t.Fatalf("outcome = %q, want %q", got, store.OutcomePolicyBlocked) + } + if got := killRows(t, dbPath); got != 1 { + t.Fatalf("kill rows = %d, want 1", got) + } + }) + } +} + +// Rung 5: a domain Pi-hole resolved to this very IP is trusted. +func TestRunSNIKnownForThisIPIsTrusted(t *testing.T) { + checker := newTestChecker(t, nil) + hexwallStore, dbPath := newTestStore(t) + if err := hexwallStore.UpsertAllowedIPDomain(ladderIP, "cdn.example.com"); err != nil { + t.Fatal(err) + } + + ladderScan(t, &Scanner{Checker: checker, Store: hexwallStore}, "www.cdn.example.com") + + if got := sniOutcome(t, dbPath, "www.cdn.example.com"); got != string(store.OutcomeIPDomainMatch) { + t.Fatalf("outcome = %q, want %q", got, store.OutcomeIPDomainMatch) + } + if got := killRows(t, dbPath); got != 0 { + t.Fatalf("kill rows = %d, want 0", got) + } +} + +// Rung 6 with deghost off (the default): an unknown domain is recorded, never killed, +// and no data leaves the machine. +func TestRunUnknownSNIWithoutDeghostIsNotKilled(t *testing.T) { + checker := newTestChecker(t, nil) + hexwallStore, dbPath := newTestStore(t) + client := deghost.NewClient(failIfCalled(t).URL, time.Second) + + ladderScan(t, &Scanner{Checker: checker, Store: hexwallStore, DeghostClient: client}, "new.example.com") + + if got := sniOutcome(t, dbPath, "new.example.com"); got != string(store.OutcomeUnknown) { + t.Fatalf("outcome = %q, want %q", got, store.OutcomeUnknown) + } + if got := killRows(t, dbPath); got != 0 { + t.Fatalf("kill rows = %d, want 0", got) + } +} + +// Rung 6 with deghost on: a blacklisted verdict kills, and is cached so the next scan +// doesn't ask the API again. +func TestRunUnknownSNIDeghostBlockedIsKilledAndCached(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + _, _ = w.Write([]byte(`{"status":"allowed","blacklisted":true}`)) + })) + t.Cleanup(srv.Close) + + checker := newTestChecker(t, nil) + hexwallStore, dbPath := newTestStore(t) + scanner := &Scanner{Checker: checker, Store: hexwallStore, DeghostClient: deghost.NewClient(srv.URL, time.Second), DeghostDomain: true} + + ladderScan(t, scanner, "bad.example.com") + ladderScan(t, scanner, "bad.example.com") + + if got := killRows(t, dbPath); got != 2 { + t.Fatalf("kill rows = %d, want 2 (one per scan)", got) + } + if got := calls.Load(); got != 1 { + t.Fatalf("deghost called %d times, want 1: the second scan must use the cached verdict", got) + } +} + +// Rung 2 → 7: no SNI for the connection means the IP-only path decides, and no SNI +// verdict is recorded. +func TestRunWithoutSNIUsesIPOnlyPath(t *testing.T) { + checker := newTestChecker(t, nil) + hexwallStore, dbPath := newTestStore(t) + + conn := connlist.Connection{LPort: "40000", RAddress: ladderIP + ":443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, ZeekClient: fakeSNI{}, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{conn}, nil }, + } + scanner.Run(context.Background()) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM sni_observations`); got != 0 { + t.Fatalf("sni_observations rows = %d, want 0", got) + } + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, ladderIP); got != 1 { + t.Fatalf("ip_observations rows = %d, want 1: the IP-only path must have run", got) + } +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index fae9e34..bdfa36d 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -35,7 +35,7 @@ type Scanner struct { Checker *pihole.Checker Store *store.Store DeghostClient *deghost.Client - ZeekClient *zeek.Client + ZeekClient SNISource ConnectionLister func() ([]connlist.Connection, error) Mode string DeghostIP bool @@ -43,6 +43,13 @@ type Scanner struct { Debug bool } +// SNISource answers rung 2: which TLS server name did this connection ask for? In +// production that's *zeek.Client, fed by Zeek's ssl.log; tests use a fixed map, so the +// SNI rungs can be exercised without a Zeek log file. +type SNISource interface { + Lookup(localPort, remoteIP, remotePort string) (sni string, ok bool) +} + // scanSummary tracks per-scan outcome counts for the summary line. type scanSummary struct { connections int @@ -455,12 +462,15 @@ func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlis if err := s.Store.UpsertFraudCheck(ipStr, false); err != nil { slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) } + if s.Debug { slog.Info("unknown but clean IP (private/reserved)", "ip", ipStr, "program", conn.Program) } + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPReserved); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + return "unknown" } @@ -483,6 +493,7 @@ func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlis if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + s.applyBlockedOutcome(ipStr, conn) return "blocked" } diff --git a/internal/zeek/setup_test.go b/internal/zeek/setup_test.go index 46d49f4..8b54b51 100644 --- a/internal/zeek/setup_test.go +++ b/internal/zeek/setup_test.go @@ -1,6 +1,7 @@ package zeek import ( + "bytes" "os" "path/filepath" "slices" @@ -160,3 +161,94 @@ func TestRemoveScript_OnlyDeletesHexwallsOwn(t *testing.T) { t.Fatalf("removeScript(missing) = %v, %v; want false, nil", deleted, err) } } + +// fakeZeek builds a Zeek install in a temp dir (site dir with a local.zeek, node.cfg +// watching iface, zeekctl.cfg) and a fake zeek-config on PATH that points at it. +func fakeZeek(t *testing.T, iface string) (siteDir, prefix string) { + t.Helper() + prefix = t.TempDir() + siteDir = filepath.Join(prefix, "share", "zeek", "site") + for _, dir := range []string{siteDir, filepath.Join(prefix, "etc"), filepath.Join(prefix, "bin")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + files := map[string]string{ + filepath.Join(siteDir, "local.zeek"): "@load misc/detect-traceroute\n", + filepath.Join(prefix, "etc", "node.cfg"): "[zeek]\ntype=standalone\ninterface=" + iface + "\n", + filepath.Join(prefix, "etc", "zeekctl.cfg"): "LogDir = " + filepath.Join(prefix, "logs") + "\n", + filepath.Join(prefix, "bin", "zeek-config"): "#!/bin/sh\ncase \"$1\" in\n --site_dir) echo '" + siteDir + "' ;;\n --prefix) echo '" + prefix + "' ;;\nesac\n", + } + for path, content := range files { + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", filepath.Join(prefix, "bin")+string(os.PathListSeparator)+os.Getenv("PATH")) + return siteDir, prefix +} + +func anyInterface(t *testing.T) string { + t.Helper() + names := interfaceNames() + if len(names) == 0 { + t.Skip("no network interfaces") + } + return names[0] +} + +// The whole user journey: zeek-setup installs and loads the script and prints the right +// hexwall flags; uninstall's Teardown gives local.zeek back exactly as it was. +func TestSetupThenTeardown(t *testing.T) { + siteDir, prefix := fakeZeek(t, anyInterface(t)) + original, _ := os.ReadFile(filepath.Join(siteDir, "local.zeek")) + + var out bytes.Buffer + if err := Setup(&out); err != nil { + t.Fatalf("Setup() error = %v\n%s", err, out.String()) + } + if installed, err := os.ReadFile(filepath.Join(siteDir, scriptName)); err != nil || !bytes.Equal(installed, DNSBypassScript) { + t.Fatalf("script not installed (err %v)", err) + } + for _, want := range []string{ + filepath.Join(prefix, "bin", "zeekctl") + " deploy", + "--zeek-log " + filepath.Join(prefix, "logs", "current", "ssl.log"), + "--zeek-notice-log " + filepath.Join(prefix, "logs", "current", "notice.log"), + } { + if !strings.Contains(out.String(), want) { + t.Errorf("Setup() output is missing %q:\n%s", want, out.String()) + } + } + + if err := Teardown(&bytes.Buffer{}); err != nil { + t.Fatalf("Teardown() error = %v", err) + } + if _, err := os.Stat(filepath.Join(siteDir, scriptName)); !os.IsNotExist(err) { + t.Fatalf("script still installed after Teardown (err %v)", err) + } + if after, _ := os.ReadFile(filepath.Join(siteDir, "local.zeek")); !bytes.Equal(after, original) { + t.Fatalf("local.zeek after Setup+Teardown:\n%q\nwant:\n%q", after, original) + } +} + +// Zeek can't start on an interface that doesn't exist; Setup must say so and fail. +func TestSetupReportsMissingInterface(t *testing.T) { + fakeZeek(t, "hexwall-test-no-such-if0") + + var out bytes.Buffer + err := Setup(&out) + if err == nil { + t.Fatalf("Setup() error = nil, want an error for a missing interface\n%s", out.String()) + } + if !strings.Contains(out.String(), `"hexwall-test-no-such-if0", which does not exist`) { + t.Fatalf("Setup() output doesn't name the missing interface:\n%s", out.String()) + } +} + +func TestZeekctlFromZeekConfig(t *testing.T) { + _, prefix := fakeZeek(t, anyInterface(t)) + got, err := Zeekctl() + if err != nil || got != filepath.Join(prefix, "bin", "zeekctl") { + t.Fatalf("Zeekctl() = %q, %v; want %q", got, err, filepath.Join(prefix, "bin", "zeekctl")) + } +} diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go index e9335d4..837ce6d 100644 --- a/internal/zeek/zeek.go +++ b/internal/zeek/zeek.go @@ -123,6 +123,11 @@ func (c *Client) Close() { // Lookup returns the SNI domain observed for the given connection tuple, // or ("", false) if no matching record exists in the cache. func (c *Client) Lookup(localPort, remoteIP, remotePort string) (string, bool) { + // A nil *Client stored in an interface is not a nil interface, so callers holding one + // can't tell it apart from a real client: answer "no SNI" instead of panicking. + if c == nil { + return "", false + } key := localPort + "|" + remoteIP + "|" + remotePort c.mu.RLock() diff --git a/internal/zeek/zeek_test.go b/internal/zeek/zeek_test.go index 2cb0976..80d3f7d 100644 --- a/internal/zeek/zeek_test.go +++ b/internal/zeek/zeek_test.go @@ -309,3 +309,12 @@ func TestClientProcessNewLines_TSVRotationRelearnsHeader(t *testing.T) { t.Fatalf("Lookup() after rotation = (%q, %v), want (%q, true)", domain, ok, "post-rotation.example") } } + +// A nil *Client can sit inside the scanner's SNISource interface; it must answer +// "no SNI", not panic. +func TestNilClientLookup(t *testing.T) { + var c *Client + if sni, ok := c.Lookup("40000", "203.0.113.1", "443"); ok || sni != "" { + t.Fatalf("nil Client.Lookup() = %q, %v; want \"\", false", sni, ok) + } +} From 5e6c11ef7791b0cf0d3c4ed8c3b23e285054530a Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 17:06:28 +0600 Subject: [PATCH 49/56] fix(allowlist): trust every range that can't be on the internet The static allowlist covered loopback, RFC 1918 and the cloud metadata address only. Connections to Tailscale peers and carrier-grade NAT (100.64.0.0/10), IPv4 link-local (169.254.0.0/16), IPv6 unique local (fc00::/7) and IPv6 link-local (fe80::/10) went through the reputation path like public addresses and, with deghost on, could be killed. A new test checks each range at its edges: the first address inside is trusted, the neighbour just outside is not. --- internal/allowlist/allowlist.go | 21 +++++++++++------- internal/allowlist/allowlist_test.go | 32 +++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/internal/allowlist/allowlist.go b/internal/allowlist/allowlist.go index f8446da..8a3fd30 100644 --- a/internal/allowlist/allowlist.go +++ b/internal/allowlist/allowlist.go @@ -1,6 +1,8 @@ // Package allowlist defines statically trusted IP networks in CIDR notation. -// These networks bypass runtime reputation checks because they cover loopback, -// private-use space, and the link-local cloud metadata endpoint. +// These networks bypass runtime reputation checks because none of them is +// reachable on the public internet: loopback, private-use, shared (CGNAT, +// Tailscale), link-local, and their IPv6 equivalents. hexwall guards data +// leaving for the internet; a connection to one of these can't be that. package allowlist import ( @@ -11,12 +13,15 @@ import ( ) var cidrs = []string{ - "127.0.0.0/8", // entire IPv4 loopback block - "::1/128", // single IPv6 loopback address - "10.0.0.0/8", // RFC 1918 private-use block - "172.16.0.0/12", // RFC 1918 private-use block - "192.168.0.0/16", // RFC 1918 private-use block - "169.254.169.254/32", // link-local cloud metadata endpoint + "127.0.0.0/8", // entire IPv4 loopback block + "::1/128", // single IPv6 loopback address + "10.0.0.0/8", // RFC 1918 private-use block + "172.16.0.0/12", // RFC 1918 private-use block + "192.168.0.0/16", // RFC 1918 private-use block + "100.64.0.0/10", // RFC 6598 shared address space: carrier-grade NAT, Tailscale + "169.254.0.0/16", // IPv4 link-local, including the cloud metadata endpoint 169.254.169.254 + "fc00::/7", // IPv6 unique local addresses, IPv6's private-use space + "fe80::/10", // IPv6 link-local } var nets []*net.IPNet diff --git a/internal/allowlist/allowlist_test.go b/internal/allowlist/allowlist_test.go index 14a947c..c8ba90c 100644 --- a/internal/allowlist/allowlist_test.go +++ b/internal/allowlist/allowlist_test.go @@ -121,7 +121,7 @@ func TestLoadCIDRs(t *testing.T) { { name: "loads built in cidrs", values: cidrs, - wantCIDRs: []string{"127.0.0.0/8", "::1/128", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.169.254/32"}, + wantCIDRs: []string{"127.0.0.0/8", "::1/128", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10", "169.254.0.0/16", "fc00::/7", "fe80::/10"}, }, { name: "skips invalid entries and keeps valid ones", @@ -241,3 +241,33 @@ func TestContainsWithNoNetworks(t *testing.T) { t.Fatal("Contains returned true with empty allowlist") } } + +// The built-in list, checked at the edges of each range: one address inside the edge must +// be trusted, the address just outside must not. Public addresses are never trusted. +func TestBuiltInRangesEdges(t *testing.T) { + trusted := []string{ + "100.64.0.1", "100.127.255.254", // shared space: CGNAT, Tailscale + "::ffff:100.101.2.3", // the same, written as IPv4-mapped IPv6 + "169.254.1.1", "169.254.169.254", // link-local, cloud metadata + "fd7a:115c:a1e0::1", "fc00::1", // IPv6 unique local (the first is Tailscale's) + "fe80::9518:1f57:3e8:56d7", // IPv6 link-local + "192.168.0.223", "10.0.0.1", "127.0.0.1", "::1", + } + public := []string{ + "100.63.255.255", "100.128.0.0", // just outside 100.64.0.0/10 + "169.253.255.255", "169.255.0.0", // just outside 169.254.0.0/16 + "fbff::1", "fe00::1", // just outside fc00::/7 + "fec0::1", // just outside fe80::/10 + "8.8.8.8", "1.1.1.1", "2001:4860:4860::8888", + } + for _, addr := range trusted { + if !Contains(net.ParseIP(addr)) { + t.Errorf("Contains(%s) = false, want true", addr) + } + } + for _, addr := range public { + if Contains(net.ParseIP(addr)) { + t.Errorf("Contains(%s) = true, want false: it can be a public address", addr) + } + } +} From 9107587224e401b73c606aeb23926212789096e7 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 17:32:13 +0600 Subject: [PATCH 50/56] style: split functions into code paragraphs (wsl) Enable the wsl_v5 linter and apply its fixes across the repo: blank lines separate each step of a function, lines stay together only when they're about the same variable, and a return that ends a longer block stands on its own. Consecutive var lines are grouped into one block. Whitespace and declaration grouping only; no behaviour change. --- .golangci.yml | 3 + internal/allowlist/allowlist.go | 2 + internal/allowlist/allowlist_test.go | 11 +++ internal/connlist/backend_test.go | 10 +++ internal/connlist/connlist.go | 1 + internal/connlist/connlist_test.go | 6 ++ internal/connlist/ss_lister.go | 2 + internal/connlist/ss_lister_test.go | 8 ++ internal/deghost/deghost.go | 3 + internal/deghost/deghost_test.go | 32 +++++++ internal/detector/detector_test.go | 18 ++++ internal/enforcer/enforcer.go | 2 + internal/enforcer/enforcer_socket.go | 2 + internal/enforcer/enforcer_socket_test.go | 24 ++++- internal/enforcer/enforcer_test.go | 9 ++ internal/enforcer/enforcer_unix.go | 1 + internal/monitor/ladder_test.go | 15 ++++ internal/monitor/monitor.go | 59 +++++++++++++ internal/monitor/monitor_test.go | 46 ++++++++++ internal/pihole/cache.go | 23 ++++- internal/pihole/cache_test.go | 14 +++ internal/pihole/pihole.go | 34 +++++++- internal/pihole/pihole_test.go | 5 ++ internal/pihole/policy_test.go | 37 ++++++++ internal/pihole/regex.go | 11 +++ internal/pihole/regex_test.go | 15 ++++ internal/service/service.go | 31 +++++++ internal/service/service_test.go | 11 +++ internal/store/store.go | 78 +++++++++++------ internal/store/store_test.go | 101 ++++++++++++++++++++-- internal/zeek/notice.go | 17 ++++ internal/zeek/notice_test.go | 50 ++++++++++- internal/zeek/setup.go | 43 +++++++++ internal/zeek/setup_test.go | 27 ++++++ internal/zeek/zeek.go | 8 ++ internal/zeek/zeek_test.go | 15 ++++ main.go | 31 ++++++- 37 files changed, 760 insertions(+), 45 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 667a405..1d62d49 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,6 +4,9 @@ linters: - staticcheck - errcheck - revive + # Blank lines split each function into "code paragraphs", one step each. + # `golangci-lint run --fix` adds them; CI checks they stay. + - wsl_v5 issues: max-issues-per-linter: 0 diff --git a/internal/allowlist/allowlist.go b/internal/allowlist/allowlist.go index 8a3fd30..0737964 100644 --- a/internal/allowlist/allowlist.go +++ b/internal/allowlist/allowlist.go @@ -37,6 +37,7 @@ func loadCIDRs(values []string) { slog.Error("failed to parse allowlist CIDRs", "err", err) continue } + nets = append(nets, net) } } @@ -49,6 +50,7 @@ func parseCIDR(value string) (*net.IPNet, error) { if ip == nil { return nil, fmt.Errorf("invalid IP or CIDR: %q", value) } + if ip.To4() != nil { cidr += "/32" } else { diff --git a/internal/allowlist/allowlist_test.go b/internal/allowlist/allowlist_test.go index c8ba90c..8565b7d 100644 --- a/internal/allowlist/allowlist_test.go +++ b/internal/allowlist/allowlist_test.go @@ -71,9 +71,11 @@ func TestParseCIDR(t *testing.T) { if err == nil { t.Fatalf("parseCIDR(%q) error = nil, want %q", tc.value, tc.wantErr) } + if err.Error() != tc.wantErr { t.Fatalf("parseCIDR(%q) error = %q, want %q", tc.value, err.Error(), tc.wantErr) } + return } @@ -94,6 +96,7 @@ func TestParseCIDR(t *testing.T) { if ip == nil { t.Fatalf("net.ParseIP(%q) = nil", ipText) } + if !got.Contains(ip) { t.Fatalf("network %q does not contain %q", got.String(), ipText) } @@ -104,6 +107,7 @@ func TestParseCIDR(t *testing.T) { if ip == nil { t.Fatalf("net.ParseIP(%q) = nil", ipText) } + if got.Contains(ip) { t.Fatalf("network %q unexpectedly contains %q", got.String(), ipText) } @@ -143,11 +147,13 @@ func TestLoadCIDRs(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { nets = nil + if tc.name == "keeps existing nets and appends new ones" { existing, err := parseCIDR("127.0.0.0/8") if err != nil { t.Fatalf("parseCIDR(existing) error = %v", err) } + nets = []*net.IPNet{existing} } @@ -161,6 +167,7 @@ func TestLoadCIDRs(t *testing.T) { if nets[i] == nil { t.Fatalf("nets[%d] = nil, want %q", i, wantCIDR) } + if nets[i].String() != wantCIDR { t.Fatalf("nets[%d] = %q, want %q", i, nets[i].String(), wantCIDR) } @@ -176,6 +183,7 @@ func TestContains(t *testing.T) { }() nets = nil + loadCIDRs([]string{"10.0.0.0/8", "::1", "169.254.169.254"}) testCases := []struct { @@ -232,6 +240,7 @@ func TestContains(t *testing.T) { func TestContainsWithNoNetworks(t *testing.T) { originalNets := nets + nets = nil defer func() { nets = originalNets @@ -260,11 +269,13 @@ func TestBuiltInRangesEdges(t *testing.T) { "fec0::1", // just outside fe80::/10 "8.8.8.8", "1.1.1.1", "2001:4860:4860::8888", } + for _, addr := range trusted { if !Contains(net.ParseIP(addr)) { t.Errorf("Contains(%s) = false, want true", addr) } } + for _, addr := range public { if Contains(net.ParseIP(addr)) { t.Errorf("Contains(%s) = true, want false: it can be a public address", addr) diff --git a/internal/connlist/backend_test.go b/internal/connlist/backend_test.go index 20470d2..6bf35e0 100644 --- a/internal/connlist/backend_test.go +++ b/internal/connlist/backend_test.go @@ -37,6 +37,7 @@ func TestSelectBackend(t *testing.T) { for _, bin := range tt.binaries { writeFakeBinary(t, dir, bin, okScript) } + t.Setenv("PATH", dir) lister, name, err := SelectBackend() @@ -44,15 +45,18 @@ func TestSelectBackend(t *testing.T) { if err == nil { t.Fatal("SelectBackend() error = nil, want error") } + return } if err != nil { t.Fatalf("SelectBackend() error = %v, want nil", err) } + if name != tt.wantName { t.Fatalf("SelectBackend() name = %q, want %q", name, tt.wantName) } + if lister == nil { t.Fatal("SelectBackend() lister = nil, want non-nil") } @@ -75,6 +79,7 @@ func TestSelectBackendDarwinNeverCallsSSAvailable(t *testing.T) { if err != nil { t.Fatalf("selectBackend(darwin) error = %v, want nil", err) } + if name != "somo" { t.Fatalf("selectBackend(darwin) name = %q, want %q (ss must never be picked on darwin, even if it claims available)", name, "somo") } @@ -87,9 +92,11 @@ func TestSelectBackendDarwinFailureMessageOmitsSS(t *testing.T) { if err == nil { t.Fatal("selectBackend(darwin) error = nil, want error when somo is also missing") } + if !strings.Contains(err.Error(), "somo") { t.Fatalf("selectBackend(darwin) error = %q, want it to mention somo", err) } + if strings.Contains(err.Error(), "install ss") { t.Fatalf("selectBackend(darwin) error = %q, want it to NOT suggest installing ss on macOS", err) } @@ -102,9 +109,11 @@ func TestSelectBackendLinuxUnaffected(t *testing.T) { if err != nil { t.Fatalf("selectBackend(linux) error = %v, want nil", err) } + if name != "ss" { t.Fatalf("selectBackend(linux) name = %q, want %q", name, "ss") } + if lister == nil { t.Fatal("selectBackend(linux) lister = nil, want non-nil") } @@ -117,6 +126,7 @@ func TestSelectBackendLinuxFailureMessageMentionsBoth(t *testing.T) { if err == nil { t.Fatal("selectBackend(linux) error = nil, want error") } + if !strings.Contains(err.Error(), "ss") || !strings.Contains(err.Error(), "somo") { t.Fatalf("selectBackend(linux) error = %q, want it to mention both ss and somo", err) } diff --git a/internal/connlist/connlist.go b/internal/connlist/connlist.go index 117ee8a..bdc1950 100644 --- a/internal/connlist/connlist.go +++ b/internal/connlist/connlist.go @@ -66,6 +66,7 @@ func selectBackend(goos string, ssAvailable func() bool, somoInstalled func() er // GetEstablishedConnections returns a list of established TCP/UDP connections. func GetEstablishedConnections() ([]Connection, error) { cmd := exec.Command("somo", "-e", "--json", "--no-pager") + output, err := cmd.CombinedOutput() if err != nil { return nil, commandError("failed to run somo", err, output) diff --git a/internal/connlist/connlist_test.go b/internal/connlist/connlist_test.go index ef413f3..7930256 100644 --- a/internal/connlist/connlist_test.go +++ b/internal/connlist/connlist_test.go @@ -15,6 +15,7 @@ func installFakeSomo(t *testing.T, script string) { t.Helper() dir := t.TempDir() + path := filepath.Join(dir, "somo") if err := os.WriteFile(path, []byte(script), 0o755); err != nil { t.Fatalf("write fake somo: %v", err) @@ -57,6 +58,7 @@ echo '[{"proto":"tcp","local_port":"51820","remote_address":"203.0.113.10","remo if err != nil { t.Fatalf("GetEstablishedConnections() error = %v", err) } + if len(conns) != 1 { t.Fatalf("GetEstablishedConnections() len = %d, want 1", len(conns)) } @@ -83,6 +85,7 @@ func TestGetEstablishedConnectionsEmptyArray(t *testing.T) { if err != nil { t.Fatalf("GetEstablishedConnections() error = %v", err) } + if len(conns) != 0 { t.Fatalf("GetEstablishedConnections() len = %d, want 0", len(conns)) } @@ -95,6 +98,7 @@ func TestGetEstablishedConnectionsMalformedJSON(t *testing.T) { if err == nil { t.Fatal("GetEstablishedConnections() error = nil, want error for malformed JSON") } + if !strings.Contains(err.Error(), "failed to parse") { t.Fatalf("GetEstablishedConnections() error = %q, want it to mention %q", err.Error(), "failed to parse") } @@ -107,6 +111,7 @@ func TestGetEstablishedConnectionsNonZeroExit(t *testing.T) { if err == nil { t.Fatal("GetEstablishedConnections() error = nil, want error for non-zero exit") } + if !strings.Contains(err.Error(), "permission denied") { t.Fatalf("GetEstablishedConnections() error = %q, want it to include command output", err.Error()) } @@ -155,6 +160,7 @@ func TestCommandError(t *testing.T) { if got.Error() != tt.want { t.Fatalf("commandError() = %q, want %q", got.Error(), tt.want) } + if !errors.Is(got, wrapped) { t.Fatal("commandError() does not wrap the original error") } diff --git a/internal/connlist/ss_lister.go b/internal/connlist/ss_lister.go index 895959c..3e45fc0 100644 --- a/internal/connlist/ss_lister.go +++ b/internal/connlist/ss_lister.go @@ -23,6 +23,7 @@ func SSAvailable() bool { // the preferred backend when iproute2 is available (Linux, not macOS). func GetEstablishedConnectionsViaSS() ([]Connection, error) { cmd := exec.Command("ss", "-H", "-tnp", "state", "established") + output, err := cmd.CombinedOutput() if err != nil { return nil, commandError("failed to run ss", err, output) @@ -50,6 +51,7 @@ func parseSSOutput(output string) []Connection { if err != nil { continue } + remoteHost, remotePort, err := splitSSAddress(fields[3]) if err != nil { continue diff --git a/internal/connlist/ss_lister_test.go b/internal/connlist/ss_lister_test.go index d795898..5d5e2a2 100644 --- a/internal/connlist/ss_lister_test.go +++ b/internal/connlist/ss_lister_test.go @@ -13,6 +13,7 @@ func installFakeSS(t *testing.T, script string) { t.Helper() dir := t.TempDir() + path := filepath.Join(dir, "ss") if err := os.WriteFile(path, []byte(script), 0o755); err != nil { t.Fatalf("write fake ss: %v", err) @@ -74,9 +75,11 @@ func TestParseSSOutputUnattributedProcess(t *testing.T) { if len(conns) != 1 { t.Fatalf("parseSSOutput() len = %d, want 1", len(conns)) } + if conns[0].Program != "" || conns[0].PID != "" { t.Fatalf("parseSSOutput()[0] = %+v, want empty Program/PID", conns[0]) } + if conns[0].RAddress != "160.79.104.10" || conns[0].RPort != "443" { t.Fatalf("parseSSOutput()[0] remote = %s:%s, want 160.79.104.10:443", conns[0].RAddress, conns[0].RPort) } @@ -90,6 +93,7 @@ func TestParseSSOutputMultipleOwners(t *testing.T) { if len(conns) != 1 { t.Fatalf("parseSSOutput() len = %d, want 1", len(conns)) } + if conns[0].Program != "nginx" || conns[0].PID != "100" { t.Fatalf("parseSSOutput()[0] = %+v, want Program=nginx PID=100", conns[0]) } @@ -105,6 +109,7 @@ func TestParseSSOutputSkipsMalformedLines(t *testing.T) { if len(conns) != 1 { t.Fatalf("parseSSOutput() len = %d, want 1 (malformed lines skipped)", len(conns)) } + if conns[0].Program != "claude" { t.Fatalf("parseSSOutput()[0].Program = %q, want claude", conns[0].Program) } @@ -112,6 +117,7 @@ func TestParseSSOutputSkipsMalformedLines(t *testing.T) { func TestSSAvailable(t *testing.T) { installFakeSS(t, "#!/bin/sh\nexit 0\n") + if !SSAvailable() { t.Fatal("SSAvailable() = false, want true with fake ss on PATH") } @@ -119,6 +125,7 @@ func TestSSAvailable(t *testing.T) { func TestSSAvailableFalseWhenNotOnPath(t *testing.T) { t.Setenv("PATH", t.TempDir()) + if SSAvailable() { t.Fatal("SSAvailable() = true, want false with empty PATH") } @@ -131,6 +138,7 @@ func TestGetEstablishedConnectionsViaSS(t *testing.T) { if err != nil { t.Fatalf("GetEstablishedConnectionsViaSS() error = %v", err) } + if len(conns) != 4 { t.Fatalf("GetEstablishedConnectionsViaSS() len = %d, want 4", len(conns)) } diff --git a/internal/deghost/deghost.go b/internal/deghost/deghost.go index 6ec2d6b..36d67c2 100644 --- a/internal/deghost/deghost.go +++ b/internal/deghost/deghost.go @@ -44,6 +44,7 @@ type SecurityReport struct { // NewClient creates a Client for the given API base URL. func NewClient(baseURL string, timeout time.Duration) *Client { trimmed := strings.TrimRight(baseURL, "/") + return &Client{ baseURL: trimmed, httpClient: &http.Client{ @@ -58,6 +59,7 @@ func (c *Client) CheckIP(ctx context.Context, ip string) (*IPReport, error) { if c == nil { return nil, errors.New("nil deghost client") } + if c.httpClient == nil { return nil, errors.New("nil deghost http client") } @@ -140,6 +142,7 @@ func (c *Client) CheckDomain(ctx context.Context, domain string) (*DomainReport, if c == nil { return nil, errors.New("nil deghost client") } + if c.httpClient == nil { return nil, errors.New("nil deghost http client") } diff --git a/internal/deghost/deghost_test.go b/internal/deghost/deghost_test.go index d3852bf..cf71055 100644 --- a/internal/deghost/deghost_test.go +++ b/internal/deghost/deghost_test.go @@ -28,28 +28,35 @@ func TestCheckDomainOK(t *testing.T) { if r.Method != http.MethodGet { t.Errorf("method = %s, want GET", r.Method) } + if r.URL.Path != "/domain/example.com" { t.Errorf("path = %s, want /domain/example.com", r.URL.Path) } + w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(want) })) defer srv.Close() client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") if err != nil { t.Fatalf("CheckDomain() error = %v", err) } + if got == nil { t.Fatal("CheckDomain() = nil, want report") } + if got.Status != want.Status { t.Errorf("Status = %q, want %q", got.Status, want.Status) } + if got.HasMX != want.HasMX { t.Errorf("HasMX = %v, want %v", got.HasMX, want.HasMX) } + if got.DomainAgeInDays != want.DomainAgeInDays { t.Errorf("DomainAgeInDays = %d, want %d", got.DomainAgeInDays, want.DomainAgeInDays) } @@ -76,13 +83,16 @@ func TestCheckDomainStatusForbiddenDecodesBody(t *testing.T) { defer srv.Close() client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "blocked.example.com") if err != nil { t.Fatalf("CheckDomain() on 403 error = %v", err) } + if got == nil { t.Fatal("CheckDomain() on 403 = nil, want decoded report (not nil like IP endpoint)") } + if got.Status != "not_allowed" { t.Errorf("Status = %q, want %q", got.Status, "not_allowed") } @@ -105,13 +115,16 @@ func TestCheckDomainStatusBadRequestDecodesBody(t *testing.T) { defer srv.Close() client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "bad.example.com") if err != nil { t.Fatalf("CheckDomain() on 400 error = %v", err) } + if got == nil { t.Fatal("CheckDomain() on 400 = nil, want decoded report") } + if !got.Blacklisted { t.Error("Blacklisted = false, want true") } @@ -126,6 +139,7 @@ func TestCheckDomainStatusInternalServerError(t *testing.T) { defer srv.Close() client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") if err == nil { t.Fatalf("CheckDomain() on 500 = %v, want error", got) @@ -141,6 +155,7 @@ func TestCheckDomainStatusGatewayTimeout(t *testing.T) { defer srv.Close() client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") if err == nil { t.Fatalf("CheckDomain() on 504 = %v, want error", got) @@ -156,6 +171,7 @@ func TestCheckDomainStatusClientClosedRequest(t *testing.T) { defer srv.Close() client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") if err == nil { t.Fatalf("CheckDomain() on 499 = %v, want error", got) @@ -167,6 +183,7 @@ func TestCheckDomainMalformedJSON(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") + if _, err := fmt.Fprint(w, `{"status": "allowed" bad json`); err != nil { t.Errorf("fmt.Fprint: %v", err) } @@ -174,6 +191,7 @@ func TestCheckDomainMalformedJSON(t *testing.T) { defer srv.Close() client := NewClient(srv.URL, srv.Client().Timeout) + got, err := client.CheckDomain(context.Background(), "example.com") if err == nil { t.Fatalf("CheckDomain() on bad JSON = %v, want error", got) @@ -184,6 +202,7 @@ func TestCheckDomainNilClient(t *testing.T) { t.Parallel() var c *Client + got, err := c.CheckDomain(context.Background(), "example.com") if err == nil { t.Fatalf("CheckDomain() on nil client = %v, want error", got) @@ -194,6 +213,7 @@ func TestCheckDomainEmptyBaseURL(t *testing.T) { t.Parallel() client := &Client{baseURL: "", httpClient: &http.Client{}} + got, err := client.CheckDomain(context.Background(), "example.com") if err == nil { t.Fatalf("CheckDomain() on empty baseURL = %v, want error", got) @@ -204,6 +224,7 @@ func TestCheckDomainEmptyDomain(t *testing.T) { t.Parallel() client := NewClient("https://example.com", 5) + got, err := client.CheckDomain(context.Background(), " ") if err == nil { t.Fatalf("CheckDomain() on empty domain = %v, want error", got) @@ -214,18 +235,22 @@ func TestCheckDomainNormalized(t *testing.T) { t.Parallel() var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(DomainReport{Status: "allowed"}) })) defer srv.Close() client := NewClient(srv.URL, srv.Client().Timeout) + _, err := client.CheckDomain(context.Background(), " Example.COM ") if err != nil { t.Fatalf("CheckDomain() error = %v", err) } + want := "/domain/example.com" if gotPath != want { t.Errorf("request path = %q, want %q (domain should be trimmed and lowercased)", gotPath, want) @@ -234,6 +259,7 @@ func TestCheckDomainNormalized(t *testing.T) { func TestShouldBlockDomainNilReport(t *testing.T) { t.Parallel() + if ShouldBlockDomain(nil) { t.Error("ShouldBlockDomain(nil) = true, want false") } @@ -286,6 +312,7 @@ func TestShouldBlockDomainSpamAloneDoesNotBlock(t *testing.T) { func TestCheckIPOK(t *testing.T) { var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotPath = r.URL.Path _, _ = w.Write([]byte(`{"ip":"203.0.113.9","security":{"is_attacker":true}}`)) @@ -296,9 +323,11 @@ func TestCheckIPOK(t *testing.T) { if err != nil { t.Fatalf("CheckIP() error = %v", err) } + if gotPath != "/ip/203.0.113.9" { t.Fatalf("request path = %q, want /ip/203.0.113.9", gotPath) } + if report == nil || !report.Security.IsAttacker { t.Fatalf("CheckIP() report = %+v, want is_attacker decoded", report) } @@ -324,6 +353,7 @@ func TestCheckIPErrors(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) return } + _, _ = w.Write([]byte(`{not json`)) })) t.Cleanup(srv.Close) @@ -334,6 +364,7 @@ func TestCheckIPErrors(t *testing.T) { t.Errorf("CheckIP(%q) = %+v, %v; want nil report and an error", ip, report, err) } } + var nilClient *Client if _, err := nilClient.CheckIP(context.Background(), "203.0.113.1"); err == nil { t.Error("nil client CheckIP() error = nil, want error") @@ -358,6 +389,7 @@ func TestShouldKill(t *testing.T) { t.Errorf("ShouldKill(%s) = %v, want %v", tt.name, got, tt.want) } } + if ShouldKill(nil) { t.Error("ShouldKill(nil) = true, want false") } diff --git a/internal/detector/detector_test.go b/internal/detector/detector_test.go index 74d9e1c..7b17bd7 100644 --- a/internal/detector/detector_test.go +++ b/internal/detector/detector_test.go @@ -15,6 +15,7 @@ func installFakeDocker(t *testing.T, script string) { t.Helper() dir := t.TempDir() + path := filepath.Join(dir, "docker") if err := os.WriteFile(path, []byte(script), 0o755); err != nil { t.Fatalf("write fake docker: %v", err) @@ -25,6 +26,7 @@ func installFakeDocker(t *testing.T, script string) { func writeRegularFile(t *testing.T, path string) { t.Helper() + if err := os.WriteFile(path, []byte("fake db"), 0o644); err != nil { t.Fatalf("write file %s: %v", path, err) } @@ -34,6 +36,7 @@ func writeRegularFile(t *testing.T, path string) { // called, for asserting the physical-path branch short-circuits correctly. func failDockerFallback(t *testing.T) func() (string, error) { t.Helper() + return func() (string, error) { t.Fatal("docker fallback should not have been called") return "", nil @@ -78,6 +81,7 @@ func TestFindDBPathPhysicalFound(t *testing.T) { if err != nil { t.Fatalf("findDBPath() error = %v", err) } + if got != physical { t.Fatalf("findDBPath() = %q, want %q", got, physical) } @@ -92,6 +96,7 @@ func TestFindDBPathPhysicalIsDirectoryShortCircuits(t *testing.T) { if err == nil { t.Fatal("findDBPath() error = nil, want error") } + if !strings.Contains(err.Error(), "failed to access physical pi-hole DB") { t.Fatalf("findDBPath() error = %q, want it to mention accessing the physical DB", err) } @@ -110,6 +115,7 @@ func TestFindDBPathFallsBackToDockerWhenPhysicalMissing(t *testing.T) { if err != nil { t.Fatalf("findDBPath() error = %v", err) } + if got != dockerPath { t.Fatalf("findDBPath() = %q, want %q", got, dockerPath) } @@ -127,6 +133,7 @@ func TestFindDBPathBothMissing(t *testing.T) { if err == nil { t.Fatal("findDBPath() error = nil, want error") } + if !strings.Contains(err.Error(), "pi-hole not found") { t.Fatalf("findDBPath() error = %q, want it to mention 'pi-hole not found'", err) } @@ -141,6 +148,7 @@ func TestFindPiholeContainerMatchesByName(t *testing.T) { if err != nil { t.Fatalf("findPiholeContainer() error = %v", err) } + if id != "abc123" { t.Fatalf("findPiholeContainer() = %q, want %q", id, "abc123") } @@ -153,6 +161,7 @@ func TestFindPiholeContainerMatchesByImage(t *testing.T) { if err != nil { t.Fatalf("findPiholeContainer() error = %v", err) } + if id != "def456" { t.Fatalf("findPiholeContainer() = %q, want %q", id, "def456") } @@ -165,6 +174,7 @@ func TestFindPiholeContainerCaseInsensitive(t *testing.T) { if err != nil { t.Fatalf("findPiholeContainer() error = %v", err) } + if id != "ghi789" { t.Fatalf("findPiholeContainer() = %q, want %q", id, "ghi789") } @@ -177,6 +187,7 @@ func TestFindPiholeContainerNoneFound(t *testing.T) { if err == nil { t.Fatal("findPiholeContainer() error = nil, want error") } + if !strings.Contains(err.Error(), "no running pi-hole container found") { t.Fatalf("findPiholeContainer() error = %q, want it to mention no container found", err) } @@ -189,6 +200,7 @@ func TestFindPiholeContainerDockerNotRunning(t *testing.T) { if err == nil { t.Fatal("findPiholeContainer() error = nil, want error") } + if !strings.Contains(err.Error(), "docker ps failed") { t.Fatalf("findPiholeContainer() error = %q, want it to mention 'docker ps failed'", err) } @@ -209,6 +221,7 @@ func TestFindDockerDBPathSuccess(t *testing.T) { if err != nil { t.Fatalf("findDockerDBPath() error = %v", err) } + want := filepath.Join(mountDir, ftlDBFilename) if got != want { t.Fatalf("findDockerDBPath() = %q, want %q", got, want) @@ -225,6 +238,7 @@ func TestFindDockerDBPathNoPiholeMount(t *testing.T) { if err == nil { t.Fatal("findDockerDBPath() error = nil, want error") } + if !strings.Contains(err.Error(), "no /etc/pihole mount detected") { t.Fatalf("findDockerDBPath() error = %q, want it to mention no mount detected", err) } @@ -242,6 +256,7 @@ func TestFindDockerDBPathMountedButDBMissing(t *testing.T) { if err == nil { t.Fatal("findDockerDBPath() error = nil, want error") } + if !strings.Contains(err.Error(), "DB not available at expected path") { t.Fatalf("findDockerDBPath() error = %q, want it to mention DB not available", err) } @@ -254,6 +269,7 @@ func TestFindDockerDBPathContainerNotFound(t *testing.T) { if err == nil { t.Fatal("findDockerDBPath() error = nil, want error") } + if !strings.Contains(err.Error(), "no running pi-hole container found") { t.Fatalf("findDockerDBPath() error = %q, want it to propagate the container-not-found error", err) } @@ -269,6 +285,7 @@ func TestFindDockerDBPathInspectFails(t *testing.T) { if err == nil { t.Fatal("findDockerDBPath() error = nil, want error") } + if !strings.Contains(err.Error(), "docker inspect failed") { t.Fatalf("findDockerDBPath() error = %q, want it to mention 'docker inspect failed'", err) } @@ -284,6 +301,7 @@ func TestFindDockerDBPathMalformedInspectJSON(t *testing.T) { if err == nil { t.Fatal("findDockerDBPath() error = nil, want error") } + if !strings.Contains(err.Error(), "failed to parse docker inspect output") { t.Fatalf("findDockerDBPath() error = %q, want it to mention JSON parse failure", err) } diff --git a/internal/enforcer/enforcer.go b/internal/enforcer/enforcer.go index e8879ab..6032a6d 100644 --- a/internal/enforcer/enforcer.go +++ b/internal/enforcer/enforcer.go @@ -40,6 +40,7 @@ func SetProtectedPrograms(programs map[string]bool) { for k, v := range programs { normalized[strings.ToLower(k)] = v } + protectedPrograms = normalized } @@ -114,6 +115,7 @@ func KillConnection(pid int, program, localPort, remoteIP, remotePort string) er slog.Info("killed single connection", "pid", pid, "program", program, "local_port", localPort, "remote", remoteAddr) return nil } + slog.Debug("connection-level kill unavailable, falling back to killing the process", "err", socketErr, "pid", pid, "program", program) slog.Warn("killing connection", "pid", pid, "program", program, "address", remoteAddr) diff --git a/internal/enforcer/enforcer_socket.go b/internal/enforcer/enforcer_socket.go index c5e46e7..2917f8b 100644 --- a/internal/enforcer/enforcer_socket.go +++ b/internal/enforcer/enforcer_socket.go @@ -30,6 +30,7 @@ func killSocket(localPort, remoteIP, remotePort string) error { if err != nil { return fmt.Errorf("could not verify socket state after ss -K: %w", err) } + if stillEstablished { return fmt.Errorf("ss -K did not close the connection (missing CAP_NET_ADMIN, unsupported kernel, or ss unavailable)") } @@ -48,6 +49,7 @@ func socketStillEstablished(localPort, remoteIP, remotePort string) (bool, error } dataLines := 0 + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { if strings.TrimSpace(line) != "" { dataLines++ diff --git a/internal/enforcer/enforcer_socket_test.go b/internal/enforcer/enforcer_socket_test.go index 23fad1e..c846123 100644 --- a/internal/enforcer/enforcer_socket_test.go +++ b/internal/enforcer/enforcer_socket_test.go @@ -19,6 +19,7 @@ func installFakeSS(t *testing.T, script string) { t.Helper() dir := t.TempDir() + path := filepath.Join(dir, "ss") if err := os.WriteFile(path, []byte(script), 0o755); err != nil { t.Fatalf("write fake ss: %v", err) @@ -52,6 +53,7 @@ func TestKillSocketFailsWhenConnectionStillThere(t *testing.T) { if err == nil { t.Fatal("killSocket() error = nil, want error when the recheck still shows the connection") } + if !strings.Contains(err.Error(), "did not close") { t.Fatalf("killSocket() error = %q, want it to mention the connection was not closed", err) } @@ -72,6 +74,7 @@ func TestKillSocketFailsWhenSSUnavailable(t *testing.T) { func requireSS(t *testing.T) { t.Helper() + if _, err := exec.LookPath("ss"); err != nil { t.Skip("ss not available in this environment") } @@ -88,9 +91,11 @@ func openLoopbackConnection(t *testing.T) (client net.Conn, clientPort, serverPo if err != nil { t.Fatalf("Listen() error = %v", err) } + t.Cleanup(func() { _ = ln.Close() }) accepted := make(chan net.Conn, 1) + go func() { c, err := ln.Accept() if err == nil { @@ -102,9 +107,11 @@ func openLoopbackConnection(t *testing.T) (client net.Conn, clientPort, serverPo if err != nil { t.Fatalf("Dial() error = %v", err) } + t.Cleanup(func() { _ = conn.Close() }) server := <-accepted + t.Cleanup(func() { _ = server.Close() }) local := conn.LocalAddr().(*net.TCPAddr) @@ -122,6 +129,7 @@ func TestSocketStillEstablishedTrueForLiveConnection(t *testing.T) { if err != nil { t.Fatalf("socketStillEstablished() error = %v", err) } + if !still { t.Fatal("socketStillEstablished() = false, want true for a live connection") } @@ -137,18 +145,23 @@ func TestSocketStillEstablishedFalseAfterClose(t *testing.T) { // Closing the local side takes the socket out of ESTABLISHED essentially // immediately, but allow a couple of short retries to avoid flakiness. - var still bool - var err error + var ( + still bool + err error + ) for i := 0; i < 10; i++ { still, err = socketStillEstablished(clientPort, "127.0.0.1", serverPort) if err != nil { t.Fatalf("socketStillEstablished() error = %v", err) } + if !still { break } + time.Sleep(20 * time.Millisecond) } + if still { t.Fatal("socketStillEstablished() = true, want false after the connection was closed") } @@ -161,6 +174,7 @@ func TestSocketStillEstablishedNoMatch(t *testing.T) { if err != nil { t.Fatalf("socketStillEstablished() error = %v", err) } + if still { t.Fatal("socketStillEstablished() = true, want false for a tuple that was never connected") } @@ -178,16 +192,19 @@ const noSuchPID = 1 << 30 func installRecordingSS(t *testing.T, stillThere bool) (callLog string) { t.Helper() callLog = filepath.Join(t.TempDir(), "ss-calls") + answer := `echo "Recv-Q Send-Q Local Address:Port Peer Address:Port"` if stillThere { answer = `printf "Recv-Q Send-Q Local Address:Port Peer Address:Port\n0 0 10.0.0.5:51820 203.0.113.10:443\n"` } + installFakeSS(t, "#!/bin/sh\n"+ "echo \"$@\" >> '"+callLog+"'\n"+ "case \"$1\" in\n"+ " -K) exit 0 ;;\n"+ " -tn) "+answer+" ;;\n"+ "esac\n") + return callLog } @@ -197,6 +214,7 @@ func TestKillConnectionClosesJustTheSocket(t *testing.T) { if err := KillConnection(noSuchPID, "curl", "51820", "203.0.113.10", "443"); err != nil { t.Fatalf("KillConnection() error = %v, want nil: ss -K closed the socket, so no process signal was needed", err) } + calls, _ := os.ReadFile(callLog) if !strings.Contains(string(calls), "-K dst 203.0.113.10 dport 443 sport 51820") { t.Fatalf("ss calls = %q, want an ss -K for exactly this connection", calls) @@ -213,6 +231,7 @@ func TestKillConnectionFallsBackToProcessKill(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "SIGTERM") { t.Fatalf("KillConnection() error = %v, want the SIGTERM fallback's error", err) } + if errors.Is(err, ErrRefused) { t.Fatalf("KillConnection() error = %v is ErrRefused, want a plain failure", err) } @@ -225,6 +244,7 @@ func TestKillConnectionRefusedBeforeTouchingSS(t *testing.T) { if err := KillConnection(1, "curl", "51820", "203.0.113.10", "443"); !errors.Is(err, ErrRefused) { t.Fatalf("KillConnection(PID 1) error = %v, want ErrRefused", err) } + if _, err := os.Stat(callLog); !os.IsNotExist(err) { t.Fatal("ss was called for a refused kill; guardrails must run first") } diff --git a/internal/enforcer/enforcer_test.go b/internal/enforcer/enforcer_test.go index 89c1e30..b4871fb 100644 --- a/internal/enforcer/enforcer_test.go +++ b/internal/enforcer/enforcer_test.go @@ -60,6 +60,7 @@ func TestKillProcessRefusesProtectedPrograms(t *testing.T) { t.Errorf("KillProcess(2, %q) = nil, want error for protected program", prog) continue } + if !strings.Contains(err.Error(), "protected") { t.Errorf("KillProcess(2, %q) error = %q, want it to mention 'protected'", prog, err) } @@ -76,6 +77,7 @@ func TestKillProcessCaseInsensitiveProtectedMatch(t *testing.T) { t.Errorf("KillProcess(2, %q) = nil, want error for case-insensitive protected match", prog) continue } + if !strings.Contains(err.Error(), "protected") { t.Errorf("KillProcess(2, %q) error = %q, want it to mention 'protected'", prog, err) } @@ -130,6 +132,7 @@ func TestKillConnectionRefusesProtectedPrograms(t *testing.T) { t.Errorf("KillConnection(2, %q) = nil, want error for protected program", prog) continue } + if !strings.Contains(err.Error(), "protected") { t.Errorf("KillConnection(2, %q) error = %q, want it to mention 'protected'", prog, err) } @@ -154,6 +157,7 @@ func TestParsePIDValid(t *testing.T) { if err != nil { t.Errorf("ParsePID(%q) error = %v", tt.input, err) } + if got != tt.want { t.Errorf("ParsePID(%q) = %d, want %d", tt.input, got, tt.want) } @@ -178,15 +182,19 @@ func TestIsProtected(t *testing.T) { if !isProtected("sshd") { t.Error("isProtected(\"sshd\") = false, want true") } + if !isProtected("SSHD") { t.Error("isProtected(\"SSHD\") = false, want true (case-insensitive)") } + if !isProtected("SystemD") { t.Error("isProtected(\"SystemD\") = false, want true (case-insensitive)") } + if isProtected("curl") { t.Error("isProtected(\"curl\") = true, want false") } + if isProtected("") { t.Error("isProtected(\"\") = true, want false") } @@ -214,6 +222,7 @@ func TestGuardrailRefusalsWrapErrRefused(t *testing.T) { t.Errorf("checkGuardrails(%d, %q) = %v, want an ErrRefused", tc.pid, tc.program, err) } } + if err := checkGuardrails(4242, "curl"); err != nil { t.Errorf("checkGuardrails(4242, curl) = %v, want nil", err) } diff --git a/internal/enforcer/enforcer_unix.go b/internal/enforcer/enforcer_unix.go index 2f3f764..e916937 100644 --- a/internal/enforcer/enforcer_unix.go +++ b/internal/enforcer/enforcer_unix.go @@ -29,6 +29,7 @@ func terminate(pid int, program string) error { // Process still alive — escalate to SIGKILL. slog.Warn("process survived SIGTERM, sending SIGKILL", "pid", pid, "program", program) + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { return fmt.Errorf("send SIGKILL to PID %d: %w", pid, err) } diff --git a/internal/monitor/ladder_test.go b/internal/monitor/ladder_test.go index e45a56d..801d79b 100644 --- a/internal/monitor/ladder_test.go +++ b/internal/monitor/ladder_test.go @@ -31,6 +31,7 @@ const ladderIP = "203.0.113.50" // ladderScan runs one enforce-mode scan of a single connection to ladderIP whose SNI is sni. func ladderScan(t *testing.T, scanner *Scanner, sni string) { t.Helper() + conn := connlist.Connection{LPort: "40000", RAddress: ladderIP + ":443", RPort: "443", PID: "1", Program: "curl"} scanner.Mode = ModeEnforce scanner.ConnectionLister = func() ([]connlist.Connection, error) { return []connlist.Connection{conn}, nil } @@ -40,15 +41,19 @@ func ladderScan(t *testing.T, scanner *Scanner, sni string) { func sniOutcome(t *testing.T, dbPath, domain string) string { t.Helper() + db, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro") if err != nil { t.Fatal(err) } + defer func() { _ = db.Close() }() + var outcome string if err := db.QueryRow(`SELECT outcome FROM sni_observations WHERE ip = ? AND domain = ?`, ladderIP, domain).Scan(&outcome); err != nil { t.Fatalf("no sni_observations row for %s: %v", domain, err) } + return outcome } @@ -63,6 +68,7 @@ func TestRunAllowlistedSNIIsTrusted(t *testing.T) { if _, err := db.Exec(`INSERT INTO gravity (domain) VALUES ('ads.example.com')`); err != nil { t.Fatal(err) } + insertDomain(t, db, "ads.example.com", 0) }) hexwallStore, dbPath := newTestStore(t) @@ -72,6 +78,7 @@ func TestRunAllowlistedSNIIsTrusted(t *testing.T) { if got := sniOutcome(t, dbPath, "ads.example.com"); got != string(store.OutcomePolicyAllowed) { t.Fatalf("outcome = %q, want %q", got, store.OutcomePolicyAllowed) } + if got := killRows(t, dbPath); got != 0 { t.Fatalf("kill rows = %d, want 0", got) } @@ -101,6 +108,7 @@ func TestRunBlockedSNIIsKilled(t *testing.T) { if got := sniOutcome(t, dbPath, tc.sni); got != string(store.OutcomePolicyBlocked) { t.Fatalf("outcome = %q, want %q", got, store.OutcomePolicyBlocked) } + if got := killRows(t, dbPath); got != 1 { t.Fatalf("kill rows = %d, want 1", got) } @@ -111,6 +119,7 @@ func TestRunBlockedSNIIsKilled(t *testing.T) { // Rung 5: a domain Pi-hole resolved to this very IP is trusted. func TestRunSNIKnownForThisIPIsTrusted(t *testing.T) { checker := newTestChecker(t, nil) + hexwallStore, dbPath := newTestStore(t) if err := hexwallStore.UpsertAllowedIPDomain(ladderIP, "cdn.example.com"); err != nil { t.Fatal(err) @@ -121,6 +130,7 @@ func TestRunSNIKnownForThisIPIsTrusted(t *testing.T) { if got := sniOutcome(t, dbPath, "www.cdn.example.com"); got != string(store.OutcomeIPDomainMatch) { t.Fatalf("outcome = %q, want %q", got, store.OutcomeIPDomainMatch) } + if got := killRows(t, dbPath); got != 0 { t.Fatalf("kill rows = %d, want 0", got) } @@ -138,6 +148,7 @@ func TestRunUnknownSNIWithoutDeghostIsNotKilled(t *testing.T) { if got := sniOutcome(t, dbPath, "new.example.com"); got != string(store.OutcomeUnknown) { t.Fatalf("outcome = %q, want %q", got, store.OutcomeUnknown) } + if got := killRows(t, dbPath); got != 0 { t.Fatalf("kill rows = %d, want 0", got) } @@ -147,8 +158,10 @@ func TestRunUnknownSNIWithoutDeghostIsNotKilled(t *testing.T) { // doesn't ask the API again. func TestRunUnknownSNIDeghostBlockedIsKilledAndCached(t *testing.T) { var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { calls.Add(1) + _, _ = w.Write([]byte(`{"status":"allowed","blacklisted":true}`)) })) t.Cleanup(srv.Close) @@ -163,6 +176,7 @@ func TestRunUnknownSNIDeghostBlockedIsKilledAndCached(t *testing.T) { if got := killRows(t, dbPath); got != 2 { t.Fatalf("kill rows = %d, want 2 (one per scan)", got) } + if got := calls.Load(); got != 1 { t.Fatalf("deghost called %d times, want 1: the second scan must use the cached verdict", got) } @@ -184,6 +198,7 @@ func TestRunWithoutSNIUsesIPOnlyPath(t *testing.T) { if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM sni_observations`); got != 0 { t.Fatalf("sni_observations rows = %d, want 0", got) } + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, ladderIP); got != 1 { t.Fatalf("ip_observations rows = %d, want 1: the IP-only path must have run", got) } diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index bdfa36d..0d5fc3d 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -107,11 +107,13 @@ func (s *Scanner) HandleZeekEvent(event zeek.Event) { if s.Checker == nil || s.Store == nil { return } + if event.SNI == "" { return } selectedMode := normalizeMode(s.Mode) + blocked, err := s.alertBlockedByPolicy(event) if err != nil { slog.Warn("zeek policy lookup failed", "sni", event.SNI, "err", err) @@ -120,17 +122,21 @@ func (s *Scanner) HandleZeekEvent(event zeek.Event) { confidence := "medium" actionTaken := "logged" + var conn connlist.Connection + found := false if blocked { confidence = "high" + slog.Warn("zeek bypass alert", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI, "mode", selectedMode) conn, found = s.findLocalConnection(event) switch { case !found: actionTaken = "not_on_this_host" + slog.Warn("zeek bypass alert: connection is not on this machine (another device, or already closed); hexwall can only kill its own machine's connections", "src_ip", event.SrcIP, "src_port", event.SrcPort, "dst_ip", event.DstIP, "sni", event.SNI) case selectedMode == ModeEnforce: @@ -160,13 +166,16 @@ func (s *Scanner) alertBlockedByPolicy(event zeek.Event) (bool, error) { if ip := net.ParseIP(event.DstIP); ip != nil && allowlist.Contains(ip) { return false, nil } + allowed, err := s.Checker.IsAllowedByPolicy(event.SNI) if err != nil { return false, fmt.Errorf("allowlist lookup: %w", err) } + if allowed { return false, nil } + return s.Checker.IsBlockedByPolicy(event.SNI) } @@ -189,10 +198,12 @@ func (s *Scanner) findLocalConnection(event zeek.Event) (connlist.Connection, bo if conn.LPort != event.SrcPort || conn.RPort != event.DstPort { continue } + if ip, err := remoteIP(conn.RAddress); err == nil && ip.Equal(wantIP) { return conn, true } } + return connlist.Connection{}, false } @@ -216,6 +227,7 @@ func (s *Scanner) Run(ctx context.Context) { if s.ConnectionLister == nil { slog.Warn("connection lister unset; defaulting to somo") + s.ConnectionLister = connlist.GetEstablishedConnections } @@ -236,13 +248,16 @@ func (s *Scanner) Run(ctx context.Context) { } var summary scanSummary + summary.connections = len(connections) for _, conn := range connections { ip, err := remoteIP(conn.RAddress) if err != nil { slog.Warn("invalid IP address", "address", conn.RAddress, "err", err) + summary.errored++ + continue } @@ -252,12 +267,15 @@ func (s *Scanner) Run(ctx context.Context) { if allowlist.Contains(ip) { summary.allowlisted++ summary.trusted++ + continue } // Rung 2: did Zeek see an SNI for this connection? var sniDomain string + sniFound := false + if s.ZeekClient != nil { if domain, ok := s.ZeekClient.Lookup(conn.LPort, ipStr, conn.RPort); ok { sniDomain = domain @@ -274,6 +292,7 @@ func (s *Scanner) Run(ctx context.Context) { // Rung 7: IP-only fallback — no SNI means non-TLS, pre-existing connection, or ECH. action := s.evaluateIPOnly(ctx, ipStr, conn) applyOutcome(&summary, action) + continue } @@ -291,7 +310,9 @@ func (s *Scanner) Run(ctx context.Context) { if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyAllowed); logErr != nil { slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } + summary.trusted++ + continue } @@ -303,15 +324,20 @@ func (s *Scanner) Run(ctx context.Context) { if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyBlocked); logErr != nil { slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } + slog.Warn("policy-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + s.applyBlockedOutcome(ipStr, conn) + summary.blocked++ + continue } } // Rung 5: inferred trust from DNS history — scoped to this IP. known := false + knownDomains, err := s.Store.DomainsForIP(ipStr) if err != nil { slog.Debug("sni domain check failed", "domain", domain, "ip", ipStr, "err", err) @@ -323,10 +349,12 @@ func (s *Scanner) Run(ctx context.Context) { } } } + outcome := store.OutcomeUnknown if known { outcome = store.OutcomeIPDomainMatch } + if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, outcome); logErr != nil { slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } @@ -341,7 +369,9 @@ func (s *Scanner) Run(ctx context.Context) { if s.Debug { slog.Info("unknown domain, deghost disabled; no action", "domain", domain, "ip", ipStr, "program", conn.Program) } + summary.unknown++ + continue } @@ -349,7 +379,9 @@ func (s *Scanner) Run(ctx context.Context) { if cached, err := s.Store.GetRecentDomainCheck(domain); err != nil { slog.Error("domain cache lookup failed", "domain", domain, "err", err) + summary.errored++ + continue } else if cached != nil { shouldBlock = cached.ShouldBlock @@ -360,12 +392,15 @@ func (s *Scanner) Run(ctx context.Context) { report, err := s.DeghostClient.CheckDomain(ctx, domain) if err != nil { slog.Error("deghost domain check failed", "domain", domain, "program", conn.Program, "err", err) + summary.errored++ + continue } shouldBlock = deghost.ShouldBlockDomain(report) reason = "live-domain-check" + if err := s.Store.UpsertDomainCheck(domain, shouldBlock); err != nil { slog.Error("failed to cache domain check", "domain", domain, "program", conn.Program, "err", err) } @@ -375,12 +410,16 @@ func (s *Scanner) Run(ctx context.Context) { if s.Debug { slog.Info("unknown but clean domain", "domain", domain, "ip", ipStr, "program", conn.Program, "reason", reason) } + summary.unknown++ + continue } slog.Warn("deghost-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", reason) + s.applyBlockedOutcome(ipStr, conn) + summary.blocked++ } @@ -395,9 +434,11 @@ func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlis allowed, err := s.Store.IsAllowed(ipStr) if err != nil { slog.Error("store lookup failed", "address", conn.RAddress, "err", err) + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + return "error" } @@ -405,9 +446,11 @@ func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlis if err := s.Store.UpdateEstablished(ipStr); err != nil { slog.Error("failed to update established", "address", conn.RAddress, "err", err) } + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPTrusted); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + return "trusted" } @@ -415,18 +458,22 @@ func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlis if s.Debug { slog.Info("unknown IP, deghost disabled; no action", "ip", ipStr, "program", conn.Program) } + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + return "unknown" } cachedFraudCheck, err := s.Store.GetRecentFraudCheck(ipStr) if err != nil { slog.Error("fraud cache lookup failed", "ip", ipStr, "program", conn.Program, "err", err) + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + return "error" } @@ -435,26 +482,33 @@ func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlis if s.Debug { slog.Info("unknown but clean IP (cached)", "ip", ipStr, "program", conn.Program) } + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + return "unknown" } slog.Warn("deghost-blocked IP (cached)", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + s.applyBlockedOutcome(ipStr, conn) + return "blocked" } report, err := s.DeghostClient.CheckIP(ctx, ipStr) if err != nil { slog.Error("deghost check failed", "ip", ipStr, "program", conn.Program, "err", err) + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + return "error" } @@ -483,18 +537,22 @@ func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlis if s.Debug { slog.Info("unknown but clean IP", "ip", ipStr, "program", conn.Program) } + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } + return "unknown" } slog.Warn("deghost-blocked IP", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } s.applyBlockedOutcome(ipStr, conn) + return "blocked" } @@ -527,6 +585,7 @@ func (s *Scanner) applyBlockedOutcome(ipStr string, conn connlist.Connection) { if err != nil { slog.Warn("invalid PID, skipping kill", "pid", conn.PID, "address", conn.RAddress, "err", err) record(store.KillFailed) + return } diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index ef03d1d..ec3f1ab 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -35,6 +35,7 @@ func newTestStore(t *testing.T) (*store.Store, string) { if err != nil { t.Fatalf("store.NewStore() error = %v", err) } + t.Cleanup(func() { if err := s.Close(); err != nil { t.Errorf("Store.Close() error = %v", err) @@ -62,6 +63,7 @@ func queryInt(t *testing.T, dbPath, query string, args ...any) int { if err := db.QueryRow(query, args...).Scan(&got); err != nil { t.Fatalf("query %q: %v", query, err) } + return got } @@ -89,6 +91,7 @@ func openTestGravityDB(t *testing.T) (*sql.DB, string) { ` if _, err := db.Exec(schema); err != nil { _ = db.Close() + t.Fatalf("create gravity schema: %v", err) } @@ -98,6 +101,7 @@ func openTestGravityDB(t *testing.T) (*sql.DB, string) { // insertDomain inserts a domain into domainlist. type 0 = allowlist, 1 = denylist. func insertDomain(t *testing.T, db *sql.DB, domain string, listType int) { t.Helper() + if _, err := db.Exec(`INSERT INTO domainlist (type, domain, enabled) VALUES (?, ?, 1)`, listType, domain); err != nil { t.Fatalf("insert domain %q type %d: %v", domain, listType, err) } @@ -109,13 +113,16 @@ func newTestChecker(t *testing.T, gravitySetup func(db *sql.DB)) *pihole.Checker t.Helper() ftlPath := filepath.Join(t.TempDir(), "pihole-FTL.db") + ftlDB, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) if err != nil { t.Fatalf("create ftl db: %v", err) } + if _, err := ftlDB.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY, timestamp INTEGER, type INTEGER, domain TEXT, status INTEGER, ip TEXT)`); err != nil { t.Fatalf("create ftl schema: %v", err) } + if err := ftlDB.Close(); err != nil { t.Fatalf("close ftl db: %v", err) } @@ -124,6 +131,7 @@ func newTestChecker(t *testing.T, gravitySetup func(db *sql.DB)) *pihole.Checker if gravitySetup != nil { gravitySetup(gravDB) } + if err := gravDB.Close(); err != nil { t.Fatalf("close gravity db: %v", err) } @@ -132,6 +140,7 @@ func newTestChecker(t *testing.T, gravitySetup func(db *sql.DB)) *pihole.Checker if err != nil { t.Fatalf("pihole.NewChecker() error = %v", err) } + t.Cleanup(func() { if err := checker.Close(); err != nil { t.Errorf("Checker.Close() error = %v", err) @@ -147,6 +156,7 @@ func installFakeSomo(t *testing.T, script string) { t.Helper() dir := t.TempDir() + path := filepath.Join(dir, "somo") if err := os.WriteFile(path, []byte(script), 0o755); err != nil { t.Fatalf("write fake somo: %v", err) @@ -159,10 +169,12 @@ func installFakeSomo(t *testing.T, script string) { // Used to assert a code path never makes a live deghost call. func failIfCalled(t *testing.T) *httptest.Server { t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { t.Errorf("unexpected HTTP call to deghost: %s", r.URL.Path) })) t.Cleanup(srv.Close) + return srv } @@ -187,6 +199,7 @@ func TestNormalizeMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() + if got := normalizeMode(tt.mode); got != tt.want { t.Fatalf("normalizeMode(%q) = %q, want %q", tt.mode, got, tt.want) } @@ -217,16 +230,20 @@ func TestRemoteIP(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() + ip, err := remoteIP(tt.address) if tt.wantErr { if err == nil { t.Fatalf("remoteIP(%q) error = nil, want error", tt.address) } + return } + if err != nil { t.Fatalf("remoteIP(%q) error = %v, want nil", tt.address, err) } + if ip.String() != tt.wantIP { t.Fatalf("remoteIP(%q) = %q, want %q", tt.address, ip.String(), tt.wantIP) } @@ -252,8 +269,10 @@ func TestApplyOutcome(t *testing.T) { for _, tt := range tests { t.Run(tt.outcome, func(t *testing.T) { t.Parallel() + var summary scanSummary applyOutcome(&summary, tt.outcome) + if got := tt.get(summary); got != 1 { t.Fatalf("applyOutcome(%q) counter = %d, want 1", tt.outcome, got) } @@ -262,9 +281,12 @@ func TestApplyOutcome(t *testing.T) { t.Run("unrecognized outcome increments nothing", func(t *testing.T) { t.Parallel() + var want scanSummary + got := want applyOutcome(&got, "garbage") + if got != want { t.Fatalf("applyOutcome(garbage) summary = %+v, want unchanged %+v", got, want) } @@ -287,9 +309,11 @@ func TestHandleZeekEventBlockedDomainPersisted(t *testing.T) { if err != nil { t.Fatalf("RecentZeekAlert() error = %v", err) } + if alert == nil { t.Fatal("RecentZeekAlert() = nil, want persisted alert") } + if !alert.Blocked { t.Fatal("alert.Blocked = false, want true for denylisted domain") } @@ -307,9 +331,11 @@ func TestHandleZeekEventUnblockedDomainPersisted(t *testing.T) { if err != nil { t.Fatalf("RecentZeekAlert() error = %v", err) } + if alert == nil { t.Fatal("RecentZeekAlert() = nil, want persisted alert") } + if alert.Blocked { t.Fatal("alert.Blocked = true, want false for domain not on denylist") } @@ -346,10 +372,12 @@ func TestHandleZeekEventEnforceKillsLocalConnection(t *testing.T) { if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.20"); got != 1 { t.Fatalf("killed_connections count = %d, want 1", got) } + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) if err != nil || alert == nil { t.Fatalf("RecentZeekAlert() = %v, %v; want persisted alert", alert, err) } + if alert.ActionTaken != "kill_attempted" { t.Fatalf("alert.ActionTaken = %q, want %q", alert.ActionTaken, "kill_attempted") } @@ -363,6 +391,7 @@ func TestHandleZeekEventAllowlistedDomainNotKilled(t *testing.T) { if _, err := db.Exec(`INSERT INTO gravity (domain) VALUES ('cdn.example.com')`); err != nil { t.Fatal(err) } + insertDomain(t, db, "cdn.example.com", 0) // allowlist }) hexwallStore, dbPath := newTestStore(t) @@ -378,10 +407,12 @@ func TestHandleZeekEventAllowlistedDomainNotKilled(t *testing.T) { if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { t.Fatalf("killed_connections count = %d, want 0: the domain is allowlisted", got) } + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) if err != nil || alert == nil { t.Fatalf("RecentZeekAlert() = %v, %v; want the alert recorded", alert, err) } + if alert.Blocked { t.Fatal("alert recorded as blocked; Pi-hole allows this domain") } @@ -425,10 +456,12 @@ func TestHandleZeekEventEnforceOtherDeviceNotKilled(t *testing.T) { if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { t.Fatalf("killed_connections count = %d, want 0", got) } + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) if err != nil || alert == nil { t.Fatalf("RecentZeekAlert() = %v, %v; want persisted alert", alert, err) } + if alert.ActionTaken != "not_on_this_host" { t.Fatalf("alert.ActionTaken = %q, want %q", alert.ActionTaken, "not_on_this_host") } @@ -476,6 +509,7 @@ func TestApplyBlockedOutcomeEnforceModeLogsKillBeforeAttemptingIt(t *testing.T) if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ? AND outcome = 'refused'`, "203.0.113.10"); got != 1 { t.Fatalf("refused rows = %d, want 1: a refused kill must be recorded as refused", got) } + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE outcome = 'killed'`); got != 0 { t.Fatalf("killed rows = %d, want 0: nothing was killed", got) } @@ -506,6 +540,7 @@ func TestEvaluateIPOnlyTrustedIPUpdatesEstablished(t *testing.T) { conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.20:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.20", conn) if got != "trusted" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "trusted") @@ -523,6 +558,7 @@ func TestEvaluateIPOnlyDeghostDisabledReturnsUnknown(t *testing.T) { conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.21:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: false, Mode: ModeWatch} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.21", conn) if got != "unknown" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") @@ -540,6 +576,7 @@ func TestEvaluateIPOnlyCachedCleanReturnsUnknownNoHTTPCall(t *testing.T) { conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.22:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.22", conn) if got != "unknown" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") @@ -558,6 +595,7 @@ func TestEvaluateIPOnlyCachedBlockedReturnsBlockedAndKills(t *testing.T) { // PID 1 keeps the actual kill attempt safely refused by enforcer guardrails. conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.23:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeEnforce} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.23", conn) if got != "blocked" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "blocked") @@ -573,11 +611,13 @@ func TestEvaluateIPOnlyLiveCheckPrivateReservedCachesClean(t *testing.T) { w.WriteHeader(http.StatusForbidden) })) defer srv.Close() + client := deghost.NewClient(srv.URL, time.Second) hexwallStore, _ := newTestStore(t) conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.24:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.24", conn) if got != "unknown" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") @@ -587,9 +627,11 @@ func TestEvaluateIPOnlyLiveCheckPrivateReservedCachesClean(t *testing.T) { if err != nil { t.Fatalf("GetRecentFraudCheck() error = %v", err) } + if entry == nil { t.Fatal("GetRecentFraudCheck() = nil, want a cached clean entry after private/reserved verdict") } + if entry.ShouldKill { t.Fatal("GetRecentFraudCheck().ShouldKill = true, want false for private/reserved verdict") } @@ -604,11 +646,13 @@ func TestEvaluateIPOnlyLiveCheckShouldKillCachesAndBlocks(t *testing.T) { }) })) defer srv.Close() + client := deghost.NewClient(srv.URL, time.Second) hexwallStore, dbPath := newTestStore(t) conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.25:443"} scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeEnforce} + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.25", conn) if got != "blocked" { t.Fatalf("evaluateIPOnly() = %q, want %q", got, "blocked") @@ -618,6 +662,7 @@ func TestEvaluateIPOnlyLiveCheckShouldKillCachesAndBlocks(t *testing.T) { if err != nil { t.Fatalf("GetRecentFraudCheck() error = %v", err) } + if entry == nil || !entry.ShouldKill { t.Fatalf("GetRecentFraudCheck() = %+v, want cached ShouldKill = true", entry) } @@ -649,6 +694,7 @@ func TestRunScanFallsBackToSomoWhenConnectionListerUnset(t *testing.T) { // Even when a caller forgets to populate the field, Run should still work. scanner := &Scanner{Store: hexwallStore, Mode: ModeWatch} scanner.Run(context.Background()) + if scanner.ConnectionLister == nil { t.Fatal("ConnectionLister fallback is nil after Run()") } diff --git a/internal/pihole/cache.go b/internal/pihole/cache.go index 256d7e4..3140906 100644 --- a/internal/pihole/cache.go +++ b/internal/pihole/cache.go @@ -64,6 +64,7 @@ func (c *IPCache) Refresh(ctx context.Context) { if ctx.Err() != nil { return } + c.mu.Lock() defer c.mu.Unlock() @@ -72,6 +73,7 @@ func (c *IPCache) Refresh(ctx context.Context) { slog.Error("cache refresh: failed to query pi-hole domains", "err", err) return } + if len(domains) == 0 { slog.Debug("cache refreshed", "domains", 0, "ips", 0) return @@ -83,9 +85,11 @@ func (c *IPCache) Refresh(ctx context.Context) { // results channel keeps a worker from ever blocking on a consumer that has not // started yet, which would otherwise pin its slot in sem and stall the spawn loop. var mu sync.Mutex + resolved := make(map[string][]string, len(domains)) var wg sync.WaitGroup + for _, domain := range domains { select { case <-ctx.Done(): @@ -107,6 +111,7 @@ func (c *IPCache) Refresh(ctx context.Context) { mu.Lock() c.ownLookups[d] = c.now() mu.Unlock() + ips, err := c.lookupHost(rctx, d) if err != nil { // DNS failure is normal for blocked domains, expired records, and similar cases. @@ -129,6 +134,7 @@ func (c *IPCache) Refresh(ctx context.Context) { // domains is sorted, so each IP's slice stays in a stable order and its first entry // is the one recorded as the representative domain in allowed_ips. ipDomains := make(map[string][]string, len(resolved)) + seen := make(map[string]struct{}, len(resolved)) for _, domain := range domains { ips, ok := resolved[domain] @@ -141,13 +147,18 @@ func (c *IPCache) Refresh(ctx context.Context) { if _, exists := seen[key]; exists { continue } + seen[key] = struct{}{} + ipDomains[ip] = append(ipDomains[ip], domain) } } - var totalIPs int - var totalPairs int + var ( + totalIPs int + totalPairs int + ) + for ip, ipDomainSet := range ipDomains { if err := c.store.UpsertAllowedIP(ip, ipDomainSet[0]); err != nil { slog.Error("cache refresh: failed to upsert IP", "ip", ip, "domain", ipDomainSet[0], "err", err) @@ -174,6 +185,7 @@ func (c *IPCache) newClientDomains() ([]string, error) { if c.ownLookups == nil { c.ownLookups = make(map[string]time.Time) } + for domain, at := range c.ownLookups { if now.Sub(at) > ownLookupMemory { delete(c.ownLookups, domain) @@ -184,8 +196,10 @@ func (c *IPCache) newClientDomains() ([]string, error) { if err != nil { return nil, err } + if maxID < c.lastID { slog.Info("cache refresh: pi-hole query log was reset; re-reading the last hour") + c.lastID = 0 } @@ -195,14 +209,18 @@ func (c *IPCache) newClientDomains() ([]string, error) { } var domains []string + for _, q := range queries { c.lastID = max(c.lastID, q.ID) if c.isOwnLookup(q) { continue } + domains = append(domains, q.Domain) } + slices.Sort(domains) + return slices.Compact(domains), nil } @@ -214,6 +232,7 @@ func (c *IPCache) isOwnLookup(q AnsweredQuery) bool { if !ok { return false } + return !q.Timestamp.Before(at.Add(-ownLookupSlack)) && !q.Timestamp.After(at.Add(lookupTimeout+ownLookupSlack)) } diff --git a/internal/pihole/cache_test.go b/internal/pihole/cache_test.go index 41b81f5..1791c47 100644 --- a/internal/pihole/cache_test.go +++ b/internal/pihole/cache_test.go @@ -13,10 +13,12 @@ import ( func TestIPCacheRefreshWritesTrustedIPAndDomains(t *testing.T) { dbPath := filepath.Join(t.TempDir(), "hexwall.db") + hexwallStore, err := store.NewStore(dbPath) if err != nil { t.Fatalf("NewStore() error = %v", err) } + defer func() { if err := hexwallStore.Close(); err != nil { t.Fatalf("Close() error = %v", err) @@ -49,6 +51,7 @@ func TestIPCacheRefreshWritesTrustedIPAndDomains(t *testing.T) { if err != nil { t.Fatalf("IsAllowed() error = %v", err) } + if !allowed { t.Fatal("IsAllowed() = false, want true") } @@ -57,6 +60,7 @@ func TestIPCacheRefreshWritesTrustedIPAndDomains(t *testing.T) { if err != nil { t.Fatalf("DomainsForIP() error = %v", err) } + want := []string{"example.com", "example.org"} if !reflect.DeepEqual(domains, want) { t.Fatalf("DomainsForIP() = %v, want %v", domains, want) @@ -77,11 +81,13 @@ func (l *fakeQueryLog) clientAsks(domain string) { func (l *fakeQueryLog) answeredQueriesAfter(afterID, since int64) ([]AnsweredQuery, error) { var out []AnsweredQuery + for _, q := range l.queries { if q.ID > afterID && q.Timestamp.Unix() >= since { out = append(out, q) } } + return out, nil } @@ -89,6 +95,7 @@ func (l *fakeQueryLog) maxQueryID() (int64, error) { if len(l.queries) == 0 { return 0, nil } + return l.queries[len(l.queries)-1].ID, nil } @@ -96,16 +103,20 @@ func (l *fakeQueryLog) maxQueryID() (int64, error) { func (l *fakeQueryLog) lookupThroughPihole(_ context.Context, host string) ([]string, error) { l.lookups = append(l.lookups, host) l.clientAsks(host) + return []string{"192.0.2.10"}, nil } func newFakeCache(t *testing.T, log *fakeQueryLog) *IPCache { t.Helper() + hexwallStore, err := store.NewStore(filepath.Join(t.TempDir(), "hexwall.db")) if err != nil { t.Fatalf("NewStore() error = %v", err) } + t.Cleanup(func() { _ = hexwallStore.Close() }) + return &IPCache{ store: hexwallStore, answeredQueriesAfter: log.answeredQueriesAfter, @@ -122,8 +133,10 @@ func TestIPCacheRefreshIgnoresOwnLookups(t *testing.T) { cache := newFakeCache(t, log) log.clientAsks("example.com") + for range 5 { cache.Refresh(context.Background()) + log.now = log.now.Add(30 * time.Second) } @@ -138,6 +151,7 @@ func TestIPCacheRefreshResolvesRepeatClientQueries(t *testing.T) { log.clientAsks("example.com") cache.Refresh(context.Background()) + log.now = log.now.Add(time.Minute) log.clientAsks("example.com") cache.Refresh(context.Background()) diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index 1ad5b1f..ed0da24 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -58,13 +58,16 @@ func NewChecker(config *Config) (*Checker, error) { if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("pihole-FTL.db not found at %s", config.DBPath) } + return nil, fmt.Errorf("cannot access %s: %w", config.DBPath, err) } + if err := file.Close(); err != nil { return nil, fmt.Errorf("close access check for %s: %w", config.DBPath, err) } dsn := "file:" + (&url.URL{Path: config.DBPath, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) if err != nil { return nil, fmt.Errorf("failed to open pihole-db: %w", err) @@ -105,11 +108,13 @@ func openGravityDB(path string, out **sql.DB) error { if err != nil { return fmt.Errorf("cannot access %s: %w", path, err) } + if err := file.Close(); err != nil { return fmt.Errorf("close access check for %s: %w", path, err) } dsn := "file:" + (&url.URL{Path: path, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) if err != nil { return fmt.Errorf("failed to open gravity-db: %w", err) @@ -121,6 +126,7 @@ func openGravityDB(path string, out **sql.DB) error { } *out = db + return nil } @@ -129,14 +135,20 @@ func (c *Checker) Close() error { if c == nil { return nil } - var errFTL error - var errGravity error + + var ( + errFTL error + errGravity error + ) + if c.db != nil { errFTL = c.db.Close() } + if c.gravityDB != nil { errGravity = c.gravityDB.Close() } + return errors.Join(errFTL, errGravity) } @@ -151,6 +163,7 @@ func (c *Checker) IsDomainKnown(domain string) (bool, error) { cutoff := time.Now().Add(-domainLookback).Unix() var found int + err := c.db.QueryRow(` SELECT 1 FROM queries @@ -162,6 +175,7 @@ func (c *Checker) IsDomainKnown(domain string) (bool, error) { if errors.Is(err, sql.ErrNoRows) { return false, nil } + if err != nil { return false, fmt.Errorf("database query failed: %w", err) } @@ -175,11 +189,13 @@ func (c *Checker) IsAllowedByPolicy(domain string) (bool, error) { if domain == "" { return false, nil } + if c == nil || c.gravityDB == nil { return false, nil } var found int + err := c.gravityDB.QueryRow(` SELECT 1 FROM vw_allowlist WHERE domain = ? COLLATE NOCASE @@ -188,6 +204,7 @@ func (c *Checker) IsAllowedByPolicy(domain string) (bool, error) { if errors.Is(err, sql.ErrNoRows) { return c.matchesRegex(domain, true), nil } + if err != nil { return false, fmt.Errorf("allowlist lookup failed: %w", err) } @@ -201,11 +218,13 @@ func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { if domain == "" { return false, nil } + if c == nil || c.gravityDB == nil { return false, nil } var found int + err := c.gravityDB.QueryRow(` SELECT 1 FROM vw_denylist WHERE domain = ? COLLATE NOCASE @@ -214,6 +233,7 @@ func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { if err == nil { return true, nil } + if !errors.Is(err, sql.ErrNoRows) { return false, fmt.Errorf("denylist lookup failed: %w", err) } @@ -226,6 +246,7 @@ func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { if errors.Is(err, sql.ErrNoRows) { return c.matchesRegex(domain, false), nil } + if err != nil { return false, fmt.Errorf("gravity lookup failed: %w", err) } @@ -245,10 +266,12 @@ func (c *Checker) PolicyCounts() (map[string]int64, error) { for _, view := range views { var count int64 + err := c.gravityDB.QueryRow(fmt.Sprintf(`SELECT COUNT(*) FROM %s`, view)).Scan(&count) if err != nil { return nil, fmt.Errorf("count %s: %w", view, err) } + counts[view] = count } @@ -285,6 +308,7 @@ type AnsweredQuery struct { func (c *Checker) AnsweredQueriesAfter(afterID, since int64) ([]AnsweredQuery, error) { placeholders := strings.TrimSuffix(strings.Repeat("?,", len(answeredStatuses)), ",") args := make([]any, 0, len(answeredStatuses)+2) + args = append(args, afterID, since) for _, status := range answeredStatuses { args = append(args, status) @@ -308,18 +332,23 @@ func (c *Checker) AnsweredQueriesAfter(afterID, since int64) ([]AnsweredQuery, e }() var queries []AnsweredQuery + for rows.Next() { var q AnsweredQuery + var timestamp float64 // FTL v6 stores fractional seconds if err := rows.Scan(&q.ID, &q.Domain, ×tamp); err != nil { return nil, fmt.Errorf("scan failed: %w", err) } + q.Timestamp = time.Unix(0, int64(timestamp*float64(time.Second))) queries = append(queries, q) } + if err := rows.Err(); err != nil { return nil, fmt.Errorf("rows iteration failed: %w", err) } + return queries, nil } @@ -330,6 +359,7 @@ func (c *Checker) MaxQueryID() (int64, error) { if err := c.db.QueryRow(`SELECT MAX(id) FROM queries`).Scan(&maxID); err != nil { return 0, fmt.Errorf("database query failed: %w", err) } + return maxID.Int64, nil } diff --git a/internal/pihole/pihole_test.go b/internal/pihole/pihole_test.go index 8c3600f..43edbb3 100644 --- a/internal/pihole/pihole_test.go +++ b/internal/pihole/pihole_test.go @@ -11,6 +11,7 @@ func TestSQLiteReadOnlyDSNRelativePath(t *testing.T) { t.Parallel() dsn := "file:" + (&url.URL{Path: "./pihole-FTL.db", RawQuery: "mode=ro"}).String() + want := "file:./pihole-FTL.db?mode=ro" if dsn != want { t.Fatalf("dsn = %q, want %q", dsn, want) @@ -21,6 +22,7 @@ func TestSQLiteReadOnlyDSNAbsolutePath(t *testing.T) { t.Parallel() dsn := "file:" + (&url.URL{Path: "/etc/pihole/pihole-FTL.db", RawQuery: "mode=ro"}).String() + want := "file:/etc/pihole/pihole-FTL.db?mode=ro" if dsn != want { t.Fatalf("dsn = %q, want %q", dsn, want) @@ -36,6 +38,7 @@ func TestAnsweredQueriesAfterExcludesBlocked(t *testing.T) { checker := newTestChecker(t, "") now := time.Now().Unix() + rows := []struct { domain string status int @@ -61,6 +64,7 @@ func TestAnsweredQueriesAfterExcludesBlocked(t *testing.T) { if err != nil { t.Fatalf("AnsweredQueriesAfter() error: %v", err) } + var got []string for _, q := range queries { got = append(got, q.Domain) @@ -76,6 +80,7 @@ func TestAnsweredQueriesAfterExcludesBlocked(t *testing.T) { if err != nil { t.Fatalf("MaxQueryID() error: %v", err) } + if rest, err := checker.AnsweredQueriesAfter(maxID, now-60); err != nil || len(rest) != 0 { t.Fatalf("AnsweredQueriesAfter(MaxQueryID()) = %v, %v; want none", rest, err) } diff --git a/internal/pihole/policy_test.go b/internal/pihole/policy_test.go index fb722b9..d7d6886 100644 --- a/internal/pihole/policy_test.go +++ b/internal/pihole/policy_test.go @@ -42,6 +42,7 @@ func openTestGravityDB(t *testing.T) (*sql.DB, string) { ` if _, err := db.Exec(schema); err != nil { _ = db.Close() + t.Fatalf("create schema: %v", err) } @@ -52,6 +53,7 @@ func openTestGravityDB(t *testing.T) (*sql.DB, string) { // type 0 = allowlist, type 1 = denylist, type 2 = regex allowlist, type 3 = regex denylist. func insertDomain(t *testing.T, db *sql.DB, domain string, listType int) { t.Helper() + _, err := db.Exec(`INSERT INTO domainlist (type, domain, enabled) VALUES (?, ?, 1)`, listType, domain) if err != nil { t.Fatalf("insert domain %q type %d: %v", domain, listType, err) @@ -61,6 +63,7 @@ func insertDomain(t *testing.T, db *sql.DB, domain string, listType int) { // insertGravity inserts a domain into the gravity table. func insertGravity(t *testing.T, db *sql.DB, domain string) { t.Helper() + _, err := db.Exec(`INSERT INTO gravity (domain, adlist_id) VALUES (?, 1)`, domain) if err != nil { t.Fatalf("insert gravity domain %q: %v", domain, err) @@ -70,12 +73,15 @@ func insertGravity(t *testing.T, db *sql.DB, domain string) { func newTestChecker(t *testing.T, gravityPath string) *Checker { t.Helper() ftlPath := filepath.Join(t.TempDir(), "pihole-FTL.db") + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) if err != nil { t.Fatalf("open test FTL db: %v", err) } + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY, timestamp INTEGER, type INTEGER, domain TEXT, status INTEGER, ip TEXT)`); err != nil { _ = db.Close() + t.Fatalf("create FTL schema: %v", err) } @@ -84,6 +90,7 @@ func newTestChecker(t *testing.T, gravityPath string) *Checker { gravityDB, err = sql.Open("sqlite", "file:"+(&url.URL{Path: gravityPath, RawQuery: "mode=ro"}).String()) if err != nil { _ = db.Close() + t.Fatalf("open gravity db: %v", err) } } @@ -108,6 +115,7 @@ func TestIsBlockedByPolicy_Denylist(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v", err) } + if !blocked { t.Fatal("IsBlockedByPolicy() = false, want true for denylist domain") } @@ -130,6 +138,7 @@ func TestIsBlockedByPolicy_Gravity(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v", err) } + if !blocked { t.Fatal("IsBlockedByPolicy() = false, want true for gravity domain") } @@ -152,6 +161,7 @@ func TestIsBlockedByPolicy_NoMatch(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v", err) } + if blocked { t.Fatal("IsBlockedByPolicy() = true, want false for unknown domain") } @@ -174,6 +184,7 @@ func TestIsAllowedByPolicy_Allowlist(t *testing.T) { if err != nil { t.Fatalf("IsAllowedByPolicy() error = %v", err) } + if !allowed { t.Fatal("IsAllowedByPolicy() = false, want true for allowlist domain") } @@ -196,6 +207,7 @@ func TestIsAllowedByPolicy_NoMatch(t *testing.T) { if err != nil { t.Fatalf("IsAllowedByPolicy() error = %v", err) } + if allowed { t.Fatal("IsAllowedByPolicy() = true, want false for unknown domain") } @@ -221,6 +233,7 @@ func TestPolicyLookup_CaseNormalization(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v", err) } + if !blocked { t.Fatal("IsBlockedByPolicy() = false, want true for case-insensitive denylist match") } @@ -230,6 +243,7 @@ func TestPolicyLookup_CaseNormalization(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v", err) } + if !blocked { t.Fatal("IsBlockedByPolicy() = false, want true for case-insensitive gravity match") } @@ -239,6 +253,7 @@ func TestPolicyLookup_CaseNormalization(t *testing.T) { if err != nil { t.Fatalf("IsAllowedByPolicy() error = %v", err) } + if !allowed { t.Fatal("IsAllowedByPolicy() = false, want true for case-insensitive allowlist match") } @@ -261,6 +276,7 @@ func TestPolicyLookup_EmptyDomain(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy(\"\") error = %v", err) } + if blocked { t.Fatal("IsBlockedByPolicy(\"\") = true, want false") } @@ -269,6 +285,7 @@ func TestPolicyLookup_EmptyDomain(t *testing.T) { if err != nil { t.Fatalf("IsAllowedByPolicy(\"\") error = %v", err) } + if allowed { t.Fatal("IsAllowedByPolicy(\"\") = true, want false") } @@ -287,6 +304,7 @@ func TestPolicyLookup_NilGravityDB(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v, want nil", err) } + if blocked { t.Fatal("IsBlockedByPolicy() = true, want false for nil gravity DB") } @@ -295,6 +313,7 @@ func TestPolicyLookup_NilGravityDB(t *testing.T) { if err != nil { t.Fatalf("IsAllowedByPolicy() error = %v, want nil", err) } + if allowed { t.Fatal("IsAllowedByPolicy() = true, want false for nil gravity DB") } @@ -325,6 +344,7 @@ func TestPolicyCounts(t *testing.T) { if err != nil { t.Fatalf("PolicyCounts() error = %v", err) } + if counts == nil { t.Fatal("PolicyCounts() = nil, want non-nil map") } @@ -345,12 +365,14 @@ func TestPolicyCounts(t *testing.T) { func TestPolicyCounts_NilGravityDB(t *testing.T) { t.Parallel() + checker := &Checker{gravityDB: nil} counts, err := checker.PolicyCounts() if err != nil { t.Fatalf("PolicyCounts() error = %v, want nil", err) } + if counts != nil { t.Fatalf("PolicyCounts() = %v, want nil for nil gravity DB", counts) } @@ -362,18 +384,23 @@ func TestGravityDBPath_Derived(t *testing.T) { // Create the FTL db in a known directory. dir := t.TempDir() ftlPath := filepath.Join(dir, "pihole-FTL.db") + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) if err != nil { t.Fatalf("open FTL db: %v", err) } + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY)`); err != nil { _ = db.Close() + t.Fatalf("init FTL db: %v", err) } + _ = db.Close() // Create gravity.db next to it. gravPath := filepath.Join(dir, "gravity.db") + gravDB, err := sql.Open("sqlite", "file:"+(&url.URL{Path: gravPath, RawQuery: "mode=rwc"}).String()) if err != nil { t.Fatalf("open gravity db: %v", err) @@ -381,8 +408,10 @@ func TestGravityDBPath_Derived(t *testing.T) { // Create minimal schema so gravity opens cleanly. if _, err := gravDB.Exec(`CREATE TABLE adlist (id INTEGER PRIMARY KEY); CREATE VIEW vw_gravity AS SELECT 1 WHERE 0;`); err != nil { _ = gravDB.Close() + t.Fatalf("create gravity schema: %v", err) } + _ = gravDB.Close() // NewChecker should auto-discover gravity.db next to pihole-FTL.db. @@ -406,14 +435,18 @@ func TestGravityDBPath_Missing(t *testing.T) { dir := t.TempDir() ftlPath := filepath.Join(dir, "pihole-FTL.db") + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) if err != nil { t.Fatalf("open FTL db: %v", err) } + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY)`); err != nil { _ = db.Close() + t.Fatalf("init FTL db: %v", err) } + _ = db.Close() // No gravity.db in dir — NewChecker should succeed with nil gravityDB. @@ -436,6 +469,7 @@ func TestGravityDBPath_Missing(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v", err) } + if blocked { t.Fatal("IsBlockedByPolicy() = true, want false when gravity unavailable") } @@ -479,6 +513,7 @@ func TestIsBlockedByPolicy_DenylistBeatsGravity(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v", err) } + if !blocked { t.Fatal("IsBlockedByPolicy() = false, want true when domain is in both denylist and gravity") } @@ -503,6 +538,7 @@ func TestIsAllowedByPolicy_TakesPrecedenceOverGravity(t *testing.T) { if err != nil { t.Fatalf("IsAllowedByPolicy() error = %v", err) } + if !allowed { t.Fatal("IsAllowedByPolicy() = false, want true for allowlisted domain") } @@ -525,6 +561,7 @@ func TestIsBlockedByPolicy_WhitespaceNormalization(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy() error = %v", err) } + if !blocked { t.Fatal("IsBlockedByPolicy() = false, want true with whitespace around domain") } diff --git a/internal/pihole/regex.go b/internal/pihole/regex.go index c0001f6..bf0ec2e 100644 --- a/internal/pihole/regex.go +++ b/internal/pihole/regex.go @@ -51,6 +51,7 @@ func parseRegexRule(raw string, allow bool) (regexRule, error) { pattern, options, _ := strings.Cut(raw, ";") var rule regexRule + if options != "" { for _, option := range strings.Split(options, ";") { switch { @@ -74,6 +75,7 @@ func parseRegexRule(raw string, allow bool) (regexRule, error) { if err != nil { return regexRule{}, fmt.Errorf("pattern does not compile in Go: %w", err) } + rule.re = re return rule, nil @@ -90,10 +92,13 @@ func compileRegexRules(raws []string, allow bool) []regexRule { } else { slog.Warn("pi-hole regex deny rule skipped", "rule", raw, "err", err) } + continue } + rules = append(rules, rule) } + return rules } @@ -109,6 +114,7 @@ func (c *Checker) RefreshRegexLists() error { if err != nil { return err } + denyRaw, err := c.regexPatterns(regexDenyView) if err != nil { return err @@ -148,13 +154,16 @@ func (c *Checker) regexPatterns(view string) ([]string, error) { }() var patterns []string + for rows.Next() { var pattern string if err := rows.Scan(&pattern); err != nil { return nil, fmt.Errorf("%s scan failed: %w", view, err) } + patterns = append(patterns, pattern) } + if err := rows.Err(); err != nil { return nil, fmt.Errorf("%s iteration failed: %w", view, err) } @@ -173,10 +182,12 @@ func (c *Checker) matchesRegex(domain string, allow bool) bool { if allow { rules = lists.allow } + for _, rule := range rules { if rule.matches(domain) { return true } } + return false } diff --git a/internal/pihole/regex_test.go b/internal/pihole/regex_test.go index e7cf7f1..9bf3f20 100644 --- a/internal/pihole/regex_test.go +++ b/internal/pihole/regex_test.go @@ -7,17 +7,21 @@ import "testing" func newRegexTestChecker(t *testing.T, rules map[string]int) *Checker { t.Helper() + gravDB, gravPath := openTestGravityDB(t) for rule, listType := range rules { insertDomain(t, gravDB, rule, listType) } + _ = gravDB.Close() checker := newTestChecker(t, gravPath) t.Cleanup(func() { _ = checker.Close() }) + if err := checker.RefreshRegexLists(); err != nil { t.Fatalf("RefreshRegexLists() error = %v", err) } + return checker } @@ -51,6 +55,7 @@ func TestIsBlockedByPolicy_RegexDenylist(t *testing.T) { if err != nil { t.Fatalf("IsBlockedByPolicy(%q) error = %v", tt.domain, err) } + if blocked != tt.want { t.Errorf("IsBlockedByPolicy(%q) = %v, want %v", tt.domain, blocked, tt.want) } @@ -78,6 +83,7 @@ func TestIsAllowedByPolicy_RegexAllowlist(t *testing.T) { if err != nil { t.Fatalf("IsAllowedByPolicy(%q) error = %v", tt.domain, err) } + if allowed != tt.want { t.Errorf("IsAllowedByPolicy(%q) = %v, want %v", tt.domain, allowed, tt.want) } @@ -86,13 +92,16 @@ func TestIsAllowedByPolicy_RegexAllowlist(t *testing.T) { func TestRegexRule_Invert(t *testing.T) { t.Parallel() + rule, err := parseRegexRule(`(^|\.)tracker\.example$;invert`, false) if err != nil { t.Fatalf("parseRegexRule() error = %v", err) } + if !rule.matches("good.example") { t.Error("inverted rule should match a domain the pattern does not") } + if rule.matches("c2.tracker.example") { t.Error("inverted rule should not match a domain the pattern does") } @@ -100,9 +109,11 @@ func TestRegexRule_Invert(t *testing.T) { func TestParseRegexRule_UnknownOption(t *testing.T) { t.Parallel() + if _, err := parseRegexRule(`^x\.;future=1`, false); err == nil { t.Error("deny rule with an unknown option should be skipped") } + if _, err := parseRegexRule(`^x\.;future=1`, true); err != nil { t.Errorf("allow rule with an unknown option should still apply, got %v", err) } @@ -115,6 +126,7 @@ func TestRefreshRegexLists_PicksUpNewRules(t *testing.T) { checker := newTestChecker(t, gravPath) t.Cleanup(func() { _ = checker.Close() }) + if err := checker.RefreshRegexLists(); err != nil { t.Fatalf("RefreshRegexLists() error = %v", err) } @@ -124,6 +136,7 @@ func TestRefreshRegexLists_PicksUpNewRules(t *testing.T) { } insertDomain(t, gravDB, `(^|\.)tracker\.example$`, 3) + if err := checker.RefreshRegexLists(); err != nil { t.Fatalf("RefreshRegexLists() error = %v", err) } @@ -135,10 +148,12 @@ func TestRefreshRegexLists_PicksUpNewRules(t *testing.T) { func TestRefreshRegexLists_NilGravityDB(t *testing.T) { t.Parallel() + checker := &Checker{} if err := checker.RefreshRegexLists(); err != nil { t.Fatalf("RefreshRegexLists() error = %v", err) } + if checker.matchesRegex("anything.example", false) { t.Fatal("no rules loaded, nothing should match") } diff --git a/internal/service/service.go b/internal/service/service.go index b2cd561..6f55a76 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -40,14 +40,17 @@ func Install(out io.Writer, opts Options) error { if os.Geteuid() != 0 { return errors.New("writing systemd units needs root (run with sudo)") } + if _, err := exec.LookPath("systemctl"); err != nil { return errors.New("systemctl not found: install-service only supports systemd") } + return install(out, opts, unitDir, vendorUnitDirs, runSystemctl) } func install(out io.Writer, opts Options, dir string, vendorDirs []string, systemctl func(args ...string) error) error { withZeek := false + if opts.Zeekctl != "" { switch owned, err := zeekUnitOwner(dir, vendorDirs); { case err != nil: @@ -57,7 +60,9 @@ func install(out io.Writer, opts Options, dir string, vendorDirs []string, syste if err != nil { return err } + report(out, zeekUnit, dir, changed) + withZeek = true default: _, _ = fmt.Fprintf(out, "✓ %s already exists and isn't hexwall's; left as is\n", zeekUnit) @@ -69,12 +74,14 @@ func install(out io.Writer, opts Options, dir string, vendorDirs []string, syste if err != nil { return err } + report(out, hexwallUnit, dir, changed) steps := [][]string{{"daemon-reload"}} if withZeek { steps = append(steps, []string{"enable", "--now", zeekUnit}) } + steps = append(steps, []string{"enable", hexwallUnit}, []string{"restart", hexwallUnit}) for _, step := range steps { if err := systemctl(step...); err != nil { @@ -88,6 +95,7 @@ hexwall now starts at boot and restarts if it crashes. logs: journalctl -u %[1]s -f remove: sudo systemctl disable --now %[1]s `, hexwallUnit) + return nil } @@ -97,10 +105,12 @@ func Uninstall(out io.Writer) error { if os.Geteuid() != 0 { return errors.New("removing systemd units needs root (run with sudo)") } + if _, err := exec.LookPath("systemctl"); err != nil { _, _ = fmt.Fprintln(out, "✓ no systemd here; no units to remove") return nil } + return uninstall(out, unitDir, runSystemctl) } @@ -109,24 +119,31 @@ func uninstall(out io.Writer, dir string, systemctl func(args ...string) error) // hexwall first: it is ordered after Zeek, so it stops before Zeek does. for _, unit := range []string{hexwallUnit, zeekUnit} { path := filepath.Join(dir, unit) + data, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { continue } + if err != nil { return err } + if !bytes.HasPrefix(data, []byte(marker)) { _, _ = fmt.Fprintf(out, "! %s wasn't written by hexwall; left in place\n", path) continue } + if err := systemctl("disable", "--now", unit); err != nil { return fmt.Errorf("systemctl disable --now %s: %w", unit, err) } + if err := os.Remove(path); err != nil { return err } + _, _ = fmt.Fprintf(out, "✓ stopped and removed %s\n", path) + removed = append(removed, unit) } @@ -134,9 +151,11 @@ func uninstall(out io.Writer, dir string, systemctl func(args ...string) error) _, _ = fmt.Fprintln(out, "✓ no hexwall systemd units installed") return nil } + if err := systemctl("daemon-reload"); err != nil { return fmt.Errorf("systemctl daemon-reload: %w", err) } + return nil } @@ -148,13 +167,16 @@ func zeekUnitOwner(dir string, vendorDirs []string) (bool, error) { return false, nil } } + data, err := os.ReadFile(filepath.Join(dir, zeekUnit)) if errors.Is(err, os.ErrNotExist) { return true, nil } + if err != nil { return false, err } + return bytes.HasPrefix(data, []byte(marker)), nil } @@ -162,9 +184,11 @@ func writeUnit(path, content string) (changed bool, err error) { if current, err := os.ReadFile(path); err == nil && string(current) == content { return false, nil } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { return false, fmt.Errorf("write %s: %w", path, err) } + return true, nil } @@ -179,6 +203,7 @@ func report(out io.Writer, unit, dir string, changed bool) { func runSystemctl(args ...string) error { cmd := exec.Command("systemctl", args...) cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + return cmd.Run() } @@ -189,10 +214,12 @@ func renderHexwallUnit(binary string, args []string, withZeek bool) string { after += " " + zeekUnit wants += " " + zeekUnit } + execStart := make([]string, 0, len(args)+1) for _, arg := range append([]string{binary}, args...) { execStart = append(execStart, quoteExecArg(arg)) } + return marker + ` [Unit] Description=hexwall outbound connection monitor @@ -236,12 +263,15 @@ WantedBy=multi-user.target // treats % as a specifier and expands $ as an environment variable. func quoteExecArg(arg string) string { arg = strings.ReplaceAll(arg, "%", "%%") + arg = strings.ReplaceAll(arg, "$", "$$") if arg != "" && !strings.ContainsAny(arg, " \t\"'\\") { return arg } + arg = strings.ReplaceAll(arg, `\`, `\\`) arg = strings.ReplaceAll(arg, `"`, `\"`) + return `"` + arg + `"` } @@ -255,5 +285,6 @@ func WithSafeMode(args []string) []string { return args } } + return append([]string{"--mode", "watch"}, args...) } diff --git a/internal/service/service_test.go b/internal/service/service_test.go index b3976c7..d951c8a 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -61,6 +61,7 @@ func TestRenderHexwallUnit(t *testing.T) { t.Errorf("unit is missing %q:\n%s", want, unit) } } + if strings.Contains(renderHexwallUnit("/x", nil, false), "zeek") { t.Error("unit without Zeek still references zeek.service") } @@ -88,6 +89,7 @@ func TestInstall_WritesUnitsAndStartsHexwall(t *testing.T) { t.Errorf("%s not written: %v", unit, err) } } + want := []string{"daemon-reload", "enable --now zeek.service", "enable hexwall.service", "restart hexwall.service"} if !slices.Equal(systemctl.calls, want) { t.Fatalf("systemctl calls = %q, want %q", systemctl.calls, want) @@ -97,6 +99,7 @@ func TestInstall_WritesUnitsAndStartsHexwall(t *testing.T) { // A zeek.service someone else wrote (a distro package, or by hand) must survive untouched. func TestInstall_KeepsForeignZeekUnit(t *testing.T) { dir := t.TempDir() + foreign := "[Service]\nExecStart=/usr/bin/zeek -i eth1\n" if err := os.WriteFile(filepath.Join(dir, zeekUnit), []byte(foreign), 0o644); err != nil { t.Fatal(err) @@ -111,6 +114,7 @@ func TestInstall_KeepsForeignZeekUnit(t *testing.T) { if string(got) != foreign { t.Fatalf("foreign zeek.service was overwritten:\n%s", got) } + hexwall, _ := os.ReadFile(filepath.Join(dir, hexwallUnit)) if !strings.Contains(string(hexwall), "After=network-online.target docker.service zeek.service") { t.Fatalf("hexwall.service doesn't order itself after the existing zeek.service:\n%s", hexwall) @@ -119,13 +123,16 @@ func TestInstall_KeepsForeignZeekUnit(t *testing.T) { func TestInstall_WithoutZeek(t *testing.T) { dir := t.TempDir() + systemctl := &fakeSystemctl{} if err := install(&bytes.Buffer{}, Options{Binary: "/x"}, dir, nil, systemctl.run); err != nil { t.Fatalf("install() error = %v", err) } + if _, err := os.Stat(filepath.Join(dir, zeekUnit)); !os.IsNotExist(err) { t.Fatalf("zeek.service written although hexwall doesn't use Zeek (err %v)", err) } + if slices.Contains(systemctl.calls, "enable --now zeek.service") { t.Fatalf("zeek.service enabled although hexwall doesn't use Zeek: %q", systemctl.calls) } @@ -136,6 +143,7 @@ func TestUninstall_RemovesOnlyHexwallsUnits(t *testing.T) { if err := install(&bytes.Buffer{}, Options{Binary: "/x"}, dir, nil, (&fakeSystemctl{}).run); err != nil { t.Fatal(err) } + foreign := "[Service]\nExecStart=/usr/bin/zeek -i eth1\n" if err := os.WriteFile(filepath.Join(dir, zeekUnit), []byte(foreign), 0o644); err != nil { t.Fatal(err) @@ -149,9 +157,11 @@ func TestUninstall_RemovesOnlyHexwallsUnits(t *testing.T) { if _, err := os.Stat(filepath.Join(dir, hexwallUnit)); !os.IsNotExist(err) { t.Fatalf("hexwall.service still there (err %v)", err) } + if got, _ := os.ReadFile(filepath.Join(dir, zeekUnit)); string(got) != foreign { t.Fatalf("foreign zeek.service changed:\n%s", got) } + want := []string{"disable --now hexwall.service", "daemon-reload"} if !slices.Equal(systemctl.calls, want) { t.Fatalf("systemctl calls = %q, want %q", systemctl.calls, want) @@ -163,6 +173,7 @@ func TestUninstall_NothingInstalled(t *testing.T) { if err := uninstall(&bytes.Buffer{}, t.TempDir(), systemctl.run); err != nil { t.Fatalf("uninstall() error = %v", err) } + if len(systemctl.calls) != 0 { t.Fatalf("systemctl called with nothing installed: %q", systemctl.calls) } diff --git a/internal/store/store.go b/internal/store/store.go index eaa0afc..a115409 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -141,6 +141,7 @@ func (o SNIOutcome) Valid() bool { case OutcomePolicyAllowed, OutcomePolicyBlocked, OutcomeIPDomainMatch, OutcomeUnknown: return true } + return false } @@ -151,6 +152,7 @@ func ParseSNIOutcome(s string) (SNIOutcome, bool) { if o.Valid() { return o, true } + return OutcomeUnknown, false } @@ -174,6 +176,7 @@ func (o IPOutcome) Valid() bool { case OutcomeIPTrusted, OutcomeIPReserved, OutcomeIPBlocked, OutcomeIPUnknown: return true } + return false } @@ -184,6 +187,7 @@ func ParseIPOutcome(s string) (IPOutcome, bool) { if o.Valid() { return o, true } + return OutcomeIPUnknown, false } @@ -222,19 +226,24 @@ func migrateSNITable(db *sql.DB) error { }() hasOldSchema := false + for cols.Next() { - var cid int - var name, ctype string - var notNull int - var dfltValue sql.NullString - var pk int + var ( + cid int + name, ctype string + notNull int + dfltValue sql.NullString + pk int + ) if err := cols.Scan(&cid, &name, &ctype, ¬Null, &dfltValue, &pk); err != nil { return fmt.Errorf("scan table info: %w", err) } + if name == "known" { hasOldSchema = true } } + if err := cols.Err(); err != nil { return fmt.Errorf("iterate table info: %w", err) } @@ -244,9 +253,11 @@ func migrateSNITable(db *sql.DB) error { } slog.Warn("dropping old sni_observations table (write-only audit data; no readers)", "reason", "schema migration to state-table format") + if _, err := db.Exec(`DROP TABLE sni_observations`); err != nil { return fmt.Errorf("drop old sni_observations: %w", err) } + return nil } @@ -258,12 +269,15 @@ func migrateKillOutcome(db *sql.DB) error { if err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info('killed_connections') WHERE name = 'outcome'`).Scan(&has); err != nil { return fmt.Errorf("query table info: %w", err) } + if has > 0 { return nil } + if _, err := db.Exec(`ALTER TABLE killed_connections ADD COLUMN outcome TEXT NOT NULL DEFAULT 'unknown'`); err != nil { return fmt.Errorf("add outcome column: %w", err) } + return nil } @@ -318,6 +332,7 @@ func NewStore(dbPath string) (*Store, error) { if err := configureConnection(readOnly, true); err != nil { _ = readOnly.Close() _ = readWrite.Close() + return nil, fmt.Errorf("failed to connect to hexwall db read-only connection: %w", err) } @@ -363,8 +378,10 @@ func (s *Store) Close() error { return nil } - var errReadWrite error - var errReadOnly error + var ( + errReadWrite error + errReadOnly error + ) if s.readWrite != nil { errReadWrite = s.readWrite.Close() @@ -389,7 +406,6 @@ func (s *Store) UpsertAllowedIP(ip, domain string) error { domain = excluded.domain, last_refreshed = excluded.last_refreshed `, ip, domain, now, now) - if err != nil { return fmt.Errorf("upsert allowed ip %s: %w", ip, err) } @@ -411,7 +427,6 @@ func (s *Store) UpsertAllowedIPDomain(ip, domain string) error { ON CONFLICT(ip, domain) DO UPDATE SET last_refreshed = excluded.last_refreshed `, ip, domain, now, now) - if err != nil { return fmt.Errorf("upsert allowed ip domain %s/%s: %w", ip, domain, err) } @@ -439,6 +454,7 @@ func (s *Store) DomainsForIP(ip string) ([]string, error) { }() domains := []string{} + for rows.Next() { var domain string if err := rows.Scan(&domain); err != nil { @@ -461,7 +477,6 @@ func (s *Store) UpdateEstablished(ip string) error { _, err := s.readWrite.Exec(` UPDATE allowed_ips SET last_established = ? WHERE ip = ? `, time.Now().Unix(), ip) - if err != nil { return fmt.Errorf("update established %s: %w", ip, err) } @@ -481,6 +496,7 @@ func (s *Store) IsAllowed(ip string) (bool, error) { establishedCutoff := now.Add(-establishedTrustWindow).Unix() var found int + err := s.readOnly.QueryRow(` SELECT 1 FROM allowed_ips WHERE ip = ? @@ -504,8 +520,11 @@ func (s *Store) IsAllowed(ip string) (bool, error) { func (s *Store) GetRecentFraudCheck(ip string) (*FraudCheckCacheEntry, error) { cutoff := time.Now().Add(-fraudCheckCacheWindow).Unix() - var shouldKill int - var checkedAt int64 + var ( + shouldKill int + checkedAt int64 + ) + err := s.readOnly.QueryRow(` SELECT should_kill, checked_at FROM fraud_checks @@ -542,7 +561,6 @@ func (s *Store) UpsertFraudCheck(ip string, shouldKill bool) error { should_kill = excluded.should_kill, checked_at = excluded.checked_at `, ip, shouldKillInt, time.Now().Unix()) - if err != nil { return fmt.Errorf("upsert fraud check %s: %w", ip, err) } @@ -555,8 +573,11 @@ func (s *Store) GetRecentDomainCheck(domain string) (*DomainCheckCacheEntry, err domain = strings.ToLower(strings.TrimSpace(domain)) cutoff := time.Now().Add(-fraudCheckCacheWindow).Unix() - var shouldBlock int - var checkedAt int64 + var ( + shouldBlock int + checkedAt int64 + ) + err := s.readOnly.QueryRow(` SELECT should_block, checked_at FROM domain_checks @@ -595,7 +616,6 @@ func (s *Store) UpsertDomainCheck(domain string, shouldBlock bool) error { should_block = excluded.should_block, checked_at = excluded.checked_at `, domain, shouldBlockInt, time.Now().Unix()) - if err != nil { return fmt.Errorf("upsert domain check %s: %w", domain, err) } @@ -619,6 +639,7 @@ func (s *Store) LogKill(ip, pid, program string) (int64, error) { if err != nil { return 0, fmt.Errorf("log kill %s: read row id: %w", ip, err) } + return id, nil } @@ -641,6 +662,7 @@ func (s *Store) SetKillOutcome(id int64, outcome KillOutcome) error { if _, err := s.readWrite.Exec(`UPDATE killed_connections SET outcome = ? WHERE id = ?`, outcome, id); err != nil { return fmt.Errorf("set kill outcome %d: %w", id, err) } + return nil } @@ -666,13 +688,15 @@ func (s *Store) LogZeekAlert(srcIP, dstIP, dstPort, sni string, blocked bool, co func (s *Store) RecentZeekAlert(srcIP, sni string) (*ZeekAlertEntry, error) { cutoff := time.Now().Add(-time.Hour).Unix() - var id int64 - var dstIP string - var dstPort string - var blockedInt int - var confidence string - var detectedAt int64 - var actionTaken string + var ( + id int64 + dstIP string + dstPort string + blockedInt int + confidence string + detectedAt int64 + actionTaken string + ) err := s.readOnly.QueryRow(` SELECT id, dst_ip, dst_port, blocked, confidence, detected_at, action_taken @@ -686,6 +710,7 @@ func (s *Store) RecentZeekAlert(srcIP, sni string) (*ZeekAlertEntry, error) { if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { return nil, fmt.Errorf("get recent zeek alert for %s: %w", srcIP, err) } @@ -718,7 +743,6 @@ func (s *Store) LogSNIObservation(ip, domain, localPort string, outcome SNIOutco last_seen = excluded.last_seen, times_seen = sni_observations.times_seen + 1 `, ip, domain, localPort, string(outcome), now, now) - if err != nil { return fmt.Errorf("log sni observation %s: %w", ip, err) } @@ -740,7 +764,6 @@ func (s *Store) LogIPObservation(ip, program string, outcome IPOutcome) error { last_seen = excluded.last_seen, times_seen = ip_observations.times_seen + 1 `, ip, program, string(outcome), now, now) - if err != nil { return fmt.Errorf("log ip observation %s: %w", ip, err) } @@ -769,6 +792,7 @@ func (s *Store) Prune() (*PruneResult, error) { if err != nil { return nil, fmt.Errorf("prune allowed_ip_domains: %w", err) } + result.AllowedIPDomains, _ = r.RowsAffected() // sni_observations: operational state data with a 7-day window. @@ -776,6 +800,7 @@ func (s *Store) Prune() (*PruneResult, error) { if err != nil { return nil, fmt.Errorf("prune sni_observations: %w", err) } + result.SNIObservations, _ = r.RowsAffected() // ip_observations: same 7-day window as sni_observations. @@ -783,6 +808,7 @@ func (s *Store) Prune() (*PruneResult, error) { if err != nil { return nil, fmt.Errorf("prune ip_observations: %w", err) } + result.IPObservations, _ = r.RowsAffected() // killed_connections and zeek_alerts: audit records of rare, potentially forensically @@ -791,12 +817,14 @@ func (s *Store) Prune() (*PruneResult, error) { if err != nil { return nil, fmt.Errorf("prune killed_connections: %w", err) } + result.KilledConnections, _ = r.RowsAffected() r, err = s.readWrite.Exec(`DELETE FROM zeek_alerts WHERE detected_at < ?`, now-int64(pruneKilledAlertsWindow.Seconds())) if err != nil { return nil, fmt.Errorf("prune zeek_alerts: %w", err) } + result.ZeekAlerts, _ = r.RowsAffected() return result, nil diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 36b0b6a..f411dac 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -11,6 +11,7 @@ func TestSQLiteDSNRelativePath(t *testing.T) { t.Parallel() dsn := sqliteDSN("./hexwall.db", "rwc") + want := "file:./hexwall.db?mode=rwc" if dsn != want { t.Fatalf("sqliteDSN() = %q, want %q", dsn, want) @@ -21,6 +22,7 @@ func TestSQLiteDSNAbsolutePath(t *testing.T) { t.Parallel() dsn := sqliteDSN("/tmp/hexwall.db", "ro") + want := "file:/tmp/hexwall.db?mode=ro" if dsn != want { t.Fatalf("sqliteDSN() = %q, want %q", dsn, want) @@ -44,6 +46,7 @@ func TestFraudCheckCacheRoundTrip(t *testing.T) { if err != nil { t.Fatalf("GetRecentFraudCheck() error = %v", err) } + if entry != nil { t.Fatalf("GetRecentFraudCheck() = %#v, want nil before insert", entry) } @@ -56,12 +59,15 @@ func TestFraudCheckCacheRoundTrip(t *testing.T) { if err != nil { t.Fatalf("GetRecentFraudCheck() after insert error = %v", err) } + if entry == nil { t.Fatal("GetRecentFraudCheck() = nil, want cached entry") } + if !entry.ShouldKill { t.Fatal("GetRecentFraudCheck().ShouldKill = false, want true") } + if entry.CheckedAt <= 0 { t.Fatalf("GetRecentFraudCheck().CheckedAt = %d, want positive unix timestamp", entry.CheckedAt) } @@ -92,6 +98,7 @@ func TestFraudCheckCacheExpiresAfterWindow(t *testing.T) { if err != nil { t.Fatalf("GetRecentFraudCheck() error = %v", err) } + if entry != nil { t.Fatalf("GetRecentFraudCheck() = %#v, want nil for stale entry", entry) } @@ -125,6 +132,7 @@ func TestAllowedIPDomainsMultiplePerIP(t *testing.T) { if len(domains) != len(want) { t.Fatalf("DomainsForIP() = %v, want %v", domains, want) } + for i, domain := range want { if domains[i] != domain { t.Fatalf("DomainsForIP()[%d] = %q, want %q", i, domains[i], domain) @@ -153,6 +161,7 @@ func TestDomainsForIPUnknownIP(t *testing.T) { if err != nil { t.Fatalf("DomainsForIP() error = %v", err) } + if len(domains) != 0 { t.Fatalf("DomainsForIP() = %v, want empty for unknown ip", domains) } @@ -187,6 +196,7 @@ func TestDomainsForIPExcludesStaleEntries(t *testing.T) { if err != nil { t.Fatalf("DomainsForIP() error = %v", err) } + if len(domains) != 1 || domains[0] != "fresh.example.com" { t.Fatalf("DomainsForIP() = %v, want only the fresh domain", domains) } @@ -217,9 +227,11 @@ func TestUpsertAllowedIPDomainRefreshesWithoutDuplicating(t *testing.T) { t.Fatalf("UpsertAllowedIPDomain() error = %v", err) } - var rowCount int - var firstSeen int64 - var lastRefreshed int64 + var ( + rowCount int + firstSeen int64 + lastRefreshed int64 + ) if err := hexwallStore.readOnly.QueryRow(` SELECT COUNT(*), MIN(first_seen), MIN(last_refreshed) FROM allowed_ip_domains @@ -231,9 +243,11 @@ func TestUpsertAllowedIPDomainRefreshesWithoutDuplicating(t *testing.T) { if rowCount != 1 { t.Fatalf("row count = %d, want 1 after re-upsert", rowCount) } + if firstSeen != originalSeen { t.Fatalf("first_seen = %d, want preserved value %d", firstSeen, originalSeen) } + if lastRefreshed <= originalSeen { t.Fatalf("last_refreshed = %d, want a value newer than %d", lastRefreshed, originalSeen) } @@ -252,6 +266,7 @@ func TestNewStoreBackfillsAllowedIPDomains(t *testing.T) { if err := hexwallStore.UpsertAllowedIP("203.0.113.60", "example.com"); err != nil { t.Fatalf("UpsertAllowedIP() error = %v", err) } + if err := hexwallStore.Close(); err != nil { t.Fatalf("Close() error = %v", err) } @@ -270,6 +285,7 @@ func TestNewStoreBackfillsAllowedIPDomains(t *testing.T) { if err != nil { t.Fatalf("DomainsForIP() error = %v", err) } + if len(domains) != 1 || domains[0] != "example.com" { t.Fatalf("DomainsForIP() = %v, want [example.com] from backfill", domains) } @@ -292,6 +308,7 @@ func TestDomainCheckCacheRoundTrip(t *testing.T) { if err != nil { t.Fatalf("GetRecentDomainCheck() error = %v", err) } + if entry != nil { t.Fatalf("GetRecentDomainCheck() = %#v, want nil before insert", entry) } @@ -304,12 +321,15 @@ func TestDomainCheckCacheRoundTrip(t *testing.T) { if err != nil { t.Fatalf("GetRecentDomainCheck() after insert error = %v", err) } + if entry == nil { t.Fatal("GetRecentDomainCheck() = nil, want cached entry") } + if !entry.ShouldBlock { t.Fatal("GetRecentDomainCheck().ShouldBlock = false, want true") } + if entry.CheckedAt <= 0 { t.Fatalf("GetRecentDomainCheck().CheckedAt = %d, want positive unix timestamp", entry.CheckedAt) } @@ -336,9 +356,11 @@ func TestDomainCheckCacheNormalizedKey(t *testing.T) { if err != nil { t.Fatalf("GetRecentDomainCheck() error = %v", err) } + if entry == nil { t.Fatal("GetRecentDomainCheck() = nil, normalized key lookup should match") } + if entry.ShouldBlock { t.Fatal("GetRecentDomainCheck().ShouldBlock = true, want false") } @@ -369,6 +391,7 @@ func TestDomainCheckCacheExpiresAfterWindow(t *testing.T) { if err != nil { t.Fatalf("GetRecentDomainCheck() error = %v", err) } + if entry != nil { t.Fatalf("GetRecentDomainCheck() = %#v, want nil for stale entry", entry) } @@ -392,6 +415,7 @@ func TestParseSNIOutcomeValid(t *testing.T) { if !ok { t.Errorf("ParseSNIOutcome(%q) ok = false, want true", tt.input) } + if got != tt.want { t.Errorf("ParseSNIOutcome(%q) = %q, want %q", tt.input, got, tt.want) } @@ -407,6 +431,7 @@ func TestParseSNIOutcomeInvalid(t *testing.T) { if ok { t.Errorf("ParseSNIOutcome(%q) ok = true, want false", input) } + if got != OutcomeUnknown { t.Errorf("ParseSNIOutcome(%q) = %q, want %q", input, got, OutcomeUnknown) } @@ -449,9 +474,12 @@ func TestLogSNIObservationInsertAndUpsert(t *testing.T) { t.Fatalf("LogSNIObservation() error = %v", err) } - var timesSeen int - var firstSeen, lastSeen int64 - var outcome string + var ( + timesSeen int + firstSeen, lastSeen int64 + outcome string + ) + err = hexwallStore.readOnly.QueryRow(` SELECT times_seen, first_seen, last_seen, outcome FROM sni_observations @@ -460,15 +488,19 @@ func TestLogSNIObservationInsertAndUpsert(t *testing.T) { if err != nil { t.Fatalf("query sni_observations error = %v", err) } + if timesSeen != 1 { t.Fatalf("times_seen = %d, want 1 after first insert", timesSeen) } + if firstSeen <= 0 { t.Fatalf("first_seen = %d, want positive unix timestamp", firstSeen) } + if lastSeen != firstSeen { t.Fatalf("last_seen = %d, want equal to first_seen on first insert, got %d", lastSeen, firstSeen) } + if outcome != "policy-allowed" { t.Fatalf("outcome = %q, want %q", outcome, "policy-allowed") } @@ -486,12 +518,15 @@ func TestLogSNIObservationInsertAndUpsert(t *testing.T) { if err != nil { t.Fatalf("query sni_observations after upsert error = %v", err) } + if timesSeen != 2 { t.Fatalf("times_seen = %d, want 2 after upsert", timesSeen) } + if outcome != "ip-domain-match" { t.Fatalf("outcome = %q, want %q after upsert", outcome, "ip-domain-match") } + if lastSeen < firstSeen { t.Fatalf("last_seen = %d should not be older than first_seen = %d", lastSeen, firstSeen) } @@ -509,9 +544,11 @@ func TestLogSNIObservationInsertAndUpsert(t *testing.T) { if err != nil { t.Fatalf("query sni_observations after third insert error = %v", err) } + if timesSeen != 3 { t.Fatalf("times_seen = %d, want 3 after third insert", timesSeen) } + if outcome != "policy-blocked" { t.Fatalf("outcome = %q, want %q after third insert", outcome, "policy-blocked") } @@ -536,6 +573,7 @@ func TestLogSNIObservationNormalizesDomain(t *testing.T) { // Lookup by the normalized form. var outcome string + err = hexwallStore.readOnly.QueryRow(` SELECT outcome FROM sni_observations WHERE ip = ? AND domain = ? AND local_port = ? @@ -543,6 +581,7 @@ func TestLogSNIObservationNormalizesDomain(t *testing.T) { if err != nil { t.Fatalf("query sni_observations for normalized domain error = %v", err) } + if outcome != "policy-allowed" { t.Fatalf("outcome = %q, want %q", outcome, "policy-allowed") } @@ -561,6 +600,7 @@ func TestNewStoreMigratesOldSNITable(t *testing.T) { if _, err := hexwallStore.readWrite.Exec(`DROP TABLE sni_observations`); err != nil { t.Fatalf("drop sni_observations error = %v", err) } + if _, err := hexwallStore.readWrite.Exec(` CREATE TABLE sni_observations ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -577,6 +617,7 @@ func TestNewStoreMigratesOldSNITable(t *testing.T) { `, "203.0.113.20", "old.example.com", "443", 1, time.Now().Unix()); err != nil { t.Fatalf("insert old sni_observations error = %v", err) } + if err := hexwallStore.Close(); err != nil { t.Fatalf("Close() error = %v", err) } @@ -594,10 +635,12 @@ func TestNewStoreMigratesOldSNITable(t *testing.T) { // The old row should be gone. var count int + err = reopened.readOnly.QueryRow(`SELECT COUNT(*) FROM sni_observations`).Scan(&count) if err != nil { t.Fatalf("query sni_observations count error = %v", err) } + if count != 0 { t.Fatalf("row count = %d, want 0 after migration dropped old table", count) } @@ -611,6 +654,7 @@ func TestNewStoreMigratesOldSNITable(t *testing.T) { if err != nil { t.Fatalf("query sni_observations count after insert error = %v", err) } + if count != 1 { t.Fatalf("row count = %d, want 1 after insert with new schema", count) } @@ -712,12 +756,15 @@ func TestPruneRemovesStaleRows(t *testing.T) { if result.AllowedIPDomains != 1 { t.Fatalf("Prune().AllowedIPDomains = %d, want 1", result.AllowedIPDomains) } + if result.SNIObservations != 1 { t.Fatalf("Prune().SNIObservations = %d, want 1", result.SNIObservations) } + if result.KilledConnections != 1 { t.Fatalf("Prune().KilledConnections = %d, want 1", result.KilledConnections) } + if result.ZeekAlerts != 1 { t.Fatalf("Prune().ZeekAlerts = %d, want 1", result.ZeekAlerts) } @@ -727,6 +774,7 @@ func TestPruneRemovesStaleRows(t *testing.T) { if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM allowed_ip_domains`).Scan(&count); err != nil { t.Fatalf("query allowed_ip_domains count error = %v", err) } + if count != 1 { t.Fatalf("allowed_ip_domains count = %d, want 1 fresh row remaining", count) } @@ -734,6 +782,7 @@ func TestPruneRemovesStaleRows(t *testing.T) { if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM sni_observations`).Scan(&count); err != nil { t.Fatalf("query sni_observations count error = %v", err) } + if count != 1 { t.Fatalf("sni_observations count = %d, want 1 fresh row remaining", count) } @@ -741,6 +790,7 @@ func TestPruneRemovesStaleRows(t *testing.T) { if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM killed_connections`).Scan(&count); err != nil { t.Fatalf("query killed_connections count error = %v", err) } + if count != 1 { t.Fatalf("killed_connections count = %d, want 1 fresh row remaining", count) } @@ -748,6 +798,7 @@ func TestPruneRemovesStaleRows(t *testing.T) { if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM zeek_alerts`).Scan(&count); err != nil { t.Fatalf("query zeek_alerts count error = %v", err) } + if count != 1 { t.Fatalf("zeek_alerts count = %d, want 1 fresh row remaining", count) } @@ -775,24 +826,28 @@ func TestPruneNoopWhenNothingExpired(t *testing.T) { `, "203.0.113.40", "fresh.example.com", now, now); err != nil { t.Fatalf("insert fresh allowed_ip_domains error = %v", err) } + if _, err := hexwallStore.readWrite.Exec(` INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) VALUES (?, ?, ?, ?, ?, ?, ?) `, "203.0.113.40", "fresh.example.com", "443", "unknown", now, now, 1); err != nil { t.Fatalf("insert fresh sni_observations error = %v", err) } + if _, err := hexwallStore.readWrite.Exec(` INSERT INTO killed_connections (ip, pid, program, killed_at) VALUES (?, ?, ?, ?) `, "203.0.113.40", "1234", "curl", now); err != nil { t.Fatalf("insert fresh killed_connections error = %v", err) } + if _, err := hexwallStore.readWrite.Exec(` INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `, "10.0.0.1", "203.0.113.40", "443", "fresh.example.com", 0, "low", now, "logged"); err != nil { t.Fatalf("insert fresh zeek_alerts error = %v", err) } + if _, err := hexwallStore.readWrite.Exec(` INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) VALUES (?, ?, ?, ?, ?, ?) @@ -829,6 +884,7 @@ func TestParseIPOutcomeValid(t *testing.T) { if !ok { t.Errorf("ParseIPOutcome(%q) ok = false, want true", tt.input) } + if got != tt.want { t.Errorf("ParseIPOutcome(%q) = %q, want %q", tt.input, got, tt.want) } @@ -844,6 +900,7 @@ func TestParseIPOutcomeInvalid(t *testing.T) { if ok { t.Errorf("ParseIPOutcome(%q) ok = true, want false", input) } + if got != OutcomeIPUnknown { t.Errorf("ParseIPOutcome(%q) = %q, want %q", input, got, OutcomeIPUnknown) } @@ -886,9 +943,12 @@ func TestLogIPObservationInsertAndUpsert(t *testing.T) { t.Fatalf("LogIPObservation() error = %v", err) } - var timesSeen int - var firstSeen, lastSeen int64 - var outcome string + var ( + timesSeen int + firstSeen, lastSeen int64 + outcome string + ) + err = hexwallStore.readOnly.QueryRow(` SELECT times_seen, first_seen, last_seen, outcome FROM ip_observations @@ -897,15 +957,19 @@ func TestLogIPObservationInsertAndUpsert(t *testing.T) { if err != nil { t.Fatalf("query ip_observations error = %v", err) } + if timesSeen != 1 { t.Fatalf("times_seen = %d, want 1 after first insert", timesSeen) } + if firstSeen <= 0 { t.Fatalf("first_seen = %d, want positive unix timestamp", firstSeen) } + if lastSeen != firstSeen { t.Fatalf("last_seen = %d, want equal to first_seen on first insert, got %d", lastSeen, firstSeen) } + if outcome != "ip-trusted" { t.Fatalf("outcome = %q, want %q", outcome, "ip-trusted") } @@ -923,12 +987,15 @@ func TestLogIPObservationInsertAndUpsert(t *testing.T) { if err != nil { t.Fatalf("query ip_observations after upsert error = %v", err) } + if timesSeen != 2 { t.Fatalf("times_seen = %d, want 2 after upsert", timesSeen) } + if outcome != "deghost-blocked" { t.Fatalf("outcome = %q, want %q after upsert", outcome, "deghost-blocked") } + if lastSeen < firstSeen { t.Fatalf("last_seen = %d should not be older than first_seen = %d", lastSeen, firstSeen) } @@ -946,9 +1013,11 @@ func TestLogIPObservationInsertAndUpsert(t *testing.T) { if err != nil { t.Fatalf("query ip_observations after third insert error = %v", err) } + if timesSeen != 3 { t.Fatalf("times_seen = %d, want 3 after third insert", timesSeen) } + if outcome != "private-reserved" { t.Fatalf("outcome = %q, want %q after third insert", outcome, "private-reserved") } @@ -971,15 +1040,18 @@ func TestLogIPObservationDifferentPrograms(t *testing.T) { if err := hexwallStore.LogIPObservation("203.0.113.20", "curl", OutcomeIPTrusted); err != nil { t.Fatalf("LogIPObservation() error = %v", err) } + if err := hexwallStore.LogIPObservation("203.0.113.20", "wget", OutcomeIPBlocked); err != nil { t.Fatalf("LogIPObservation() error = %v", err) } var count int + err = hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, "203.0.113.20").Scan(&count) if err != nil { t.Fatalf("query ip_observations count error = %v", err) } + if count != 2 { t.Fatalf("row count = %d, want 2 for same IP with different programs", count) } @@ -1033,6 +1105,7 @@ func TestPruneRemovesStaleIPObservations(t *testing.T) { if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM ip_observations`).Scan(&count); err != nil { t.Fatalf("query ip_observations count error = %v", err) } + if count != 1 { t.Fatalf("ip_observations count = %d, want 1 fresh row remaining", count) } @@ -1042,28 +1115,33 @@ func TestPruneRemovesStaleIPObservations(t *testing.T) { // rows must say 'unknown': back then a refused kill was recorded like a real one. func TestNewStoreAddsKillOutcomeToOldDatabase(t *testing.T) { dbPath := filepath.Join(t.TempDir(), "hexwall.db") + old, err := sql.Open("sqlite", dbPath) if err != nil { t.Fatal(err) } + if _, err := old.Exec(` CREATE TABLE killed_connections (id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, pid TEXT NOT NULL, program TEXT NOT NULL, killed_at INTEGER NOT NULL); INSERT INTO killed_connections (ip, pid, program, killed_at) VALUES ('203.0.113.1', '1', 'curl', 1); `); err != nil { t.Fatal(err) } + _ = old.Close() s, err := NewStore(dbPath) if err != nil { t.Fatalf("NewStore() on an old database error = %v", err) } + t.Cleanup(func() { _ = s.Close() }) var outcome string if err := s.readWrite.QueryRow(`SELECT outcome FROM killed_connections WHERE ip = '203.0.113.1'`).Scan(&outcome); err != nil { t.Fatalf("read old row: %v", err) } + if outcome != "unknown" { t.Fatalf("old row outcome = %q, want %q", outcome, "unknown") } @@ -1074,25 +1152,30 @@ func TestLogKillThenSetOutcome(t *testing.T) { if err != nil { t.Fatal(err) } + t.Cleanup(func() { _ = s.Close() }) id, err := s.LogKill("203.0.113.2", "4242", "curl") if err != nil || id == 0 { t.Fatalf("LogKill() = %d, %v; want a row id", id, err) } + read := func() string { var outcome string if err := s.readWrite.QueryRow(`SELECT outcome FROM killed_connections WHERE id = ?`, id).Scan(&outcome); err != nil { t.Fatal(err) } + return outcome } if got := read(); got != string(KillAttempting) { t.Fatalf("outcome before the kill = %q, want %q", got, KillAttempting) } + if err := s.SetKillOutcome(id, KillKilled); err != nil { t.Fatal(err) } + if got := read(); got != string(KillKilled) { t.Fatalf("outcome after SetKillOutcome = %q, want %q", got, KillKilled) } diff --git a/internal/zeek/notice.go b/internal/zeek/notice.go index 21e4a94..b04e726 100644 --- a/internal/zeek/notice.go +++ b/internal/zeek/notice.go @@ -66,6 +66,7 @@ func (p *noticeParser) parseLine(line string) (event Event, ok bool, err error) if fields := parseTSVFieldsLine(line); fields != nil { p.tsvFields = fields } + return Event{}, false, nil } @@ -73,6 +74,7 @@ func (p *noticeParser) parseLine(line string) (event Event, ok bool, err error) if err != nil { return Event{}, false, err } + if fields["note"] != bypassNote { return Event{}, false, nil } @@ -107,6 +109,7 @@ func (p *noticeParser) fields(line string) (map[string]string, error) { for name, value := range payload { fields[name] = fmt.Sprint(value) } + return fields, nil } @@ -115,6 +118,7 @@ func (p *noticeParser) fields(line string) (map[string]string, error) { } cols := strings.Split(line, "\t") + fields := make(map[string]string, len(p.tsvFields)) for name, i := range p.tsvFields { // "-" is Zeek's unset-field marker, "(empty)" its empty-set marker. @@ -122,6 +126,7 @@ func (p *noticeParser) fields(line string) (map[string]string, error) { fields[name] = cols[i] } } + return fields, nil } @@ -132,9 +137,11 @@ func parseZeekTime(value string) time.Time { whole, frac := math.Modf(seconds) return time.Unix(int64(whole), int64(frac*1e9)) } + if parsed, err := time.Parse(time.RFC3339Nano, value); err == nil { return parsed } + return time.Now() } @@ -144,6 +151,7 @@ func firstNonEmpty(values ...string) string { return value } } + return "" } @@ -187,6 +195,7 @@ func (t *noticeTailer) read(events chan<- Event) { return } defer func() { _ = file.Close() }() + if t.state != fileOpen { slog.Info("zeek: notice log found, watching for DNS-bypass alerts", "path", t.path) t.state = fileOpen @@ -204,6 +213,7 @@ func (t *noticeTailer) read(events chan<- Event) { t.offset = 0 t.parser = noticeParser{} } + if fi.Size() == t.offset { return } @@ -212,6 +222,7 @@ func (t *noticeTailer) read(events chan<- Event) { slog.Debug("zeek: seek notice log failed", "path", t.path, "err", err) return } + data, err := io.ReadAll(file) if err != nil { slog.Debug("zeek: read notice log failed", "path", t.path, "err", err) @@ -224,6 +235,7 @@ func (t *noticeTailer) read(events chan<- Event) { if end == -1 { return } + t.offset += int64(end + 1) for _, line := range strings.Split(string(data[:end]), "\n") { @@ -237,8 +249,10 @@ func (t *noticeTailer) read(events chan<- Event) { } else { slog.Debug("zeek: notice log line could not be parsed", "path", t.path, "err", err) } + continue } + if !ok || events == nil { continue } @@ -262,8 +276,10 @@ func (t *noticeTailer) reportOpenError(err error) { // Whatever appears at this path next is a new file: read it from the start. t.offset = 0 t.parser = noticeParser{} + return } + if t.state != fileUnreadable { slog.Warn("zeek: cannot open notice log; DNS-bypass alerts are not being read", "path", t.path, "err", err) t.state = fileUnreadable @@ -279,6 +295,7 @@ func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error tailer := ¬iceTailer{path: logPath} tailer.skipExisting() + poller := time.NewTicker(2 * time.Second) defer poller.Stop() diff --git a/internal/zeek/notice_test.go b/internal/zeek/notice_test.go index 01d8aca..e4dd313 100644 --- a/internal/zeek/notice_test.go +++ b/internal/zeek/notice_test.go @@ -19,51 +19,66 @@ import ( func readFixtureLines(t *testing.T, name string) []string { t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", name)) if err != nil { t.Fatalf("read fixture %s: %v", name, err) } + return strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") } func parseAll(t *testing.T, lines []string) []Event { t.Helper() - var parser noticeParser - var events []Event + + var ( + parser noticeParser + events []Event + ) + for _, line := range lines { event, ok, err := parser.parseLine(line) if err != nil { t.Fatalf("parseLine(%q) error = %v", line, err) } + if ok { events = append(events, event) } } + return events } func assertGoogleBypassEvent(t *testing.T, events []Event) { t.Helper() + if len(events) != 1 { t.Fatalf("got %d events, want 1: %+v", len(events), events) } + event := events[0] if event.SNI != "google.de" { t.Errorf("SNI = %q, want %q", event.SNI, "google.de") } + if event.SrcIP != "192.168.4.149" { t.Errorf("SrcIP = %q, want %q", event.SrcIP, "192.168.4.149") } + if event.SrcPort != "55881" { t.Errorf("SrcPort = %q, want %q", event.SrcPort, "55881") } + if event.DstIP != "74.125.239.152" { t.Errorf("DstIP = %q, want %q", event.DstIP, "74.125.239.152") } + if event.DstPort != "443" { t.Errorf("DstPort = %q, want %q", event.DstPort, "443") } + want := time.Unix(1398362902, 726337000) if diff := event.Timestamp.Sub(want); diff < -time.Microsecond || diff > time.Microsecond { t.Errorf("Timestamp = %v, want %v (Zeek's ts, not the parse time)", event.Timestamp, want) @@ -80,6 +95,7 @@ func TestNoticeParser_RealJSON(t *testing.T) { func TestNoticeParser_TSVLineBeforeHeader(t *testing.T) { var dataLine string + for _, line := range readFixtureLines(t, "notice.tsv") { if !strings.HasPrefix(line, "#") { dataLine = line @@ -97,6 +113,7 @@ func TestNoticeParser_IgnoresOtherNoticeTypes(t *testing.T) { line = strings.Replace(line, bypassNote, "SSL::Invalid_Server_Cert", 1) var parser noticeParser + _, ok, err := parser.parseLine(line) if err != nil || ok { t.Fatalf("parseLine() = ok %v, err %v; want a silent skip", ok, err) @@ -129,6 +146,7 @@ func TestNoticeTailer_WaitsForCompleteLine(t *testing.T) { if err != nil { t.Fatal(err) } + cut := strings.Index(string(data), "google.de") + len("goo") path := filepath.Join(t.TempDir(), "notice.log") @@ -140,6 +158,7 @@ func TestNoticeTailer_WaitsForCompleteLine(t *testing.T) { tailer := ¬iceTailer{path: path} tailer.poll(events) + if len(events) != 0 { t.Fatalf("got %d events from a half-written line, want 0", len(events)) } @@ -147,13 +166,16 @@ func TestNoticeTailer_WaitsForCompleteLine(t *testing.T) { if err := os.WriteFile(path, data, 0o600); err != nil { t.Fatal(err) } + tailer.poll(events) close(events) + var got []Event for event := range events { got = append(got, event) } + assertGoogleBypassEvent(t, got) } @@ -162,6 +184,7 @@ func TestNoticeTailer_RelearnsHeaderAfterRotation(t *testing.T) { if err != nil { t.Fatal(err) } + path := filepath.Join(t.TempDir(), "notice.log") if err := os.WriteFile(path, data, 0o600); err != nil { t.Fatal(err) @@ -173,8 +196,11 @@ func TestNoticeTailer_RelearnsHeaderAfterRotation(t *testing.T) { <-events // Rotation: a fresh, smaller file with its own header. - var header strings.Builder - var dataLine string + var ( + header strings.Builder + dataLine string + ) + for _, line := range strings.Split(string(data), "\n") { if strings.HasPrefix(line, "#fields") || strings.HasPrefix(line, "#separator") { header.WriteString(line + "\n") @@ -182,9 +208,11 @@ func TestNoticeTailer_RelearnsHeaderAfterRotation(t *testing.T) { dataLine = line } } + if err := os.WriteFile(path, []byte(header.String()+dataLine+"\n"), 0o600); err != nil { t.Fatal(err) } + tailer.poll(events) if len(events) != 1 { @@ -200,6 +228,7 @@ func TestNoticeTailer_SkipsExistingAlerts(t *testing.T) { if err != nil { t.Fatal(err) } + path := filepath.Join(t.TempDir(), "notice.log") if err := os.WriteFile(path, data, 0o600); err != nil { t.Fatal(err) @@ -209,26 +238,32 @@ func TestNoticeTailer_SkipsExistingAlerts(t *testing.T) { tailer := ¬iceTailer{path: path} tailer.skipExisting() tailer.poll(events) + if len(events) != 0 { t.Fatalf("got %d replayed events, want 0", len(events)) } var dataLine string + for _, line := range strings.Split(string(data), "\n") { if line != "" && !strings.HasPrefix(line, "#") { dataLine = line } } + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) if err != nil { t.Fatal(err) } + if _, err := file.WriteString(dataLine + "\n"); err != nil { t.Fatal(err) } + _ = file.Close() tailer.poll(events) + if len(events) != 1 { t.Fatalf("got %d events for a new alert, want 1", len(events)) } @@ -245,7 +280,9 @@ func TestDNSBypassScriptEmbedded(t *testing.T) { // Not parallel: it swaps the default logger. func TestNoticeTailer_LogsFileStateOnce(t *testing.T) { var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) t.Cleanup(func() { slog.SetDefault(previous) }) @@ -254,9 +291,11 @@ func TestNoticeTailer_LogsFileStateOnce(t *testing.T) { events := make(chan Event, 10) tailer.skipExisting() + for range 5 { tailer.poll(events) } + if got := strings.Count(logs.String(), "waiting for notice log"); got != 1 { t.Fatalf("logged 'waiting' %d times over 6 polls, want 1:\n%s", got, logs.String()) } @@ -265,12 +304,15 @@ func TestNoticeTailer_LogsFileStateOnce(t *testing.T) { if err != nil { t.Fatal(err) } + if err := os.WriteFile(path, data, 0o600); err != nil { t.Fatal(err) } + for range 3 { tailer.poll(events) } + if got := strings.Count(logs.String(), "notice log found"); got != 1 { t.Fatalf("logged 'found' %d times, want 1:\n%s", got, logs.String()) } diff --git a/internal/zeek/setup.go b/internal/zeek/setup.go index c7bbd63..0f573de 100644 --- a/internal/zeek/setup.go +++ b/internal/zeek/setup.go @@ -32,10 +32,12 @@ func Setup(out io.Writer) error { if err != nil { return err } + siteDir, err := zeekConfigValue(zeekConfig, "--site_dir") if err != nil { return err } + prefix, err := zeekConfigValue(zeekConfig, "--prefix") if err != nil { return err @@ -45,6 +47,7 @@ func Setup(out io.Writer) error { if err != nil { return err } + if changed { _, _ = fmt.Fprintf(out, "✓ installed %s\n", filepath.Join(siteDir, scriptName)) } else { @@ -52,10 +55,12 @@ func Setup(out io.Writer) error { } localZeek := filepath.Join(siteDir, "local.zeek") + added, err := ensureLoaded(localZeek) if err != nil { return err } + if added { _, _ = fmt.Fprintf(out, "✓ added %q to %s\n", loadLine, localZeek) } else { @@ -63,10 +68,12 @@ func Setup(out io.Writer) error { } nodeCfg := filepath.Join(prefix, "etc", "node.cfg") + missing, err := missingInterfaces(nodeCfg, interfaceExists) if err != nil { _, _ = fmt.Fprintf(out, "! could not check %s: %v\n", nodeCfg, err) } + for _, name := range missing { _, _ = fmt.Fprintf(out, "✗ %s tells Zeek to watch %q, which does not exist on this machine (available: %s)\n", nodeCfg, name, strings.Join(interfaceNames(), ", ")) @@ -86,6 +93,7 @@ Next steps: if len(missing) > 0 { return fmt.Errorf("fix the interface in %s before deploying: Zeek cannot start on an interface that does not exist", nodeCfg) } + return nil } @@ -98,30 +106,36 @@ func Teardown(out io.Writer) error { _, _ = fmt.Fprintln(out, "✓ Zeek not found; nothing to remove there") return nil } + siteDir, err := zeekConfigValue(zeekConfig, "--site_dir") if err != nil { return err } localZeek := filepath.Join(siteDir, "local.zeek") + removed, err := removeLoad(localZeek) if err != nil { return err } + if removed { _, _ = fmt.Fprintf(out, "✓ removed %q from %s\n", loadLine, localZeek) } script := filepath.Join(siteDir, scriptName) + deleted, err := removeScript(script) if err != nil { return err } + if deleted { _, _ = fmt.Fprintf(out, "✓ deleted %s\n", script) } else if _, statErr := os.Stat(script); statErr == nil { _, _ = fmt.Fprintf(out, "! %s wasn't written by hexwall; left in place\n", script) } + return nil } @@ -132,11 +146,13 @@ func removeLoad(localZeek string) (removed bool, err error) { if errors.Is(err, fs.ErrNotExist) { return false, nil } + if err != nil { return false, permissionHint(fmt.Errorf("read %s: %w", localZeek, err)) } lines := strings.Split(string(data), "\n") + kept := make([]string, 0, len(lines)) for i, line := range lines { switch trimmed := strings.TrimSpace(line); { @@ -148,12 +164,15 @@ func removeLoad(localZeek string) (removed bool, err error) { kept = append(kept, line) } } + if !removed { return false, nil } + if err := os.WriteFile(localZeek, []byte(strings.Join(kept, "\n")), 0o644); err != nil { return false, permissionHint(fmt.Errorf("write %s: %w", localZeek, err)) } + return true, nil } @@ -163,15 +182,19 @@ func removeScript(path string) (deleted bool, err error) { if errors.Is(err, fs.ErrNotExist) { return false, nil } + if err != nil { return false, permissionHint(fmt.Errorf("read %s: %w", path, err)) } + if !bytes.Contains(data, []byte(scriptSignature)) { return false, nil } + if err := os.Remove(path); err != nil { return false, permissionHint(fmt.Errorf("delete %s: %w", path, err)) } + return true, nil } @@ -181,10 +204,12 @@ func Zeekctl() (string, error) { if err != nil { return "", err } + prefix, err := zeekConfigValue(zeekConfig, "--prefix") if err != nil { return "", err } + return filepath.Join(prefix, "bin", "zeekctl"), nil } @@ -192,9 +217,11 @@ func findZeekConfig() (string, error) { if path, err := exec.LookPath("zeek-config"); err == nil { return path, nil } + if _, err := os.Stat(defaultZeekConfig); err == nil { return defaultZeekConfig, nil } + return "", errors.New("zeek-config not found on PATH or at " + defaultZeekConfig + "; is Zeek installed?") } @@ -203,6 +230,7 @@ func zeekConfigValue(zeekConfig, flag string) (string, error) { if err != nil { return "", fmt.Errorf("%s %s: %w", zeekConfig, flag, err) } + return strings.TrimSpace(string(out)), nil } @@ -212,9 +240,11 @@ func installScript(siteDir string, script []byte) (changed bool, err error) { if current, err := os.ReadFile(path); err == nil && bytes.Equal(current, script) { return false, nil } + if err := os.WriteFile(path, script, 0o644); err != nil { return false, permissionHint(fmt.Errorf("write %s: %w", path, err)) } + return true, nil } @@ -224,6 +254,7 @@ func ensureLoaded(localZeek string) (added bool, err error) { if err != nil { return false, permissionHint(fmt.Errorf("read %s: %w", localZeek, err)) } + for _, line := range strings.Split(string(data), "\n") { if strings.TrimSpace(line) == loadLine { return false, nil @@ -234,14 +265,17 @@ func ensureLoaded(localZeek string) (added bool, err error) { if len(data) > 0 && !bytes.HasSuffix(data, []byte("\n")) { addition = "\n" + addition } + file, err := os.OpenFile(localZeek, os.O_APPEND|os.O_WRONLY, 0) if err != nil { return false, permissionHint(fmt.Errorf("open %s: %w", localZeek, err)) } + if _, err := file.WriteString(addition); err != nil { _ = file.Close() return false, fmt.Errorf("append to %s: %w", localZeek, err) } + return true, file.Close() } @@ -255,21 +289,26 @@ func missingInterfaces(nodeCfg string, exists func(string) bool) ([]string, erro defer func() { _ = file.Close() }() var missing []string + scanner := bufio.NewScanner(file) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) + key, value, ok := strings.Cut(line, "=") if !ok || strings.HasPrefix(line, "#") || strings.TrimSpace(key) != "interface" { continue } + name := strings.TrimSpace(value) if i := strings.LastIndex(name, "::"); i >= 0 { name = name[i+2:] } + if name != "" && !exists(name) { missing = append(missing, name) } } + return missing, scanner.Err() } @@ -283,10 +322,12 @@ func interfaceNames() []string { if err != nil { return nil } + names := make([]string, 0, len(interfaces)) for _, iface := range interfaces { names = append(names, iface.Name) } + return names } @@ -300,6 +341,7 @@ func zeekLogDir(prefix string) string { } } } + return filepath.Join(prefix, "logs") } @@ -307,5 +349,6 @@ func permissionHint(err error) error { if errors.Is(err, fs.ErrPermission) { return fmt.Errorf("%w (run with sudo)", err) } + return err } diff --git a/internal/zeek/setup_test.go b/internal/zeek/setup_test.go index 8b54b51..9e68171 100644 --- a/internal/zeek/setup_test.go +++ b/internal/zeek/setup_test.go @@ -16,6 +16,7 @@ func TestInstallScript_WritesThenSkipsIdentical(t *testing.T) { if err != nil || !changed { t.Fatalf("first installScript() = %v, %v; want true, nil", changed, err) } + got, err := os.ReadFile(filepath.Join(siteDir, scriptName)) if err != nil || string(got) != string(DNSBypassScript) { t.Fatalf("installed script differs from the embedded one (err %v)", err) @@ -42,6 +43,7 @@ func TestInstallScript_ReplacesOutdatedCopy(t *testing.T) { func TestEnsureLoaded_AppendsOnce(t *testing.T) { localZeek := filepath.Join(t.TempDir(), "local.zeek") + original := "@load misc/detect-traceroute\n@load tuning/defaults" // no trailing newline if err := os.WriteFile(localZeek, []byte(original), 0o644); err != nil { t.Fatal(err) @@ -55,10 +57,12 @@ func TestEnsureLoaded_AppendsOnce(t *testing.T) { } data, _ := os.ReadFile(localZeek) + lines := strings.Split(string(data), "\n") if !slices.Contains(lines, "@load tuning/defaults") { t.Fatalf("last existing line was mangled:\n%s", data) } + if strings.Count(string(data), loadLine) != 1 { t.Fatalf("want exactly one %q line:\n%s", loadLine, data) } @@ -83,10 +87,12 @@ interface=af_packet::wlx001ea6c37e44 } exists := func(name string) bool { return name == "wlx001ea6c37e44" } + missing, err := missingInterfaces(nodeCfg, exists) if err != nil { t.Fatalf("missingInterfaces() error = %v", err) } + if !slices.Equal(missing, []string{"eth0"}) { t.Fatalf("missingInterfaces() = %v, want [eth0]", missing) } @@ -101,10 +107,12 @@ func TestZeekLogDir(t *testing.T) { if err := os.MkdirAll(filepath.Join(prefix, "etc"), 0o755); err != nil { t.Fatal(err) } + cfg := "# Log rotation\nLogRotationInterval = 3600\nLogDir = /var/log/zeek\n" if err := os.WriteFile(filepath.Join(prefix, "etc", "zeekctl.cfg"), []byte(cfg), 0o644); err != nil { t.Fatal(err) } + if got := zeekLogDir(prefix); got != "/var/log/zeek" { t.Fatalf("zeekLogDir() = %q, want %q", got, "/var/log/zeek") } @@ -113,6 +121,7 @@ func TestZeekLogDir(t *testing.T) { // Setup then Teardown must give local.zeek back exactly as it was. func TestRemoveLoad_UndoesEnsureLoaded(t *testing.T) { localZeek := filepath.Join(t.TempDir(), "local.zeek") + original := "@load misc/detect-traceroute\n\n@load tuning/defaults\n" if err := os.WriteFile(localZeek, []byte(original), 0o644); err != nil { t.Fatal(err) @@ -121,10 +130,12 @@ func TestRemoveLoad_UndoesEnsureLoaded(t *testing.T) { if _, err := ensureLoaded(localZeek); err != nil { t.Fatal(err) } + removed, err := removeLoad(localZeek) if err != nil || !removed { t.Fatalf("removeLoad() = %v, %v; want true, nil", removed, err) } + got, _ := os.ReadFile(localZeek) if string(got) != original { t.Fatalf("local.zeek after setup+teardown:\n%q\nwant:\n%q", got, original) @@ -142,6 +153,7 @@ func TestRemoveScript_OnlyDeletesHexwallsOwn(t *testing.T) { if err := os.WriteFile(ours, DNSBypassScript, 0o644); err != nil { t.Fatal(err) } + if deleted, err := removeScript(ours); err != nil || !deleted { t.Fatalf("removeScript(hexwall's script) = %v, %v; want true, nil", deleted, err) } @@ -150,9 +162,11 @@ func TestRemoveScript_OnlyDeletesHexwallsOwn(t *testing.T) { if err := os.WriteFile(foreign, []byte("# someone's own dns-bypass\n"), 0o644); err != nil { t.Fatal(err) } + if deleted, err := removeScript(foreign); err != nil || deleted { t.Fatalf("removeScript(foreign) = %v, %v; want false, nil", deleted, err) } + if _, err := os.Stat(foreign); err != nil { t.Fatalf("foreign script was deleted: %v", err) } @@ -167,12 +181,14 @@ func TestRemoveScript_OnlyDeletesHexwallsOwn(t *testing.T) { func fakeZeek(t *testing.T, iface string) (siteDir, prefix string) { t.Helper() prefix = t.TempDir() + siteDir = filepath.Join(prefix, "share", "zeek", "site") for _, dir := range []string{siteDir, filepath.Join(prefix, "etc"), filepath.Join(prefix, "bin")} { if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } } + files := map[string]string{ filepath.Join(siteDir, "local.zeek"): "@load misc/detect-traceroute\n", filepath.Join(prefix, "etc", "node.cfg"): "[zeek]\ntype=standalone\ninterface=" + iface + "\n", @@ -184,16 +200,20 @@ func fakeZeek(t *testing.T, iface string) (siteDir, prefix string) { t.Fatal(err) } } + t.Setenv("PATH", filepath.Join(prefix, "bin")+string(os.PathListSeparator)+os.Getenv("PATH")) + return siteDir, prefix } func anyInterface(t *testing.T) string { t.Helper() + names := interfaceNames() if len(names) == 0 { t.Skip("no network interfaces") } + return names[0] } @@ -207,9 +227,11 @@ func TestSetupThenTeardown(t *testing.T) { if err := Setup(&out); err != nil { t.Fatalf("Setup() error = %v\n%s", err, out.String()) } + if installed, err := os.ReadFile(filepath.Join(siteDir, scriptName)); err != nil || !bytes.Equal(installed, DNSBypassScript) { t.Fatalf("script not installed (err %v)", err) } + for _, want := range []string{ filepath.Join(prefix, "bin", "zeekctl") + " deploy", "--zeek-log " + filepath.Join(prefix, "logs", "current", "ssl.log"), @@ -223,9 +245,11 @@ func TestSetupThenTeardown(t *testing.T) { if err := Teardown(&bytes.Buffer{}); err != nil { t.Fatalf("Teardown() error = %v", err) } + if _, err := os.Stat(filepath.Join(siteDir, scriptName)); !os.IsNotExist(err) { t.Fatalf("script still installed after Teardown (err %v)", err) } + if after, _ := os.ReadFile(filepath.Join(siteDir, "local.zeek")); !bytes.Equal(after, original) { t.Fatalf("local.zeek after Setup+Teardown:\n%q\nwant:\n%q", after, original) } @@ -236,10 +260,12 @@ func TestSetupReportsMissingInterface(t *testing.T) { fakeZeek(t, "hexwall-test-no-such-if0") var out bytes.Buffer + err := Setup(&out) if err == nil { t.Fatalf("Setup() error = nil, want an error for a missing interface\n%s", out.String()) } + if !strings.Contains(out.String(), `"hexwall-test-no-such-if0", which does not exist`) { t.Fatalf("Setup() output doesn't name the missing interface:\n%s", out.String()) } @@ -247,6 +273,7 @@ func TestSetupReportsMissingInterface(t *testing.T) { func TestZeekctlFromZeekConfig(t *testing.T) { _, prefix := fakeZeek(t, anyInterface(t)) + got, err := Zeekctl() if err != nil || got != filepath.Join(prefix, "bin", "zeekctl") { t.Fatalf("Zeekctl() = %q, %v; want %q", got, err, filepath.Join(prefix, "bin", "zeekctl")) diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go index 837ce6d..e5c743e 100644 --- a/internal/zeek/zeek.go +++ b/internal/zeek/zeek.go @@ -128,6 +128,7 @@ func (c *Client) Lookup(localPort, remoteIP, remotePort string) (string, bool) { if c == nil { return "", false } + key := localPort + "|" + remoteIP + "|" + remotePort c.mu.RLock() @@ -154,6 +155,7 @@ func parseTSVFieldsLine(line string) map[string]int { } cols := strings.Split(line, "\t") + fields := make(map[string]int, len(cols)-1) for i, name := range cols[1:] { fields[name] = i @@ -179,6 +181,7 @@ func readTSVHeader(logPath string) map[string]int { if fields := parseTSVFieldsLine(line); fields != nil { return fields } + if !strings.HasPrefix(line, "#") { return nil // reached a data line without finding #fields } @@ -207,6 +210,7 @@ func (c *Client) recordFromLine(line string) (origH, origP, respH, respP, server if fields := parseTSVFieldsLine(line); fields != nil { c.tsvFields = fields } + return "", "", "", "", "", false case strings.HasPrefix(line, "{"): @@ -214,6 +218,7 @@ func (c *Client) recordFromLine(line string) (origH, origP, respH, respP, server if err := json.Unmarshal([]byte(line), &rec); err != nil { return "", "", "", "", "", false } + return rec.OrigH, fmt.Sprint(rec.OrigP), rec.RespH, fmt.Sprint(rec.RespP), rec.ServerName, true default: @@ -227,9 +232,11 @@ func (c *Client) recordFromLine(line string) (origH, origP, respH, respP, server if !ok || i >= len(cols) { return "" } + if cols[i] == "-" { // Zeek's unset-field marker return "" } + return cols[i] } @@ -305,6 +312,7 @@ func (c *Client) processNewLines() { c.offset = fi.Size() buf := append(c.pending, data...) + lastNewline := bytes.LastIndexByte(buf, '\n') if lastNewline == -1 { c.pending = buf diff --git a/internal/zeek/zeek_test.go b/internal/zeek/zeek_test.go index 80d3f7d..98ce85c 100644 --- a/internal/zeek/zeek_test.go +++ b/internal/zeek/zeek_test.go @@ -13,6 +13,7 @@ import ( // at controlled points instead of racing a real ticker. func newTestClient(t *testing.T, path string) *Client { t.Helper() + return &Client{ logPath: path, cache: make(map[string]sniEntry), @@ -21,6 +22,7 @@ func newTestClient(t *testing.T, path string) *Client { func writeFile(t *testing.T, path, content string) { t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatalf("write file: %v", err) } @@ -37,6 +39,7 @@ func TestClientProcessNewLines_BasicParse(t *testing.T) { if !ok { t.Fatal("Lookup() ok = false, want true") } + if domain != "example.com" { t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") } @@ -47,6 +50,7 @@ func TestClientProcessNewLines_BasicParse(t *testing.T) { if !ok { t.Fatal("Lookup() reverse orientation ok = false, want true") } + if domain != "example.com" { t.Fatalf("Lookup() reverse orientation domain = %q, want %q", domain, "example.com") } @@ -81,6 +85,7 @@ func TestClientProcessNewLines_PartialLineHeldAcrossTicks(t *testing.T) { if _, ok := c.Lookup("51820", "203.0.113.20", "443"); ok { t.Fatal("Lookup() found a record from a partial line, want none yet") } + if len(c.pending) == 0 { t.Fatal("pending is empty, want the partial line to be held over") } @@ -93,6 +98,7 @@ func TestClientProcessNewLines_PartialLineHeldAcrossTicks(t *testing.T) { if !ok { t.Fatal("Lookup() ok = false after completing the line, want true") } + if domain != "example.com" { t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") } @@ -154,17 +160,21 @@ func TestParseTSVFieldsLine(t *testing.T) { t.Parallel() lines := strings.Split(strings.TrimRight(realZeekTSVHeader, "\n"), "\n") + var fieldsLine string + for _, l := range lines { if strings.HasPrefix(l, "#fields\t") { fieldsLine = l } } + if fieldsLine == "" { t.Fatal("test fixture has no #fields line") } fields := parseTSVFieldsLine(fieldsLine) + want := map[string]int{ "ts": 0, "uid": 1, "id.orig_h": 2, "id.orig_p": 3, "id.resp_h": 4, "id.resp_p": 5, "server_name": 9, @@ -190,6 +200,7 @@ func TestReadTSVHeader(t *testing.T) { if fields == nil { t.Fatal("readTSVHeader() = nil, want a field map for a real TSV file") } + if fields["server_name"] != 9 { t.Fatalf("readTSVHeader()[server_name] = %d, want 9", fields["server_name"]) } @@ -213,6 +224,7 @@ func TestReadTSVHeaderNilForJSON(t *testing.T) { // column order at all. func newTestClientTSV(t *testing.T, path string) *Client { t.Helper() + return &Client{ logPath: path, cache: make(map[string]sniEntry), @@ -231,6 +243,7 @@ func TestClientProcessNewLines_TSVRealFixture(t *testing.T) { if !ok { t.Fatal("Lookup() ok = false, want true for the real captured TSV line") } + if domain != "example.com" { t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") } @@ -291,6 +304,7 @@ func TestClientProcessNewLines_TSVRotationRelearnsHeader(t *testing.T) { c := newTestClientTSV(t, path) c.processNewLines() + if _, ok := c.Lookup("37526", "172.66.147.243", "443"); !ok { t.Fatal("Lookup() ok = false before rotation, want true") } @@ -299,6 +313,7 @@ func TestClientProcessNewLines_TSVRotationRelearnsHeader(t *testing.T) { // header re-written from scratch (a real Zeek restart does exactly this). newHeader := "#separator \\x09\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tserver_name\n" newLine := "1790163999.0\tCnew\t10.0.0.5\t51820\t203.0.113.30\t443\tpost-rotation.example\n" + writeFile(t, path, "") // straddling tick, matches the JSON rotation test above c.processNewLines() writeFile(t, path, newHeader+newLine) diff --git a/main.go b/main.go index 6c83911..40d4a35 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,7 @@ func parseFlags(args []string) (*runConfig, error) { enableDeghostDomain := flags.Bool("enable-deghost-domain", false, "send unrecognized SNI domains to the third-party reputation API (deghostapi.hexbyte.dev) at rung 6") mode := flags.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") debug := flags.Bool("debug", false, "enable debug-level logging, including verbose per-connection scan logging") + showVersion := flags.Bool("version", false, "print version and exit") if err := flags.Parse(args); err != nil { return nil, err @@ -93,6 +94,7 @@ func parseFlags(args []string) (*runConfig, error) { if selectedMode != monitor.ModeWatch && selectedMode != monitor.ModeEnforce { return nil, fmt.Errorf("invalid --mode value %q: allowed %q or %q", cfg.mode, monitor.ModeWatch, monitor.ModeEnforce) } + cfg.mode = selectedMode if cfg.hexwallDBPath == "" { @@ -110,13 +112,16 @@ func run() int { fmt.Fprintln(os.Stderr, "zeek-setup:", err) return 1 } + return 0 } + if len(os.Args) > 1 && os.Args[1] == "install-service" { if err := installService(os.Args[2:]); err != nil { fmt.Fprintln(os.Stderr, "install-service:", err) return 1 } + return 0 } @@ -125,6 +130,7 @@ func run() int { fmt.Fprintln(os.Stderr, "uninstall:", err) return 1 } + return 0 } @@ -132,6 +138,7 @@ func run() int { if errors.Is(err, flag.ErrHelp) { return 0 } + if err != nil { slog.Error(err.Error()) return 1 @@ -146,6 +153,7 @@ func run() int { if cfg.debug { logLevel = slog.LevelDebug } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))) if cfg.showVersion { @@ -158,6 +166,7 @@ func run() int { slog.Error("windows is not supported by hexwall") return 1 } + slog.Info("detected OS", "os", runtime.GOOS) if fatal, msg := privilegeCheck(cfg.mode, os.Geteuid()); fatal { @@ -176,8 +185,10 @@ func run() int { if err != nil { fmt.Println("No connection-listing tool available:", err) slog.Error("no connection-listing tool available", "err", err) + return 1 } + fmt.Printf("> Using %s for connection listing.\n", toolName) slog.Info("using connection-listing tool", "tool", toolName) @@ -215,6 +226,7 @@ func run() int { "gravity", counts["vw_gravity"], "regex_allow", counts["vw_regex_allowlist"], "regex_deny", counts["vw_regex_denylist"]) + if counts["vw_gravity"] == 0 { slog.Warn("pi-hole gravity list is empty; blocklist may be broken or not yet downloaded") } @@ -224,9 +236,11 @@ func run() int { hexwallStore, err := store.NewStore(cfg.hexwallDBPath) if err != nil { slog.Error("failed to open hexwall database", "path", cfg.hexwallDBPath, "err", err) + if strings.Contains(err.Error(), "readonly database") || errors.Is(err, fs.ErrPermission) { slog.Error("the database file belongs to another user (often root, from an earlier sudo run): run with sudo, or pick another --hexwall-db") } + return 1 } defer func() { @@ -251,17 +265,18 @@ func run() int { // 4b. Start the Zeek ssl.log tailer when a path is provided. var zeekClient *zeek.Client + if cfg.zeekLogPath != "" { zc, err := zeek.NewClient(&zeek.Config{ LogPath: cfg.zeekLogPath, }) - if err != nil { slog.Error("failed to start zeek log watcher", "path", cfg.zeekLogPath, "err", err) return 1 } defer zc.Close() + zeekClient = zc } @@ -297,6 +312,7 @@ func run() int { } var zeekEvents = make(chan zeek.Event, 100) + if cfg.enableZeek && cfg.zeekNoticeLog != "" { go func() { if err := zeek.WatchNoticeLog(ctx, cfg.zeekNoticeLog, zeekEvents); err != nil { @@ -330,6 +346,7 @@ func run() int { return case <-ticker.C: cache.Refresh(ctx) + if err := checker.RefreshRegexLists(); err != nil { slog.Warn("pi-hole regex list refresh failed; keeping previous rules", "err", err) } @@ -363,6 +380,7 @@ func run() int { fmt.Println("Starting network monitor...") fmt.Printf("> Connections will be checked every %s in %s mode.\n", connectionScanInterval, cfg.mode) + if cfg.debug { fmt.Println("> Debug logging is enabled for every scanned connection.") } @@ -387,6 +405,7 @@ func installService(args []string) error { if runtime.GOOS != "linux" { return errors.New("only Linux with systemd is supported") } + args = service.WithSafeMode(args) // Reject bad flags now: in a unit they would crash-loop at every boot instead. cfg, err := parseFlags(args) @@ -398,9 +417,11 @@ func installService(args []string) error { if err != nil { return err } + if binary, err = filepath.EvalSymlinks(binary); err != nil { return err } + if strings.Contains(binary, "go-build") { return errors.New("this binary is temporary (go run); build hexwall and run the built binary instead") } @@ -411,12 +432,15 @@ func installService(args []string) error { return err } } + if err := service.Install(os.Stdout, opts); err != nil { return err } + if cfg.mode == monitor.ModeWatch { fmt.Println("\nRunning in watch mode: nothing is killed. Re-run with --mode enforce once the logs look right.") } + return nil } @@ -431,14 +455,17 @@ func uninstall() error { if err := service.Uninstall(os.Stdout); err != nil { return err } + if err := zeek.Teardown(os.Stdout); err != nil { return err } + fmt.Println(` Kept on purpose: /var/lib/hexwall the service's database, with its kill and alert history (delete it with: sudo rm -r /var/lib/hexwall) this binary delete it yourself if you installed it (e.g. /usr/local/bin/hexwall)`) + return nil } @@ -450,8 +477,10 @@ func privilegeCheck(mode string, euid int) (fatal bool, msg string) { if euid == 0 { return false, "" } + if mode == monitor.ModeEnforce { return true, "enforce mode needs root: without it hexwall can't close connections or stop other users' programs (run with sudo)" } + return false, "not running as root: other users' programs show without a name, and Zeek's logs may be unreadable (run with sudo for the full picture)" } From 29f94e390b280d6e5bf160eec44b7aa43ea44f25 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 18:00:54 +0600 Subject: [PATCH 51/56] feat: warn at startup when Pi-hole uses groups hexwall can't tell which Pi-hole group this machine's queries fall into, so a domain blocked for any group counts as blocked here. With only the Default group that is exact; with more, it can be stricter than Pi-hole's own rules for this machine. Warn once at startup when enabled groups other than Default exist, instead of supporting groups. --- internal/pihole/pihole.go | 17 +++++++++++++++++ internal/pihole/policy_test.go | 35 ++++++++++++++++++++++++++++++++++ main.go | 7 +++++++ 3 files changed, 59 insertions(+) diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index ed0da24..fcd53eb 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -278,6 +278,23 @@ func (c *Checker) PolicyCounts() (map[string]int64, error) { return counts, nil } +// ExtraGroups returns how many enabled Pi-hole groups exist besides Default (id 0). +// hexwall doesn't know which group this machine's queries fall into, so it treats a +// block in any group as a block here; with extra groups, that can be stricter than +// what Pi-hole itself would apply to this machine. Returns 0 if gravity is unavailable. +func (c *Checker) ExtraGroups() (int, error) { + if c == nil || c.gravityDB == nil { + return 0, nil + } + + var count int + if err := c.gravityDB.QueryRow(`SELECT COUNT(*) FROM "group" WHERE id != 0 AND enabled = 1`).Scan(&count); err != nil { + return 0, fmt.Errorf("count groups: %w", err) + } + + return count, nil +} + // answeredStatuses lists the FTL query status codes that count as "Pi-hole answered this // domain". Only these feed the trusted-IP cache. A blocked query is still written to the // query log, and without this filter its domain would be re-resolved by the host's own diff --git a/internal/pihole/policy_test.go b/internal/pihole/policy_test.go index d7d6886..32bf9a8 100644 --- a/internal/pihole/policy_test.go +++ b/internal/pihole/policy_test.go @@ -566,3 +566,38 @@ func TestIsBlockedByPolicy_WhitespaceNormalization(t *testing.T) { t.Fatal("IsBlockedByPolicy() = false, want true with whitespace around domain") } } + +// Only enabled groups other than Default count: those are the setups where a block may +// apply to some devices and not to this machine. +func TestExtraGroups(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + groups string + want int + }{ + {"default only", `(0, 'Default', 1)`, 0}, + {"one extra, one disabled", `(0, 'Default', 1), (1, 'Kids', 1), (2, 'Old', 0)`, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + gravDB, gravPath := openTestGravityDB(t) + if _, err := gravDB.Exec(`CREATE TABLE "group" (id INTEGER PRIMARY KEY, enabled INTEGER, name TEXT); + INSERT INTO "group" (id, name, enabled) VALUES ` + tc.groups); err != nil { + t.Fatal(err) + } + + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + t.Cleanup(func() { _ = checker.Close() }) + + got, err := checker.ExtraGroups() + if err != nil || got != tc.want { + t.Fatalf("ExtraGroups() = %d, %v; want %d, nil", got, err, tc.want) + } + }) + } +} diff --git a/main.go b/main.go index 40d4a35..3926b50 100644 --- a/main.go +++ b/main.go @@ -232,6 +232,13 @@ func run() int { } } + if groups, err := checker.ExtraGroups(); err != nil { + slog.Debug("could not count pi-hole groups", "err", err) + } else if groups > 0 { + slog.Warn("pi-hole uses groups; hexwall treats a block in any group as a block for this machine, which can be stricter than Pi-hole's own rules for it", + "extra_groups", groups) + } + // 4. Open the local hexwall database, creating it if needed. hexwallStore, err := store.NewStore(cfg.hexwallDBPath) if err != nil { From f433d435dbe8bdc63b1afc24afd6e7465991ace2 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 18:06:35 +0600 Subject: [PATCH 52/56] fix(zeek): wait for ssl.log instead of exiting Zeek creates ssl.log at the first TLS connection it sees, so on a quiet server or right after boot it may not exist yet. hexwall exited, and under systemd restarted every 5s, checking nothing in between although the IP-only path needs no Zeek. It now starts, says once that it is waiting, keeps scanning, and reads the file from its first line once it appears. A file that can't be opened for another reason still stops startup. A missing file now also resets the read position, which fixes a rotation bug: a new ssl.log that grew past the old file's offset before the next poll had its first lines skipped. The waiting/found/unreadable messages are shared with notice.log in logfile.go. --- internal/zeek/logfile.go | 66 +++++++++++++++++++++++++++++++++++ internal/zeek/notice.go | 43 ++++------------------- internal/zeek/zeek.go | 64 +++++++++++++++++++++------------- internal/zeek/zeek_test.go | 70 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 61 deletions(-) create mode 100644 internal/zeek/logfile.go diff --git a/internal/zeek/logfile.go b/internal/zeek/logfile.go new file mode 100644 index 0000000..6605cfe --- /dev/null +++ b/internal/zeek/logfile.go @@ -0,0 +1,66 @@ +package zeek + +import ( + "errors" + "io/fs" + "log/slog" +) + +// Zeek creates each log file when it writes the file's first line, so a log can be +// missing for a while: until the first TLS connection for ssl.log, until the first +// alert for notice.log, and briefly after each rotation. The pollers check every few +// seconds; these helpers make each change of state show up in the logs once. + +// fileState is what a poller last found at its log's path. +type fileState int + +const ( + fileUnchecked fileState = iota + fileMissing // not created yet, or between rotations: normal + fileUnreadable // exists but can't be opened, e.g. permissions + fileOpen +) + +// zeekLog names a log for the state messages. +type zeekLog struct { + name string // "ssl.log" + createdBy string // why it may not exist yet + gives string // what hexwall reads from it +} + +var ( + sslLog = zeekLog{name: "ssl.log", createdBy: "Zeek creates it at the first TLS connection it sees", gives: "TLS server names"} + noticeLog = zeekLog{name: "notice log", createdBy: "Zeek creates it at its first alert", gives: "DNS-bypass alerts"} +) + +// reportOpened logs that the file can be read, if that's news. +func (l zeekLog) reportOpened(state *fileState, path string) { + if *state != fileOpen { + slog.Info("zeek: "+l.name+" found, watching for "+l.gives, "path", path) + + *state = fileOpen + } +} + +// reportOpenError logs why the file can't be opened, if that's news, and reports +// whether it is missing. When it is, whatever appears at the path next is a new file, +// which the caller must read from the start. +func (l zeekLog) reportOpenError(state *fileState, path string, err error) (missing bool) { + if errors.Is(err, fs.ErrNotExist) { + if *state != fileMissing { + slog.Info("zeek: waiting for "+l.name+"; "+l.createdBy, "path", path) + + *state = fileMissing + } + + return true + } + + if *state != fileUnreadable { + slog.Warn("zeek: cannot open "+l.name+"; "+l.gives+" are not being read", "path", path, "err", err) + + *state = fileUnreadable + } + + return false +} diff --git a/internal/zeek/notice.go b/internal/zeek/notice.go index b04e726..fc01c2c 100644 --- a/internal/zeek/notice.go +++ b/internal/zeek/notice.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "io" - "io/fs" "log/slog" "math" "os" @@ -164,16 +163,6 @@ type noticeTailer struct { state fileState // what was last reported about the file, so each change is logged once } -// fileState is what the tailer last found at its path. -type fileState int - -const ( - fileUnchecked fileState = iota - fileMissing // Zeek writes notice.log at its first alert, so this is normal for a while - fileUnreadable // exists but can't be opened, e.g. permissions - fileOpen -) - // skipExisting reads the file as it is now without sending anything: the parser still // learns the TSV "#fields" header at the top, but alerts already in the file are not // replayed. In enforce mode an old alert would otherwise trigger a kill on every restart. @@ -191,15 +180,16 @@ func (t *noticeTailer) poll(events chan<- Event) { func (t *noticeTailer) read(events chan<- Event) { file, err := os.Open(t.path) if err != nil { - t.reportOpenError(err) + if noticeLog.reportOpenError(&t.state, t.path, err) { + t.offset = 0 + t.parser = noticeParser{} + } + return } defer func() { _ = file.Close() }() - if t.state != fileOpen { - slog.Info("zeek: notice log found, watching for DNS-bypass alerts", "path", t.path) - t.state = fileOpen - } + noticeLog.reportOpened(&t.state, t.path) fi, err := file.Stat() if err != nil { @@ -265,27 +255,6 @@ func (t *noticeTailer) read(events chan<- Event) { } } -// reportOpenError logs why notice.log can't be read, once per change of reason, so a -// 2-second poll doesn't repeat it. -func (t *noticeTailer) reportOpenError(err error) { - if errors.Is(err, fs.ErrNotExist) { - if t.state != fileMissing { - slog.Info("zeek: waiting for notice log; Zeek creates it at its first alert", "path", t.path) - t.state = fileMissing - } - // Whatever appears at this path next is a new file: read it from the start. - t.offset = 0 - t.parser = noticeParser{} - - return - } - - if t.state != fileUnreadable { - slog.Warn("zeek: cannot open notice log; DNS-bypass alerts are not being read", "path", t.path, "err", err) - t.state = fileUnreadable - } -} - // WatchNoticeLog tails a Zeek notice log and emits parsed DNS-bypass Events. func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error { logPath := strings.TrimSpace(path) diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go index e5c743e..dec3990 100644 --- a/internal/zeek/zeek.go +++ b/internal/zeek/zeek.go @@ -57,7 +57,8 @@ type Client struct { tsvFields map[string]int offset int64 - pending []byte // bytes read past the last newline; held until the rest of the line arrives + pending []byte // bytes read past the last newline; held until the rest of the line arrives + state fileState // what was last reported about the file, so each change is logged once stopCh chan struct{} done chan struct{} } @@ -73,33 +74,39 @@ func NewClient(cfg *Config) (*Client, error) { logPath := strings.TrimSpace(cfg.LogPath) - // Verify the file exists and is readable before starting the tailer. - f, err := os.Open(logPath) - if err != nil { - return nil, fmt.Errorf("failed to open zeek ssl.log %s: %w", logPath, err) + c := &Client{ + logPath: logPath, + cache: make(map[string]sniEntry), + stopCh: make(chan struct{}), + done: make(chan struct{}), } - fi, err := f.Stat() + // A missing ssl.log is normal (Zeek creates it at the first TLS connection) and + // must not stop hexwall: the scan runs by IP meanwhile, and the tailer reads the + // file from its first line once it appears. Any other error, such as no + // permission, won't fix itself, so it stops startup loudly. + f, err := os.Open(logPath) + if err != nil { + if !sslLog.reportOpenError(&c.state, logPath, err) { + return nil, fmt.Errorf("failed to open zeek ssl.log %s: %w", logPath, err) + } + } else { + fi, err := f.Stat() - _ = f.Close() + _ = f.Close() - if err != nil { - return nil, fmt.Errorf("failed to stat zeek ssl.log %s: %w", logPath, err) - } + if err != nil { + return nil, fmt.Errorf("failed to stat zeek ssl.log %s: %w", logPath, err) + } - c := &Client{ - logPath: logPath, - cache: make(map[string]sniEntry), - offset: fi.Size(), - // The tailer starts reading from the current end of the file (above), - // which is always past the "#fields" header Zeek writes once, at the - // very top, when it first creates the log. Without this separate, - // from-the-start read, TSV parsing would never learn the column - // order in the normal case (Zeek already running before hexwall - // starts) and would silently never match anything. - tsvFields: readTSVHeader(logPath), - stopCh: make(chan struct{}), - done: make(chan struct{}), + // The tailer starts reading from the current end of the file, which is always + // past the "#fields" header Zeek writes once, at the very top, when it creates + // the log. Without this separate, from-the-start read, TSV parsing would never + // learn the column order in the normal case (Zeek already running before + // hexwall starts) and would silently never match anything. + c.offset = fi.Size() + c.tsvFields = readTSVHeader(logPath) + sslLog.reportOpened(&c.state, logPath) } go c.tailLoop(context.Background()) @@ -265,10 +272,19 @@ func (c *Client) tailLoop(ctx context.Context) { func (c *Client) processNewLines() { file, err := os.Open(c.logPath) if err != nil { - slog.Debug("zeek: cannot open ssl.log", "path", c.logPath, "err", err) + if sslLog.reportOpenError(&c.state, c.logPath, err) { + // Whatever appears here next is a new file, with its own "#fields" header + // at the top: read it from the start and learn that header again. + c.offset = 0 + c.pending = nil + c.tsvFields = nil + } + return } + sslLog.reportOpened(&c.state, c.logPath) + defer func() { if err := file.Close(); err != nil { slog.Debug("zeek: failed to close ssl.log", "path", c.logPath, "err", err) diff --git a/internal/zeek/zeek_test.go b/internal/zeek/zeek_test.go index 98ce85c..9f293d1 100644 --- a/internal/zeek/zeek_test.go +++ b/internal/zeek/zeek_test.go @@ -1,6 +1,8 @@ package zeek import ( + "bytes" + "log/slog" "os" "path/filepath" "strings" @@ -333,3 +335,71 @@ func TestNilClientLookup(t *testing.T) { t.Fatalf("nil Client.Lookup() = %q, %v; want \"\", false", sni, ok) } } + +// Zeek creates ssl.log at the first TLS connection it sees, so it can be missing when +// hexwall starts (a quiet server, right after boot) and between rotations. hexwall must +// keep running, say "waiting" once, and read the file from its first line when it +// appears. Not parallel: it swaps the default logger. +func TestClientWaitsForMissingSSLLog(t *testing.T) { + var logs bytes.Buffer + + previous := slog.Default() + + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + path := filepath.Join(t.TempDir(), "ssl.log") + + c, err := NewClient(&Config{LogPath: path}) + if err != nil || c == nil { + t.Fatalf("NewClient() with no ssl.log yet = %v, %v; want a client that waits", c, err) + } + + c.Close() // stop the 5 s ticker; the test drives each poll itself + + for range 3 { + c.processNewLines() + } + + if got := strings.Count(logs.String(), "waiting for ssl.log"); got != 1 { + t.Fatalf("logged 'waiting' %d times, want 1:\n%s", got, logs.String()) + } + + // The file appears: read from its first line, header included. + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine) + c.processNewLines() + + if got := strings.Count(logs.String(), "ssl.log found"); got != 1 { + t.Fatalf("logged 'found' %d times, want 1:\n%s", got, logs.String()) + } + + if domain, ok := c.Lookup("37526", "172.66.147.243", "443"); !ok || domain != "example.com" { + t.Fatalf("Lookup() = %q, %v; want example.com from the file that appeared", domain, ok) + } +} + +// A rotation removes ssl.log and Zeek creates a new one later. The new file must be +// read from the start, with its own header, even if it grows past the old offset before +// the next poll. +func TestClientReadsNewFileAfterRotation(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVHeader+strings.Repeat("# padding so the old file is long\n", 20)) + + c := newTestClientTSV(t, path) + c.processNewLines() + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + + c.processNewLines() // sees the gap + + // Longer than the old file: an offset kept from the old file would land past the + // data line and skip it. + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine+strings.Repeat("# more traffic\n", 60)) + c.processNewLines() + + if domain, ok := c.Lookup("37526", "172.66.147.243", "443"); !ok || domain != "example.com" { + t.Fatalf("Lookup() after rotation = %q, %v; want example.com", domain, ok) + } +} From 5b9bb034af4687fe06f43b08f57b0e6ccf73c585 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 18:37:40 +0600 Subject: [PATCH 53/56] fix(deghost): kill on a blacklisted domain only, not on status ShouldBlockDomain also blocked on status "not_allowed". The API's status judges a domain as an email domain: it says "not_allowed" for any domain without an MX record, which covers most API and CDN hosts (api.github.com, cdn.jsdelivr.net), and "allowed" for an ad tracker with MX records. With --enable-deghost-domain in enforce mode, hexwall would have killed GitHub's API. Only blacklisted counts now. Verdicts are cached for 6h, so on a machine that ran the old rule with domain reputation on, old verdicts expire within 6h of upgrading. --- internal/deghost/deghost.go | 12 ++++++++---- internal/deghost/deghost_test.go | 11 +++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/internal/deghost/deghost.go b/internal/deghost/deghost.go index 36d67c2..6801a8c 100644 --- a/internal/deghost/deghost.go +++ b/internal/deghost/deghost.go @@ -188,13 +188,17 @@ func (c *Client) CheckDomain(ctx context.Context, domain string) (*DomainReport, } // ShouldBlockDomain reports whether the domain report matches the current block policy. -// It triggers only on explicit negative signals (status "not_allowed" or blacklisted), -// ignoring softer signals like disposable, spam, or domain age which are too noisy -// to act on alone — consistent with how ShouldKill only fires on explicit threat fields. +// Only an explicit blacklist entry counts, consistent with how ShouldKill only fires on +// explicit threat fields. +// +// Status is not used: it judges a domain as an email domain. "not_allowed" comes back +// for any domain that can't receive mail (no MX record), which covers most API and CDN +// hosts (api.github.com, cdn.jsdelivr.net), while an ad tracker with MX records is +// "allowed". Softer signals (disposable, spam, domain age) are too noisy to act on. func ShouldBlockDomain(report *DomainReport) bool { if report == nil { return false } - return report.Status == "not_allowed" || report.Blacklisted + return report.Blacklisted } diff --git a/internal/deghost/deghost_test.go b/internal/deghost/deghost_test.go index cf71055..241b26b 100644 --- a/internal/deghost/deghost_test.go +++ b/internal/deghost/deghost_test.go @@ -265,12 +265,15 @@ func TestShouldBlockDomainNilReport(t *testing.T) { } } -func TestShouldBlockDomainNotAllowed(t *testing.T) { +// "not_allowed" is the API's verdict on a domain as an email domain. This is its real +// answer for api.github.com (2026-10-12): no MX record, so "not_allowed". Killing on it +// would kill GitHub's API. +func TestShouldBlockDomainNotAllowedDoesNotBlock(t *testing.T) { t.Parallel() - report := &DomainReport{Status: "not_allowed"} - if !ShouldBlockDomain(report) { - t.Error("ShouldBlockDomain(not_allowed) = false, want true") + report := &DomainReport{Status: "not_allowed", HasMX: false, Blacklisted: false} + if ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(not_allowed, no MX) = true, want false: that's an email check, not a threat verdict") } } From cb7952efba888235b653f810fa0780f53624cee5 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 18:37:40 +0600 Subject: [PATCH 54/56] fix: a reputation verdict can't kill when an allow pattern is unreadable A Pi-hole regex allow rule Go can't compile is skipped, so a domain the user allowed with it reads as "not allowed", and a domain-reputation verdict could then kill it. While any allow rule is unreadable, rung 6 reports the verdict instead of killing, the same rule as a rung-3 lookup error. --- internal/monitor/ladder_test.go | 24 ++++++++++++++++++++++++ internal/monitor/monitor.go | 11 +++++++++++ internal/pihole/regex.go | 14 ++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/internal/monitor/ladder_test.go b/internal/monitor/ladder_test.go index 801d79b..900c115 100644 --- a/internal/monitor/ladder_test.go +++ b/internal/monitor/ladder_test.go @@ -203,3 +203,27 @@ func TestRunWithoutSNIUsesIPOnlyPath(t *testing.T) { t.Fatalf("ip_observations rows = %d, want 1: the IP-only path must have run", got) } } + +// Item 8: an allow pattern Go can't compile might have allowed this domain, so a +// reputation verdict alone must not kill it. `(a)\1` (a back-reference) is valid in +// Pi-hole's regex flavour but not in Go's. +func TestRunDeghostBlockNotKilledWhenAllowPatternUnreadable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"status":"allowed","blacklisted":true}`)) + })) + t.Cleanup(srv.Close) + + checker := newTestChecker(t, func(db *sql.DB) { insertDomain(t, db, `(a)\1`, 2) }) + if checker.AllowListComplete() { + t.Fatal("AllowListComplete() = true, want false with an unreadable allow pattern") + } + + hexwallStore, dbPath := newTestStore(t) + scanner := &Scanner{Checker: checker, Store: hexwallStore, DeghostClient: deghost.NewClient(srv.URL, time.Second), DeghostDomain: true} + + ladderScan(t, scanner, "bad.example.com") + + if got := killRows(t, dbPath); got != 0 { + t.Fatalf("kill rows = %d, want 0: the user may have allowed this domain", got) + } +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 0d5fc3d..c7fa8af 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -416,6 +416,17 @@ func (s *Scanner) Run(ctx context.Context) { continue } + // Unsure whether the user allowed this domain (a Pi-hole allow pattern couldn't be + // read): report, don't kill. Same rule as a rung-3 lookup error. + if !s.Checker.AllowListComplete() { + slog.Warn("deghost-blocked domain NOT killed: a Pi-hole allow pattern couldn't be read, so it may be allowed", + "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + + summary.unknown++ + + continue + } + slog.Warn("deghost-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", reason) s.applyBlockedOutcome(ipStr, conn) diff --git a/internal/pihole/regex.go b/internal/pihole/regex.go index bf0ec2e..2f52a6c 100644 --- a/internal/pihole/regex.go +++ b/internal/pihole/regex.go @@ -171,6 +171,20 @@ func (c *Checker) regexPatterns(view string) ([]string, error) { return patterns, nil } +// AllowListComplete reports whether every Pi-hole regex allow rule could be applied. +// When one couldn't (a pattern Go can't compile), IsAllowedByPolicy may answer "not +// allowed" for a domain the user did allow, so checks hexwall adds on top of Pi-hole, +// like reputation, must not kill on their own. +func (c *Checker) AllowListComplete() bool { + if c == nil { + return true + } + + lists := c.regex.Load() + + return lists == nil || len(lists.allow) == len(lists.allowRaw) +} + // matchesRegex reports whether domain matches any loaded allow (or deny) regex rule. func (c *Checker) matchesRegex(domain string, allow bool) bool { lists := c.regex.Load() From 165c4aa029ddd5c853f7b65614f7cded10dbe260 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 18:43:54 +0600 Subject: [PATCH 55/56] fix(zeek): report a read error while looking for the ssl.log header readTSVHeader never checked the scanner's error, so a read failure (or a header line over bufio.Scanner's 64 KiB limit) looked the same as "no header", after which no TSV line can be read, silently. It now warns. --- internal/zeek/zeek.go | 7 +++++++ internal/zeek/zeek_test.go | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go index dec3990..e1f9a0f 100644 --- a/internal/zeek/zeek.go +++ b/internal/zeek/zeek.go @@ -194,6 +194,13 @@ func readTSVHeader(logPath string) map[string]int { } } + // Stopping on an error looks the same as "no header" to the caller, and without a + // header no TSV line can be read: say so instead of failing silently. + if err := scanner.Err(); err != nil { + slog.Warn("zeek: could not read the ssl.log header; TLS server names can't be read until Zeek starts a new file", + "path", logPath, "err", err) + } + return nil } diff --git a/internal/zeek/zeek_test.go b/internal/zeek/zeek_test.go index 9f293d1..2278cd6 100644 --- a/internal/zeek/zeek_test.go +++ b/internal/zeek/zeek_test.go @@ -403,3 +403,26 @@ func TestClientReadsNewFileAfterRotation(t *testing.T) { t.Fatalf("Lookup() after rotation = %q, %v; want example.com", domain, ok) } } + +// A read error while looking for the header must be reported, not look like "no header": +// without a header, hexwall can't read any TSV line. A line longer than bufio.Scanner's +// limit (64 KiB) is an easy way to make the read fail. Not parallel: it swaps the logger. +func TestReadTSVHeaderReportsReadError(t *testing.T) { + var logs bytes.Buffer + + previous := slog.Default() + + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, "#"+strings.Repeat("x", 70*1024)+"\n") + + if fields := readTSVHeader(path); fields != nil { + t.Fatalf("readTSVHeader() = %v, want nil", fields) + } + + if !strings.Contains(logs.String(), "could not read the ssl.log header") { + t.Fatalf("no warning logged for the read error:\n%s", logs.String()) + } +} From 536a6d4d1ca06b1daa25079e67093cd27786dff4 Mon Sep 17 00:00:00 2001 From: Loknath Dhar Date: Mon, 12 Oct 2026 18:43:54 +0600 Subject: [PATCH 56/56] refactor(pihole): remove unused IsDomainKnown --- internal/pihole/pihole.go | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index fcd53eb..40b6e42 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -33,8 +33,6 @@ type Checker struct { regex atomic.Pointer[regexLists] } -const domainLookback = time.Hour - // NewChecker opens a read-only Checker for the configured Pi-hole database. func NewChecker(config *Config) (*Checker, error) { if config == nil { @@ -152,37 +150,6 @@ func (c *Checker) Close() error { return errors.Join(errFTL, errGravity) } -// IsDomainKnown reports whether domain appeared in Pi-hole query history within the last hour. -// Domains missing from that history bypassed Pi-hole DNS resolution and are treated as suspicious. -func (c *Checker) IsDomainKnown(domain string) (bool, error) { - domain = normalizeDomain(domain) - if domain == "" { - return false, nil - } - - cutoff := time.Now().Add(-domainLookback).Unix() - - var found int - - err := c.db.QueryRow(` - SELECT 1 - FROM queries - WHERE timestamp >= ? - AND TRIM(domain) <> '' - AND domain = ? COLLATE NOCASE - LIMIT 1 - `, cutoff, domain).Scan(&found) - if errors.Is(err, sql.ErrNoRows) { - return false, nil - } - - if err != nil { - return false, fmt.Errorf("database query failed: %w", err) - } - - return true, nil -} - // IsAllowedByPolicy reports whether the domain is on Pi-hole's allowlist or matches an allow regex. func (c *Checker) IsAllowedByPolicy(domain string) (bool, error) { domain = normalizeDomain(domain)