diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4aa85b5..f344004 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -3,16 +3,16 @@ name: Go on: push: paths: - - '**/*.go' - - 'go.mod' - - 'go.sum' - - '**/*.yml' + - "**/*.go" + - "go.mod" + - "go.sum" + - "**/*.yml" pull_request: paths: - - '**/*.go' - - 'go.mod' - - 'go.sum' - - '**/*.yml' + - "**/*.go" + - "go.mod" + - "go.sum" + - "**/*.yml" create: permissions: @@ -48,38 +48,26 @@ jobs: - name: Examine source code for MacOS AMD run: GOOS=darwin GOARCH=amd64 go vet -v ./... - - name: Examine source code for Windows AMD - run: GOOS=windows GOARCH=amd64 go vet -v ./... - - name: Examine source code for Linux ARM run: GOOS=linux GOARCH=arm64 go vet -v ./... - name: Examine source code for MacOS ARM run: GOOS=darwin GOARCH=arm64 go vet -v ./... - - name: Examine source code for Windows ARM - run: GOOS=windows GOARCH=arm64 go vet -v ./... - - name: Build for Linux AMD run: GOOS=linux GOARCH=amd64 go build -v ./... - name: Build for MacOS AMD run: GOOS=darwin GOARCH=amd64 go build -v ./... - - name: Build for Windows AMD - run: GOOS=windows GOARCH=amd64 go build -v ./... - - name: Build for Linux ARM run: GOOS=linux GOARCH=arm64 go build -v ./... - name: Build for MacOS ARM run: GOOS=darwin GOARCH=arm64 go build -v ./... - - name: Build for Windows ARM - run: GOOS=windows GOARCH=arm64 go build -v ./... - - name: Test - # https://stackoverflow.com/questions/58139406/only-run-job-on-specific-branch-with-github-actions + # https://stackoverflow.com/questions/58139406/only-run-job-on-specific-branch-with-github-actions if: github.event_name == 'push' && github.ref == 'refs/heads/main' run: go test -race -v -cover -coverpkg=./... -coverprofile=coverage.txt -covermode=atomic ./... diff --git a/.gitignore b/.gitignore index 66a86d9..f46064e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,14 @@ tmp .vscode coverage.txt .build +pihole-guard.db* +inventory.ndjson + +# local design notes and session artifacts — not for publication +CHAT_SUMMARY_*.md +CLAUDE.md +OVERVIEW.md +SETUP.md +SNI_DETECTION_DESIGN.md +zeek-integration-guide.md +IMPLEMENTATION_NOTES.md diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..1d62d49 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,13 @@ +version: "2" +linters: + enable: + - staticcheck + - errcheck + - revive + # Blank lines split each function into "code paragraphs", one step each. + # `golangci-lint run --fix` adds them; CI checks they stay. + - wsl_v5 + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/README.md b/README.md index 2e2c4f5..8aa4ab3 100644 --- a/README.md +++ b/README.md @@ -8,22 +8,22 @@ [![Dependabot Updates][11]][12] [![Go][13]][14] -`hexwall` is an outbound exfiltration brake for servers already protected by Pi-hole. When a supply-chain attack, malware implant, or compromised package is already running inside the machine, it can skip DNS entirely and send stolen data straight to a hard-coded IP. Pi-hole does not see that hop. `somo` can see the live connection. `hexwall` uses that visibility to flag and, in `enforce` mode, kill suspicious direct-IP connections before data leaves the server. +`hexwall` is an outbound exfiltration brake for servers already protected by Pi-hole. When a supply-chain attack, malware implant, or compromised package is already running inside the machine, it can skip DNS entirely and send stolen data straight to a hard-coded IP. Pi-hole does not see that hop. `ss` (falling back to `somo` when unavailable) can see the live connection. `hexwall` uses that visibility to flag and, in `enforce` mode, kill suspicious direct-IP connections before data leaves the server. It monitors active network connections and kills any that connect to IPs that are not trusted by Pi-hole history, the built-in local allowlist, or recent already-established traffic — closing the gap that Pi-hole leaves open for direct-IP connections. -In short: `Pi-hole + somo + hexwall` gives you a practical containment layer for post-compromise outbound traffic, especially the kind of direct-IP exfiltration occasionally used in supply-chain attacks. +In short: `Pi-hole + hexwall` gives you a practical containment layer for post-compromise outbound traffic, especially the kind of direct-IP exfiltration occasionally used in supply-chain attacks. ## At a glance -| Situation | Server without Pi-hole DNS | Server with only Pi-hole | Server with Pi-hole + somo + hexwall | -| --- | --- | --- | --- | -| Malicious domain resolved over DNS | ❌ Exposed | ✅ Can block at DNS layer | ✅ Can block at DNS layer | -| Malware connects to a hard-coded IP directly | ❌ Exposed | ❌ Pi-hole does not see it | ✅ Direct-IP connection is visible and checked | -| Identify which process owns the connection | ❌ No built-in visibility | ❌ No process visibility | ✅ `somo` shows PID/program | -| Stop outbound exfiltration after compromise | ❌ No containment layer | ⚠️ Only if the attacker still uses DNS | ✅ Can kill suspicious established connections | -| Supply-chain malware bypassing DNS | ❌ High risk | ❌ Still exposed | ✅ Stronger containment | -| Overall outbound protection posture | ❌ Weak | ⚠️ Partial | 🛡️ Strongest of the three | +| Situation | Server without Pi-hole DNS | Server with only Pi-hole | Server with Pi-hole + hexwall | +| -------------------------------------------- | -------------------------- | ------------------------------------- | --------------------------------------------- | +| Malicious domain resolved over DNS | ❌ Exposed | ✅ Can block at DNS layer | ✅ Can block at DNS layer | +| Malware connects to a hard-coded IP directly | ❌ Exposed | ❌ Pi-hole does not see it | ✅ Direct-IP connection is visible and checked | +| Identify which process owns the connection | ❌ No built-in visibility | ❌ No process visibility | ✅ `ss`/`somo` shows PID/program | +| Stop outbound exfiltration after compromise | ❌ No containment layer | ⚠️ Only if the attacker still uses DNS | ✅ Can kill suspicious established connections | +| Supply-chain malware bypassing DNS | ❌ High risk | ❌ Still exposed | ✅ Stronger containment | +| Overall outbound protection posture | ❌ Weak | ⚠️ Partial | 🛡️ Strongest of the three | --- @@ -31,17 +31,21 @@ In short: `Pi-hole + somo + hexwall` gives you a practical containment layer for Pi-hole operates at the DNS layer. It can block a domain, but it has no visibility into connections that bypass DNS entirely — processes that dial a hard-coded IP address directly. Malware and compromised packages commonly use this technique to phone home without triggering any DNS-based block. -hexwall works from the inverse assumption: **if an IP is not trusted, the connection is suspicious by default.** Trust is granted when the IP matches the built-in CIDR allowlist, was refreshed from Pi-hole query history within the last hour, or was already observed as an established connection within the last 60 seconds. On startup it immediately refreshes trusted IPs from Pi-hole, then refreshes them every 30 seconds while scanning connections every 10 seconds. Anything that falls outside that trust set is checked against an external fraud API, with results cached locally for 6 hours per IP. Confirmed threats are either logged (watch mode) or killed (enforce mode). +hexwall works from the inverse assumption: **if a connection is not trusted, it is suspicious by default.** Trust is granted when the IP matches the built-in CIDR allowlist, was resolved from a domain a client asked Pi-hole for within the last hour, or was already observed as an established connection within the last 60 seconds. On startup it immediately reads the last hour of Pi-hole history, then picks up new queries every 30 seconds while scanning connections every 10 seconds. + +When Zeek is enabled (`--zeek-log`), hexwall also captures the TLS SNI for each connection and checks that exact domain against Pi-hole's own allow/deny lists before ever falling back to third-party reputation — this is what lets it tell apart a blocked site from a harmless one even when both happen to share the same IP, as is common behind a CDN. Anything that still falls outside every trust check is optionally escalated to an external reputation API, with results cached locally for 6 hours. Confirmed threats are either logged (watch mode) or killed (enforce mode). --- ## Dependencies -| Dependency | Role | Required | -| ----------------------------------------- | --------------------------------------------------- | --------------------------- | -| [`somo`](https://github.com/theopfr/somo) | Lists established TCP/UDP connections; kills by PID | Must be in `$PATH` | -| Pi-hole FTL database | Source of trusted DNS history | Auto-detected or via `--db` | -| `deghostapi.hexbyte.dev` | Fraud/threat classification for unknown IPs | Network access required | +| Dependency | Role | Required | +| ------------------------------------------------------------ | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| [`ss`](https://wiki.linuxfoundation.org/networking/iproute2) | Lists established TCP connections; kills a single connection via `ss -K` | Default — part of iproute2, present on virtually all Linux systems | +| [`somo`](https://github.com/theopfr/somo) | Fallback connection lister when `ss` is unavailable; kills by PID | Fallback — only required if `ss` (iproute2) is unavailable, e.g. on macOS | +| Pi-hole FTL database | Source of trusted DNS history and allow/deny policy lists | Auto-detected or via `--db` | +| `deghostapi.hexbyte.dev` | Fraud/threat classification for unrecognized IPs and domains | Optional — see `--enable-deghost-*` below | +| Zeek | Passive capture of TLS SNI (`ssl.log`) and DNS-bypass notices (`notice.log`) | Optional — see `--zeek-*`/`--enable-zeek` below | **Build dependency:** `modernc.org/sqlite` (pure-Go SQLite, no CGo or system libsqlite3 required). @@ -58,10 +62,18 @@ You can download them from the project's GitHub Releases page: For production Linux servers, download the matching binary from Releases, place it where your deployment expects `./app/hexwall`, and use the included demo Docker Compose file at [`docker-compose.yml`](./docker-compose.yml) to launch it on the server. -For macOS and Windows, prebuilt binaries are not published by CI. Clone the repository and build the binary manually before deployment. +To run hexwall directly on the host instead, so that it starts at boot and restarts if it crashes, use [`hexwall install-service`](#run-as-a-systemd-service). + +hexwall targets Linux, where prebuilt binaries are published by CI. macOS is best-effort: clone the repository and build the binary manually, and install `somo` as a fallback since `ss` (Linux's iproute2) isn't available there. Windows is not supported. + +### Connection backend + +At startup, hexwall picks a connection-listing backend and prints which one it chose. On Linux, `ss` is used if it's on `PATH`, otherwise `somo`; hexwall exits with an install message if neither is found. `ss` is therefore the default on real Linux boxes (iproute2 ships with virtually every distro), and `somo` only comes into play on a stripped-down system without iproute2. On macOS, `ss` is never probed: its single-connection kill relies on a Linux kernel feature (`CONFIG_INET_DIAG_DESTROY`), so hexwall goes straight to `somo` and every kill there is process-level. The kill path works the same whichever lister found the connection (see [When a connection is killed](#when-a-connection-is-killed)). ### Install `somo` +The steps below are only needed on the fallback path described above (macOS, or a Linux box without `ss`). + If `somo` is not already in your `$PATH`, this is the quickest way to install and verify it on Ubuntu or Debian: ```sh @@ -95,63 +107,147 @@ If neither is found, start fails. Use `--db` to specify the path manually. ## CLI flags -| Flag | Default | Description | -| ------------ | ------------------- | ---------------------------------------------------------------- | -| `--db` | _(auto-detected)_ | Path to `pihole-FTL.db` | -|`--hexwall-db`| `./hexwall.db` | Path to the local hexwall database (created on first run) | -| `--mode` | `watch` | `watch` — detect and log only; `enforce` — detect, log, and kill | -| `--debug` | `false` | Enable verbose per-connection scan logging | +| Flag | Default | Description | +| ------------------------- | ----------------------------------- | ------------------------------------------------------------------------------------------------- | +| `--db` | _(auto-detected)_ | Path to `pihole-FTL.db` | +| `--hexwall-db` | `./hexwall.db` | Path to the local hexwall database (created on first run) | +| `--mode` | `watch` | `watch` — detect and log only; `enforce` — detect, log, and kill | +| `--debug` | `false` | Enable verbose per-connection scan logging | +| `--enable-zeek` | `false` | Enable the Zeek `notice.log` DNS-bypass alert path (independent of `--zeek-log`) | +| `--zeek-log` | _(unset — SNI matching skipped)_ | Path to Zeek's `ssl.log`, used to match a connection's TLS SNI against Pi-hole's allow/deny lists | +| `--zeek-notice-log` | `/opt/zeek/logs/current/notice.log` | Path to Zeek's `notice.log`; only read when `--enable-zeek` is set | +| `--enable-deghost-ip` | `false` | Send unrecognized IPs (connections with no SNI) to the third-party reputation API | +| `--enable-deghost-domain` | `false` | Send unrecognized SNI domains to the third-party reputation API | +| `--version` | `false` | Print version and exit | `--mode` accepts `watch` or `enforce` (case-insensitive). Any other value exits with an error. +`--enable-zeek` and `--zeek-log` are independent switches for two unrelated Zeek consumers: `--zeek-log` feeds SNI-based domain matching into the main decision ladder, while `--enable-zeek` (plus `--zeek-notice-log`) turns on a separate, standalone DNS-bypass alert path. Either, both, or neither can be set. + +### Zeek setup + +hexwall reads two Zeek logs, in whichever of Zeek's two output formats Zeek is producing: + +- **TSV** (Zeek's default `LogAscii` output, used by a stock install with no extra config) — the column order is learned from the log file's own `#fields` header rather than assumed, so it keeps working across Zeek versions or configurations that add extra columns. +- **JSON** (`redef LogAscii::use_json = T;`) — also supported, parsed line by line. + +Tested against Zeek 9.0.0. + +`ssl.log` (for `--zeek-log`) is written by every Zeek install; nothing needs configuring. `notice.log` alerts (for `--enable-zeek`) come from a DNS-bypass detection script that ships inside the hexwall binary. Install it with: + +```sh +sudo hexwall zeek-setup +``` + +`zeek-setup` finds Zeek through `zeek-config` (on `PATH`, or `/opt/zeek/bin/zeek-config`), then: + +1. writes `dns-bypass.zeek` into Zeek's site directory, replacing an older copy so the script always matches the hexwall binary that reads its alerts; +2. adds `@load dns-bypass` to `local.zeek` once, leaving the rest of the file alone; +3. checks that every `interface=` in `node.cfg` exists on this machine, and lists the real interfaces if one doesn't (Zeek can't start on a missing interface; the stock `node.cfg` says `eth0`); +4. prints the next steps: `sudo zeekctl deploy` to restart Zeek with the script, and the exact `--zeek-log`/`--zeek-notice-log` flags for the configured log directory. + +It does not edit `node.cfg` or restart Zeek itself. Re-running it is safe. + +Under `zeekctl`, both logs are in `/current/` (`/opt/zeek/logs/current/` by default). For a plain `zeek -i ` invocation, they're written to the directory Zeek was started from. + +Both logs are read from the moment hexwall starts: alerts already in `notice.log` are skipped, so a restart can't act on old ones, and SNI is only known for TLS handshakes made after startup. + +### DNS-bypass alerts + +The script raises a `DNSBypass::PossibleBypass` notice when a TLS connection's SNI names a domain that the client never looked up through DNS, a sign that it bypassed Pi-hole. For each alert hexwall checks the domain against Pi-hole's lists and records it in the `zeek_alerts` table. If the domain is blocked: + +- in `--mode watch`, the alert is logged; +- in `--mode enforce`, hexwall looks for the alerted connection among **this machine's** sockets and kills it through the same guarded path as every other kill (see [When a connection is killed](#when-a-connection-is-killed)). + +Zeek sees the whole network it listens on, but a socket can only be closed on the machine that owns it. An alert about another device is recorded as `not_on_this_host` with a warning; to block that device's connection, run hexwall on it too. + ### Debug logging -When `--debug` is enabled, each scan cycle logs every connection it encounters with its status: +When `--debug` is enabled, hexwall also logs intermediate, non-actionable outcomes it would otherwise stay quiet about — a domain or IP coming back clean from a reputation check, or a check being skipped because its flag is disabled. Warnings, blocks, and kills are always logged regardless of `--debug`. + +Every scan tick prints a one-line summary regardless of `--debug` — connection count, and how many were trusted, blocked, unknown, or errored — and an empty scan (the connection-listing tool returning zero connections) is always logged too. + +### Run as a systemd service + +```sh +sudo hexwall install-service [hexwall flags...] +# e.g. +sudo /usr/local/bin/hexwall install-service --zeek-log /opt/zeek/logs/current/ssl.log --enable-zeek +``` + +This makes hexwall start at boot and restart if it crashes. The flags after `install-service` are the ones the service runs with, and they're checked first, so a typo fails here instead of making the service crash at every boot. Then it: + +1. writes `/etc/systemd/system/hexwall.service`, running this binary (by its absolute path) with those flags from `/var/lib/hexwall`, where the default `./hexwall.db` is created; +2. if the flags use Zeek (`--zeek-log` or `--enable-zeek`), writes a `zeek.service` that runs `zeekctl deploy` at boot and `zeekctl stop` on shutdown, and orders hexwall after it. A `zeek.service` that hexwall didn't write (from a package, or written by hand) is left untouched and used as is; +3. runs `systemctl daemon-reload`, enables both units, and (re)starts hexwall. + +**The service runs in `--mode watch` unless you pass `--mode` yourself.** A service starts before anyone is watching, so killing has to be switched on deliberately: check the logs first, then re-run `install-service` with `--mode enforce`. + +Re-running `install-service` updates the units. Build hexwall before you run it: the binary that `go run` makes is deleted when it exits, so it's refused. + +```sh +systemctl status hexwall # is it running? +journalctl -u hexwall -f # follow its logs +sudo systemctl disable --now hexwall # stop it and remove it from boot +``` + +Linux with systemd only. -- **allowed** — IP is in the local trust cache or allowlist -- **unrecognized-clean** — IP is not in cache, but the fraud API returned a clean verdict (or 403 for private/reserved ranges) -- **vulnerable** — IP is not in cache, fraud API flagged it as abuser/attacker/threat +### Uninstall + +```sh +sudo hexwall uninstall +``` -When debug is disabled (default), only non-allowed results are logged. Additionally, empty scans (somo returning zero connections) are always logged regardless of debug mode. +Undoes `install-service` and `zeek-setup`: it stops, disables and deletes the units hexwall wrote, removes the `@load dns-bypass` lines from `local.zeek`, and deletes the script. Only files carrying hexwall's signature are touched, so a `zeek.service` or `dns-bypass.zeek` someone else wrote is left in place. Changes you made yourself, such as the interface in `node.cfg`, stay. + +Two things are kept on purpose: `/var/lib/hexwall` (the database with the kill and alert history; delete it with `sudo rm -r /var/lib/hexwall`) and the hexwall binary, which you installed. --- ## When a connection is killed -A connection is killed only when all of the following are true: +A connection is only ever killed in `--mode enforce`, for a connection the active connection-listing tool (`ss` or `somo`) reports as currently established. Four independent checks can trigger it: + +1. **Pi-hole domain denylist match** — Zeek captured the connection's TLS SNI (via `--zeek-log`) and that domain is on Pi-hole's own denylist, a regex denylist entry, or the gravity blocklist. Requires only Pi-hole; no third-party service is involved, and this check runs regardless of the `--enable-deghost-*` flags. +2. **Third-party domain reputation match** — an SNI was captured, the domain is on neither Pi-hole list and isn't already known-trusted for this IP, and the reputation API's verdict (fetched live or reused from a 6-hour cache) says to block it. Requires `--enable-deghost-domain` (default `false`). +3. **Third-party IP reputation match** — no SNI was available at all (non-TLS traffic, ECH, or a connection predating startup), the IP is not trusted, and the reputation API's verdict on the bare IP (live or 6-hour cached) marks it `is_abuser`, `is_attacker`, or `is_threat`. Requires `--enable-deghost-ip` (default `false`). + +4. **Zeek DNS-bypass alert for a blocked domain** — Zeek raised a bypass alert (`--enable-zeek`), the domain is blocked by Pi-hole's lists, and the alerted connection is on this machine. See [DNS-bypass alerts](#dns-bypass-alerts). -1. `--mode enforce` is active. -2. `somo` reports the connection as currently established. -3. The remote IP is not trusted. -4. The remote IP's current fraud decision, either fetched live or reused from the local 6-hour cache, marks it as `is_abuser`, `is_attacker`, or `is_threat`. +Both `--enable-deghost-*` flags default to `false`, because they send connection data off the machine. With the defaults, only the Pi-hole checks (1 and 4) can trigger a kill, and both need Zeek. So `--mode enforce` without `--zeek-log` kills nothing unless you opt in to `--enable-deghost-ip`; hexwall warns about this at startup. -An IP is considered trusted when any of these are true: +Pi-hole regex entries are matched the way Pi-hole matches them: case-insensitive, with `;invert` honoured. Options hexwall can't evaluate for a connection (`;querytype=`, `;reply=`, unknown ones) resolve toward not killing: an allow rule with them still applies, and a deny rule with them is skipped. Regex lists are reloaded on the 30-second refresh, so edits in Pi-hole take effect without a restart. -- It matches the built-in CIDR allowlist. -- It was refreshed from Pi-hole history within the last hour. -- It was already seen as an established allowed connection within the last 60 seconds. +A connection is treated as trusted, and skipped entirely, when any of these are true: + +- Its IP matches the built-in CIDR allowlist. +- Its domain is on Pi-hole's allowlist (exact or regex). +- Its IP was resolved from a domain a client asked Pi-hole for within the last hour, or was already seen as an established connection within the last 60 seconds. +- Its domain has already been seen for this exact IP in hexwall's own history. Connections are not killed in these cases: -- `--mode watch` is active. The connection is only logged as `would kill`. -- The IP is on the built-in allowlist. -- The IP is still trusted from a recent Pi-hole refresh or recent established-connection activity. -- The IP already has a cached clean fraud verdict from the last 6 hours. -- The fraud API returns HTTP `403`, which is treated as clean/private/reserved. -- The fraud API returns a report but none of `is_abuser`, `is_attacker`, or `is_threat` are true. -- The fraud lookup itself fails. +- `--mode watch` is active — a would-be kill is only logged as `would kill`. +- The relevant reputation check is disabled via its flag. +- A cached reputation verdict from the last 6 hours says clean. +- The IP reputation API returns HTTP `403`, treated as clean/private/reserved. +- The reputation lookup itself fails — a failed lookup is never treated as a block. + +When a kill does happen, hexwall first records it in the local `killed_connections` audit table with the outcome `attempting`, so the record exists even if hexwall dies mid-kill. It then runs its guardrails: it refuses PID 1, its own PID, invalid PIDs, and a hardcoded set of protected system processes (`systemd`, `sshd`, `dockerd`, and similar), whichever check triggered the match. Once the attempt ends, the row's `outcome` says what happened: `killed`, `refused` (a guardrail declined it), or `failed` (no usable PID, or the kill itself errored). Rows written before this column existed say `unknown`. A kill the guardrails allow closes the connection in one of two ways: -When a kill does happen, hexwall first records the event in the local `killed_connections` audit table and then asks `somo` to kill the owning PID. +1. **Just that connection**: `ss -K` asks the kernel to destroy the single socket, leaving every other connection the process holds untouched. This needs `CAP_NET_ADMIN` (e.g. running as root) and a kernel built with `CONFIG_INET_DIAG_DESTROY`. Because `ss -K` can exit successfully even when the kernel refused, hexwall re-lists the socket afterwards and only counts the kill if the connection is actually gone. +2. **The whole process** (fallback): if the connection is still there, or `ss` is unavailable (always the case on macOS), hexwall signals the owning PID: `SIGTERM` first, escalating to `SIGKILL` after a 2-second grace period if the process is still alive. This also drops every other connection that process holds. -### Fraud API cache behavior +### Reputation API cache behavior -Fraud lookups are cached in the local SQLite database for 6 hours per IP. +Both IP and domain reputation lookups are cached in the local SQLite database for 6 hours, keyed by IP or by domain respectively. -- If an untrusted IP has a cached fraud decision newer than 6 hours, hexwall reuses that cached result and does not call the fraud API again. -- If the cached decision is older than 6 hours, the next scan calls the fraud API again and refreshes the cache timestamp. -- Both clean and kill-worthy fraud decisions are cached. -- HTTP `403` responses are treated as clean/private/reserved and cached as a non-kill result. -- Fraud lookup failures are not cached. +- If a cached decision is newer than 6 hours, hexwall reuses it and does not call the reputation API again. +- If the cached decision is older than 6 hours, the next scan calls the API again and refreshes the cache timestamp. +- Both clean and block-worthy decisions are cached. +- HTTP `403` responses (IP checks only) are treated as clean/private/reserved and cached as a non-kill result. +- A failed lookup is not cached, so it will be retried on the next scan. This cache lives in the local hexwall database as Unix timestamps (`INTEGER` / int64-style seconds) and survives restarts. @@ -161,29 +257,31 @@ This cache lives in the local hexwall database as Unix timestamps (`INTEGER` / i Pi-hole trust is built from recent DNS history, not by directly trusting every current connection. The refresh flow is: -1. Read the `queries` table from `pihole-FTL.db`. -2. Select distinct non-empty domains seen within the last hour. +1. Read the queries logged in `pihole-FTL.db` since the previous refresh (the last hour, on the first refresh). Each query is read once, tracked by its id. +2. Keep the non-empty domains **that Pi-hole answered**. Queries Pi-hole blocked are skipped, so a blocked domain's IP never becomes trusted, even if the host's resolver could look it up through another nameserver. Only the FTL statuses known to mean "answered" (forwarded, cached, retried, and their variants) count; a status hexwall doesn't recognise counts as not answered. 3. Normalize them to lowercase and trim whitespace. 4. Resolve each domain through the system DNS resolver, with bounded concurrency and a 1 second lookup timeout per domain. -5. Ignore domains that do not resolve. This is normal for blocked domains, expired records, and similar cases. +5. Ignore domains that do not resolve, such as expired records. + If the host's resolver is Pi-hole itself, these lookups are logged as queries too. hexwall remembers when it looked up each domain and skips the logged query that matches, so its own lookups never count as a client asking again. Without that, every domain would stay "recently seen", and trusted, forever. 6. Deduplicate resolved IPs. If multiple domains resolve to the same IP, the first domain in sorted order is stored for that IP. 7. Upsert each resolved IP into the local `allowed_ips` table, setting or refreshing `last_refreshed`. Important details: - Startup performs this refresh immediately before the first monitoring scan, so normal traffic seen in recent Pi-hole history is trusted before enforcement begins. -- The refresh repeats every 30 seconds. -- Trust from Pi-hole refresh lasts for 1 hour from the most recent successful upsert. +- The refresh repeats every 30 seconds, and only resolves domains clients asked for since the previous one. +- Trust from Pi-hole lasts for 1 hour from the most recent successful upsert, which is shortly after a client last asked for one of the IP's domains. +- If Pi-hole's query log is flushed or recreated (its ids restart), hexwall notices and re-reads the last hour. - Existing rows keep their original `first_approved` and `last_established` values when refreshed; only the stored domain and `last_refreshed` timestamp are updated. - The monitor also extends trust for long-lived allowed connections by updating `last_established` whenever an already-trusted connection is seen still established. This means an IP becomes trusted from Pi-hole history only after all of these happen: -1. Pi-hole recorded a domain query for it within the last hour. -2. That domain still resolves during a refresh cycle. +1. A client asked Pi-hole for one of its domains, and Pi-hole answered. +2. That domain still resolves at the next refresh. 3. One of the resolved IPs is written into the local `allowed_ips` cache. -If a domain was seen in Pi-hole history but no longer resolves during refresh, no IP is added for it, so there is nothing to trust from that domain alone. +If a domain was seen in Pi-hole history but doesn't resolve at the refresh, no IP is added for it, so there is nothing to trust from that domain alone. --- diff --git a/internal/allowlist/allowlist.go b/internal/allowlist/allowlist.go index f8446da..0737964 100644 --- a/internal/allowlist/allowlist.go +++ b/internal/allowlist/allowlist.go @@ -1,6 +1,8 @@ // Package allowlist defines statically trusted IP networks in CIDR notation. -// These networks bypass runtime reputation checks because they cover loopback, -// private-use space, and the link-local cloud metadata endpoint. +// These networks bypass runtime reputation checks because none of them is +// reachable on the public internet: loopback, private-use, shared (CGNAT, +// Tailscale), link-local, and their IPv6 equivalents. hexwall guards data +// leaving for the internet; a connection to one of these can't be that. package allowlist import ( @@ -11,12 +13,15 @@ import ( ) var cidrs = []string{ - "127.0.0.0/8", // entire IPv4 loopback block - "::1/128", // single IPv6 loopback address - "10.0.0.0/8", // RFC 1918 private-use block - "172.16.0.0/12", // RFC 1918 private-use block - "192.168.0.0/16", // RFC 1918 private-use block - "169.254.169.254/32", // link-local cloud metadata endpoint + "127.0.0.0/8", // entire IPv4 loopback block + "::1/128", // single IPv6 loopback address + "10.0.0.0/8", // RFC 1918 private-use block + "172.16.0.0/12", // RFC 1918 private-use block + "192.168.0.0/16", // RFC 1918 private-use block + "100.64.0.0/10", // RFC 6598 shared address space: carrier-grade NAT, Tailscale + "169.254.0.0/16", // IPv4 link-local, including the cloud metadata endpoint 169.254.169.254 + "fc00::/7", // IPv6 unique local addresses, IPv6's private-use space + "fe80::/10", // IPv6 link-local } var nets []*net.IPNet @@ -32,6 +37,7 @@ func loadCIDRs(values []string) { slog.Error("failed to parse allowlist CIDRs", "err", err) continue } + nets = append(nets, net) } } @@ -44,6 +50,7 @@ func parseCIDR(value string) (*net.IPNet, error) { if ip == nil { return nil, fmt.Errorf("invalid IP or CIDR: %q", value) } + if ip.To4() != nil { cidr += "/32" } else { diff --git a/internal/allowlist/allowlist_test.go b/internal/allowlist/allowlist_test.go index 14a947c..8565b7d 100644 --- a/internal/allowlist/allowlist_test.go +++ b/internal/allowlist/allowlist_test.go @@ -71,9 +71,11 @@ func TestParseCIDR(t *testing.T) { if err == nil { t.Fatalf("parseCIDR(%q) error = nil, want %q", tc.value, tc.wantErr) } + if err.Error() != tc.wantErr { t.Fatalf("parseCIDR(%q) error = %q, want %q", tc.value, err.Error(), tc.wantErr) } + return } @@ -94,6 +96,7 @@ func TestParseCIDR(t *testing.T) { if ip == nil { t.Fatalf("net.ParseIP(%q) = nil", ipText) } + if !got.Contains(ip) { t.Fatalf("network %q does not contain %q", got.String(), ipText) } @@ -104,6 +107,7 @@ func TestParseCIDR(t *testing.T) { if ip == nil { t.Fatalf("net.ParseIP(%q) = nil", ipText) } + if got.Contains(ip) { t.Fatalf("network %q unexpectedly contains %q", got.String(), ipText) } @@ -121,7 +125,7 @@ func TestLoadCIDRs(t *testing.T) { { name: "loads built in cidrs", values: cidrs, - wantCIDRs: []string{"127.0.0.0/8", "::1/128", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.169.254/32"}, + wantCIDRs: []string{"127.0.0.0/8", "::1/128", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10", "169.254.0.0/16", "fc00::/7", "fe80::/10"}, }, { name: "skips invalid entries and keeps valid ones", @@ -143,11 +147,13 @@ func TestLoadCIDRs(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { nets = nil + if tc.name == "keeps existing nets and appends new ones" { existing, err := parseCIDR("127.0.0.0/8") if err != nil { t.Fatalf("parseCIDR(existing) error = %v", err) } + nets = []*net.IPNet{existing} } @@ -161,6 +167,7 @@ func TestLoadCIDRs(t *testing.T) { if nets[i] == nil { t.Fatalf("nets[%d] = nil, want %q", i, wantCIDR) } + if nets[i].String() != wantCIDR { t.Fatalf("nets[%d] = %q, want %q", i, nets[i].String(), wantCIDR) } @@ -176,6 +183,7 @@ func TestContains(t *testing.T) { }() nets = nil + loadCIDRs([]string{"10.0.0.0/8", "::1", "169.254.169.254"}) testCases := []struct { @@ -232,6 +240,7 @@ func TestContains(t *testing.T) { func TestContainsWithNoNetworks(t *testing.T) { originalNets := nets + nets = nil defer func() { nets = originalNets @@ -241,3 +250,35 @@ func TestContainsWithNoNetworks(t *testing.T) { t.Fatal("Contains returned true with empty allowlist") } } + +// The built-in list, checked at the edges of each range: one address inside the edge must +// be trusted, the address just outside must not. Public addresses are never trusted. +func TestBuiltInRangesEdges(t *testing.T) { + trusted := []string{ + "100.64.0.1", "100.127.255.254", // shared space: CGNAT, Tailscale + "::ffff:100.101.2.3", // the same, written as IPv4-mapped IPv6 + "169.254.1.1", "169.254.169.254", // link-local, cloud metadata + "fd7a:115c:a1e0::1", "fc00::1", // IPv6 unique local (the first is Tailscale's) + "fe80::9518:1f57:3e8:56d7", // IPv6 link-local + "192.168.0.223", "10.0.0.1", "127.0.0.1", "::1", + } + public := []string{ + "100.63.255.255", "100.128.0.0", // just outside 100.64.0.0/10 + "169.253.255.255", "169.255.0.0", // just outside 169.254.0.0/16 + "fbff::1", "fe00::1", // just outside fc00::/7 + "fec0::1", // just outside fe80::/10 + "8.8.8.8", "1.1.1.1", "2001:4860:4860::8888", + } + + for _, addr := range trusted { + if !Contains(net.ParseIP(addr)) { + t.Errorf("Contains(%s) = false, want true", addr) + } + } + + for _, addr := range public { + if Contains(net.ParseIP(addr)) { + t.Errorf("Contains(%s) = true, want false: it can be a public address", addr) + } + } +} diff --git a/internal/connlist/backend_test.go b/internal/connlist/backend_test.go new file mode 100644 index 0000000..6bf35e0 --- /dev/null +++ b/internal/connlist/backend_test.go @@ -0,0 +1,133 @@ +package connlist + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// writeFakeBinary writes an executable script into dir under the given name. +func writeFakeBinary(t *testing.T, dir, name, script string) { + t.Helper() + + if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil { + t.Fatalf("write fake %s: %v", name, err) + } +} + +func TestSelectBackend(t *testing.T) { + okScript := "#!/bin/sh\nexit 0\n" + + tests := []struct { + name string + binaries []string // names to place on an otherwise-empty PATH + wantName string + wantErr bool + }{ + {name: "ss present picks ss", binaries: []string{"ss", "somo"}, wantName: "ss"}, + {name: "only somo present picks somo", binaries: []string{"somo"}, wantName: "somo"}, + {name: "neither present fails", binaries: nil, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + for _, bin := range tt.binaries { + writeFakeBinary(t, dir, bin, okScript) + } + + t.Setenv("PATH", dir) + + lister, name, err := SelectBackend() + if tt.wantErr { + if err == nil { + t.Fatal("SelectBackend() error = nil, want error") + } + + return + } + + if err != nil { + t.Fatalf("SelectBackend() error = %v, want nil", err) + } + + if name != tt.wantName { + t.Fatalf("SelectBackend() name = %q, want %q", name, tt.wantName) + } + + if lister == nil { + t.Fatal("SelectBackend() lister = nil, want non-nil") + } + }) + } +} + +// ---- selectBackend: macOS never probes for ss --------------------------- + +func TestSelectBackendDarwinNeverCallsSSAvailable(t *testing.T) { + t.Parallel() + + ssAvailableCalled := func() bool { + t.Error("ssAvailable() was called on darwin, want it never probed") + return true + } + somoInstalled := func() error { return nil } + + _, name, err := selectBackend("darwin", ssAvailableCalled, somoInstalled) + if err != nil { + t.Fatalf("selectBackend(darwin) error = %v, want nil", err) + } + + if name != "somo" { + t.Fatalf("selectBackend(darwin) name = %q, want %q (ss must never be picked on darwin, even if it claims available)", name, "somo") + } +} + +func TestSelectBackendDarwinFailureMessageOmitsSS(t *testing.T) { + t.Parallel() + + _, _, err := selectBackend("darwin", func() bool { return false }, func() error { return errors.New("not found") }) + if err == nil { + t.Fatal("selectBackend(darwin) error = nil, want error when somo is also missing") + } + + if !strings.Contains(err.Error(), "somo") { + t.Fatalf("selectBackend(darwin) error = %q, want it to mention somo", err) + } + + if strings.Contains(err.Error(), "install ss") { + t.Fatalf("selectBackend(darwin) error = %q, want it to NOT suggest installing ss on macOS", err) + } +} + +func TestSelectBackendLinuxUnaffected(t *testing.T) { + t.Parallel() + + lister, name, err := selectBackend("linux", func() bool { return true }, func() error { return nil }) + if err != nil { + t.Fatalf("selectBackend(linux) error = %v, want nil", err) + } + + if name != "ss" { + t.Fatalf("selectBackend(linux) name = %q, want %q", name, "ss") + } + + if lister == nil { + t.Fatal("selectBackend(linux) lister = nil, want non-nil") + } +} + +func TestSelectBackendLinuxFailureMessageMentionsBoth(t *testing.T) { + t.Parallel() + + _, _, err := selectBackend("linux", func() bool { return false }, func() error { return errors.New("not found") }) + if err == nil { + t.Fatal("selectBackend(linux) error = nil, want error") + } + + if !strings.Contains(err.Error(), "ss") || !strings.Contains(err.Error(), "somo") { + t.Fatalf("selectBackend(linux) error = %q, want it to mention both ss and somo", err) + } +} diff --git a/internal/connlist/connlist.go b/internal/connlist/connlist.go new file mode 100644 index 0000000..bdc1950 --- /dev/null +++ b/internal/connlist/connlist.go @@ -0,0 +1,90 @@ +// Package connlist lists established network connections via ss (preferred) +// or somo (fallback) — see SelectBackend. +package connlist + +import ( + "encoding/json" + "fmt" + "os/exec" + "runtime" + "strings" +) + +// Connection matches somo's default JSON output. +type Connection struct { + Protocol string `json:"proto"` + LPort string `json:"local_port"` + RAddress string `json:"remote_address"` + RPort string `json:"remote_port"` + Program string `json:"program"` + PID string `json:"pid"` + State string `json:"state"` + AddressType string `json:"address_type"` +} + +// CheckInstalled reports whether the somo CLI is available in PATH. +func CheckInstalled() error { + _, err := exec.LookPath("somo") + if err != nil { + return fmt.Errorf("somo is not installed in the path: %w", err) + } + + return nil +} + +// SelectBackend picks a connection-listing tool by availability: ss +// (iproute2) first since it needs no extra install on virtually any Linux +// box, falling back to somo when ss isn't on PATH. name is "ss" or "somo", +// for logging which tool is in effect. +func SelectBackend() (lister func() ([]Connection, error), name string, err error) { + return selectBackend(runtime.GOOS, SSAvailable, CheckInstalled) +} + +// selectBackend is SelectBackend's logic with the OS and both availability +// checks injectable, so tests can exercise the macOS branch deterministically +// regardless of which OS the test itself runs on. +// +// ss depends on CONFIG_INET_DIAG_DESTROY, a Linux kernel feature — a binary +// named "ss" on macOS could never actually destroy a socket even if one +// existed, so macOS never probes for it at all and goes straight to somo. +func selectBackend(goos string, ssAvailable func() bool, somoInstalled func() error) (lister func() ([]Connection, error), name string, err error) { + if goos != "darwin" && ssAvailable() { + return GetEstablishedConnectionsViaSS, "ss", nil + } + + if somoInstalled() == nil { + return GetEstablishedConnections, "somo", nil + } + + if goos == "darwin" { + return nil, "", fmt.Errorf("no connection-listing tool available: install somo (https://github.com/theopfr/somo) — ss (iproute2) is Linux-only and cannot work on macOS") + } + + return nil, "", fmt.Errorf("no connection-listing tool available: install ss (iproute2) or somo (https://github.com/theopfr/somo)") +} + +// GetEstablishedConnections returns a list of established TCP/UDP connections. +func GetEstablishedConnections() ([]Connection, error) { + cmd := exec.Command("somo", "-e", "--json", "--no-pager") + + output, err := cmd.CombinedOutput() + if err != nil { + return nil, commandError("failed to run somo", err, output) + } + + var result []Connection + if err := json.Unmarshal(output, &result); err != nil { + return nil, fmt.Errorf("failed to parse somo JSON: %w", err) + } + + return result, nil +} + +func commandError(message string, err error, output []byte) error { + trimmed := strings.TrimSpace(string(output)) + if trimmed == "" { + return fmt.Errorf("%s: %w", message, err) + } + + return fmt.Errorf("%s: %w: %s", message, err, trimmed) +} diff --git a/internal/connlist/connlist_test.go b/internal/connlist/connlist_test.go new file mode 100644 index 0000000..7930256 --- /dev/null +++ b/internal/connlist/connlist_test.go @@ -0,0 +1,169 @@ +package connlist + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// installFakeSomo writes an executable shell script named "somo" into a temp +// directory and prepends that directory to PATH, so exec.LookPath("somo") and +// exec.Command("somo", ...) resolve to the fake instead of a real install. +func installFakeSomo(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + + path := filepath.Join(dir, "somo") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake somo: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestCheckInstalledSucceedsWhenOnPath(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\nexit 0\n") + + if err := CheckInstalled(); err != nil { + t.Fatalf("CheckInstalled() error = %v, want nil", err) + } +} + +func TestCheckInstalledFailsWhenPathEmpty(t *testing.T) { + t.Setenv("PATH", "") + + if err := CheckInstalled(); err == nil { + t.Fatal("CheckInstalled() error = nil, want error for empty PATH") + } +} + +func TestCheckInstalledFailsWhenNotOnPath(t *testing.T) { + // A PATH pointing only at an empty directory has no somo binary. + t.Setenv("PATH", t.TempDir()) + + if err := CheckInstalled(); err == nil { + t.Fatal("CheckInstalled() error = nil, want error when somo is not on PATH") + } +} + +func TestGetEstablishedConnectionsParsesAllFields(t *testing.T) { + installFakeSomo(t, `#!/bin/sh +echo '[{"proto":"tcp","local_port":"51820","remote_address":"203.0.113.10","remote_port":"443","program":"curl","pid":"1234","state":"ESTABLISHED","address_type":"ipv4"}]' +`) + + conns, err := GetEstablishedConnections() + if err != nil { + t.Fatalf("GetEstablishedConnections() error = %v", err) + } + + if len(conns) != 1 { + t.Fatalf("GetEstablishedConnections() len = %d, want 1", len(conns)) + } + + want := Connection{ + Protocol: "tcp", + LPort: "51820", + RAddress: "203.0.113.10", + RPort: "443", + Program: "curl", + PID: "1234", + State: "ESTABLISHED", + AddressType: "ipv4", + } + if conns[0] != want { + t.Fatalf("GetEstablishedConnections()[0] = %+v, want %+v", conns[0], want) + } +} + +func TestGetEstablishedConnectionsEmptyArray(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho '[]'\n") + + conns, err := GetEstablishedConnections() + if err != nil { + t.Fatalf("GetEstablishedConnections() error = %v", err) + } + + if len(conns) != 0 { + t.Fatalf("GetEstablishedConnections() len = %d, want 0", len(conns)) + } +} + +func TestGetEstablishedConnectionsMalformedJSON(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho 'not json'\n") + + _, err := GetEstablishedConnections() + if err == nil { + t.Fatal("GetEstablishedConnections() error = nil, want error for malformed JSON") + } + + if !strings.Contains(err.Error(), "failed to parse") { + t.Fatalf("GetEstablishedConnections() error = %q, want it to mention %q", err.Error(), "failed to parse") + } +} + +func TestGetEstablishedConnectionsNonZeroExit(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho 'somo: permission denied' 1>&2\nexit 1\n") + + _, err := GetEstablishedConnections() + if err == nil { + t.Fatal("GetEstablishedConnections() error = nil, want error for non-zero exit") + } + + if !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("GetEstablishedConnections() error = %q, want it to include command output", err.Error()) + } +} + +func TestCommandError(t *testing.T) { + t.Parallel() + + wrapped := errors.New("exit status 1") + + tests := []struct { + name string + message string + err error + output []byte + want string + }{ + { + name: "empty output", + message: "failed to run somo", + err: wrapped, + output: nil, + want: "failed to run somo: exit status 1", + }, + { + name: "whitespace-only output", + message: "failed to run somo", + err: wrapped, + output: []byte(" \n\t "), + want: "failed to run somo: exit status 1", + }, + { + name: "non-empty output", + message: "failed to run somo", + err: wrapped, + output: []byte("permission denied\n"), + want: "failed to run somo: exit status 1: permission denied", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := commandError(tt.message, tt.err, tt.output) + if got.Error() != tt.want { + t.Fatalf("commandError() = %q, want %q", got.Error(), tt.want) + } + + if !errors.Is(got, wrapped) { + t.Fatal("commandError() does not wrap the original error") + } + }) + } +} diff --git a/internal/connlist/ss_lister.go b/internal/connlist/ss_lister.go new file mode 100644 index 0000000..3e45fc0 --- /dev/null +++ b/internal/connlist/ss_lister.go @@ -0,0 +1,94 @@ +package connlist + +import ( + "bufio" + "fmt" + "os/exec" + "regexp" + "strings" +) + +// processFieldPattern extracts the first "prog",pid=N entry from ss's +// users:(("prog",pid=N,fd=M),...) process column. Only the first owner is +// kept, matching somo's one-program-per-connection shape. +var processFieldPattern = regexp.MustCompile(`users:\(\("([^"]+)",pid=(\d+),fd=\d+\)`) + +// SSAvailable reports whether the ss CLI (iproute2) is available in PATH. +func SSAvailable() bool { + _, err := exec.LookPath("ss") + return err == nil +} + +// GetEstablishedConnectionsViaSS lists established TCP connections using ss, +// the preferred backend when iproute2 is available (Linux, not macOS). +func GetEstablishedConnectionsViaSS() ([]Connection, error) { + cmd := exec.Command("ss", "-H", "-tnp", "state", "established") + + output, err := cmd.CombinedOutput() + if err != nil { + return nil, commandError("failed to run ss", err, output) + } + + return parseSSOutput(string(output)), nil +} + +// parseSSOutput parses `ss -H -tnp state established` output into Connections. +// A malformed line is skipped rather than aborting the whole parse. A +// connection whose owning process ss couldn't identify (no users:(...) field, +// e.g. owned by another user without CAP_NET_ADMIN) yields an empty +// Program/PID, since enforcer.ParsePID already treats "" as invalid. +func parseSSOutput(output string) []Connection { + var connections []Connection + + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 4 { + continue + } + + _, localPort, err := splitSSAddress(fields[2]) + if err != nil { + continue + } + + remoteHost, remotePort, err := splitSSAddress(fields[3]) + if err != nil { + continue + } + + conn := Connection{ + Protocol: "tcp", + LPort: localPort, + RAddress: remoteHost, + RPort: remotePort, + State: "ESTAB", + } + + if len(fields) >= 5 { + if m := processFieldPattern.FindStringSubmatch(fields[4]); m != nil { + conn.Program = m[1] + conn.PID = m[2] + } + } + + connections = append(connections, conn) + } + + return connections +} + +// splitSSAddress splits ss's "host:port" (or "[ipv6]:port") address column. +func splitSSAddress(field string) (host, port string, err error) { + idx := strings.LastIndex(field, ":") + if idx < 0 { + return "", "", fmt.Errorf("no port separator in address %q", field) + } + + host, port = field[:idx], field[idx+1:] + if host == "" || port == "" { + return "", "", fmt.Errorf("empty host or port in address %q", field) + } + + return host, port, nil +} diff --git a/internal/connlist/ss_lister_test.go b/internal/connlist/ss_lister_test.go new file mode 100644 index 0000000..5d5e2a2 --- /dev/null +++ b/internal/connlist/ss_lister_test.go @@ -0,0 +1,153 @@ +package connlist + +import ( + "os" + "path/filepath" + "testing" +) + +// installFakeSS writes an executable shell script named "ss" into a temp +// directory and prepends it to PATH, mirroring installFakeSomo below and the +// same trick used in internal/enforcer/enforcer_socket_test.go. +func installFakeSS(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + + path := filepath.Join(dir, "ss") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake ss: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +// realSSFixture is real `ss -H -tnp state established` output captured on a +// live Linux box (iproute2-6.1.0), used verbatim so the parser is validated +// against actual formatting rather than assumed grammar. +const realSSFixture = `0 0 192.168.0.223:53404 160.79.104.10:443 users:(("claude",pid=6039,fd=11)) +0 0 192.168.0.223:56926 160.79.104.10:443 users:(("claude-desktop",pid=5831,fd=22)) +0 0 192.168.0.223:41492 140.82.114.25:443 users:(("firefox-bin",pid=4076,fd=52)) +0 0 192.168.0.223:48878 64.233.170.188:5228 users:(("chrome",pid=5068,fd=37)) +` + +func TestParseSSOutputRealFixture(t *testing.T) { + conns := parseSSOutput(realSSFixture) + if len(conns) != 4 { + t.Fatalf("parseSSOutput() len = %d, want 4", len(conns)) + } + + want := Connection{Protocol: "tcp", LPort: "53404", RAddress: "160.79.104.10", RPort: "443", Program: "claude", PID: "6039", State: "ESTAB"} + if conns[0] != want { + t.Fatalf("parseSSOutput()[0] = %+v, want %+v", conns[0], want) + } + + if conns[3].RPort != "5228" || conns[3].RAddress != "64.233.170.188" { + t.Fatalf("parseSSOutput()[3] remote = %s:%s, want 64.233.170.188:5228", conns[3].RAddress, conns[3].RPort) + } +} + +func TestParseSSOutputIPv6(t *testing.T) { + // Captured live from a real IPv6 loopback connection: ss brackets IPv6 + // addresses, unlike its IPv4 output. splitSSAddress's LastIndex(":") + // split leaves the brackets in RAddress rather than stripping them — + // this is only safe because monitor.remoteIP() strips brackets itself as + // a fallback. This test exists to catch it if that ever stops being true. + const line = "0 0 [::1]:56328 [::1]:12450 users:((\"nc\",pid=48399,fd=3))\n" + + conns := parseSSOutput(line) + if len(conns) != 1 { + t.Fatalf("parseSSOutput() len = %d, want 1", len(conns)) + } + + want := Connection{Protocol: "tcp", LPort: "56328", RAddress: "[::1]", RPort: "12450", Program: "nc", PID: "48399", State: "ESTAB"} + if conns[0] != want { + t.Fatalf("parseSSOutput()[0] = %+v, want %+v", conns[0], want) + } +} + +func TestParseSSOutputUnattributedProcess(t *testing.T) { + // ss omits the whole process column when it can't attribute the socket + // to a process (e.g. owned by another user without CAP_NET_ADMIN). + const line = "0 0 192.168.0.223:53404 160.79.104.10:443\n" + + conns := parseSSOutput(line) + if len(conns) != 1 { + t.Fatalf("parseSSOutput() len = %d, want 1", len(conns)) + } + + if conns[0].Program != "" || conns[0].PID != "" { + t.Fatalf("parseSSOutput()[0] = %+v, want empty Program/PID", conns[0]) + } + + if conns[0].RAddress != "160.79.104.10" || conns[0].RPort != "443" { + t.Fatalf("parseSSOutput()[0] remote = %s:%s, want 160.79.104.10:443", conns[0].RAddress, conns[0].RPort) + } +} + +func TestParseSSOutputMultipleOwners(t *testing.T) { + const line = `0 0 192.168.0.223:53404 160.79.104.10:443 users:(("nginx",pid=100,fd=5),("nginx",pid=100,fd=6)) +` + + conns := parseSSOutput(line) + if len(conns) != 1 { + t.Fatalf("parseSSOutput() len = %d, want 1", len(conns)) + } + + if conns[0].Program != "nginx" || conns[0].PID != "100" { + t.Fatalf("parseSSOutput()[0] = %+v, want Program=nginx PID=100", conns[0]) + } +} + +func TestParseSSOutputSkipsMalformedLines(t *testing.T) { + const output = `not even close to a valid line +0 0 192.168.0.223:53404 160.79.104.10:443 users:(("claude",pid=6039,fd=11)) +192.168.0.223:missingport +` + + conns := parseSSOutput(output) + if len(conns) != 1 { + t.Fatalf("parseSSOutput() len = %d, want 1 (malformed lines skipped)", len(conns)) + } + + if conns[0].Program != "claude" { + t.Fatalf("parseSSOutput()[0].Program = %q, want claude", conns[0].Program) + } +} + +func TestSSAvailable(t *testing.T) { + installFakeSS(t, "#!/bin/sh\nexit 0\n") + + if !SSAvailable() { + t.Fatal("SSAvailable() = false, want true with fake ss on PATH") + } +} + +func TestSSAvailableFalseWhenNotOnPath(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + + if SSAvailable() { + t.Fatal("SSAvailable() = true, want false with empty PATH") + } +} + +func TestGetEstablishedConnectionsViaSS(t *testing.T) { + installFakeSS(t, "#!/bin/sh\ncat <<'EOF'\n"+realSSFixture+"EOF\n") + + conns, err := GetEstablishedConnectionsViaSS() + if err != nil { + t.Fatalf("GetEstablishedConnectionsViaSS() error = %v", err) + } + + if len(conns) != 4 { + t.Fatalf("GetEstablishedConnectionsViaSS() len = %d, want 4", len(conns)) + } +} + +func TestGetEstablishedConnectionsViaSSCommandFailure(t *testing.T) { + installFakeSS(t, "#!/bin/sh\necho 'ss: permission denied' 1>&2\nexit 1\n") + + if _, err := GetEstablishedConnectionsViaSS(); err == nil { + t.Fatal("GetEstablishedConnectionsViaSS() error = nil, want error when ss fails") + } +} diff --git a/internal/deghost/deghost.go b/internal/deghost/deghost.go index f7e6788..6801a8c 100644 --- a/internal/deghost/deghost.go +++ b/internal/deghost/deghost.go @@ -44,6 +44,7 @@ type SecurityReport struct { // NewClient creates a Client for the given API base URL. func NewClient(baseURL string, timeout time.Duration) *Client { trimmed := strings.TrimRight(baseURL, "/") + return &Client{ baseURL: trimmed, httpClient: &http.Client{ @@ -58,6 +59,7 @@ func (c *Client) CheckIP(ctx context.Context, ip string) (*IPReport, error) { if c == nil { return nil, errors.New("nil deghost client") } + if c.httpClient == nil { return nil, errors.New("nil deghost http client") } @@ -119,3 +121,84 @@ func ShouldKill(report *IPReport) bool { return report.Security.IsAbuser || report.Security.IsAttacker || report.Security.IsThreat } + +// DomainReport matches the API payload for domain reputation checks. +type DomainReport struct { + Status string `json:"status"` + HasMX bool `json:"has_mx"` + Disposable bool `json:"disposable"` + Spam bool `json:"spam"` + PublicDomain bool `json:"public_domain"` + RelayDomain bool `json:"relay_domain"` + Blacklisted bool `json:"blacklisted"` + DomainAgeInDays int `json:"domain_age_in_days"` +} + +// CheckDomain fetches a reputation report for a single domain. +// Unlike CheckIP's 403 handling (nil report for private/reserved IPs), this endpoint +// returns a real JSON body on HTTP 200, 400, and 403 — all are decoded into a DomainReport. +// Only 499/504/500/other status codes are treated as hard errors. +func (c *Client) CheckDomain(ctx context.Context, domain string) (*DomainReport, error) { + if c == nil { + return nil, errors.New("nil deghost client") + } + + if c.httpClient == nil { + return nil, errors.New("nil deghost http client") + } + + baseURL := strings.TrimSpace(c.baseURL) + if baseURL == "" { + return nil, errors.New("deghost base URL is required") + } + + domain = strings.ToLower(strings.TrimSpace(domain)) + if domain == "" { + return nil, errors.New("domain is required") + } + + endpoint, err := url.JoinPath(baseURL, "domain", domain) + if err != nil { + return nil, fmt.Errorf("build endpoint: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusBadRequest && resp.StatusCode != http.StatusForbidden { + return nil, fmt.Errorf("deghost returned status %d %s", resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + var report DomainReport + if err := json.NewDecoder(resp.Body).Decode(&report); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + + return &report, nil +} + +// ShouldBlockDomain reports whether the domain report matches the current block policy. +// Only an explicit blacklist entry counts, consistent with how ShouldKill only fires on +// explicit threat fields. +// +// Status is not used: it judges a domain as an email domain. "not_allowed" comes back +// for any domain that can't receive mail (no MX record), which covers most API and CDN +// hosts (api.github.com, cdn.jsdelivr.net), while an ad tracker with MX records is +// "allowed". Softer signals (disposable, spam, domain age) are too noisy to act on. +func ShouldBlockDomain(report *DomainReport) bool { + if report == nil { + return false + } + + return report.Blacklisted +} diff --git a/internal/deghost/deghost_test.go b/internal/deghost/deghost_test.go new file mode 100644 index 0000000..241b26b --- /dev/null +++ b/internal/deghost/deghost_test.go @@ -0,0 +1,399 @@ +package deghost + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestCheckDomainOK(t *testing.T) { + t.Parallel() + + want := DomainReport{ + Status: "allowed", + HasMX: true, + Disposable: false, + Spam: false, + PublicDomain: true, + RelayDomain: false, + Blacklisted: false, + DomainAgeInDays: 11117, + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("method = %s, want GET", r.Method) + } + + if r.URL.Path != "/domain/example.com" { + t.Errorf("path = %s, want /domain/example.com", r.URL.Path) + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(want) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + + got, err := client.CheckDomain(context.Background(), "example.com") + if err != nil { + t.Fatalf("CheckDomain() error = %v", err) + } + + if got == nil { + t.Fatal("CheckDomain() = nil, want report") + } + + if got.Status != want.Status { + t.Errorf("Status = %q, want %q", got.Status, want.Status) + } + + if got.HasMX != want.HasMX { + t.Errorf("HasMX = %v, want %v", got.HasMX, want.HasMX) + } + + if got.DomainAgeInDays != want.DomainAgeInDays { + t.Errorf("DomainAgeInDays = %d, want %d", got.DomainAgeInDays, want.DomainAgeInDays) + } +} + +func TestCheckDomainStatusForbiddenDecodesBody(t *testing.T) { + t.Parallel() + + want := DomainReport{ + Status: "not_allowed", + HasMX: false, + Disposable: false, + Spam: false, + PublicDomain: false, + RelayDomain: false, + Blacklisted: false, + DomainAgeInDays: 0, + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(want) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + + got, err := client.CheckDomain(context.Background(), "blocked.example.com") + if err != nil { + t.Fatalf("CheckDomain() on 403 error = %v", err) + } + + if got == nil { + t.Fatal("CheckDomain() on 403 = nil, want decoded report (not nil like IP endpoint)") + } + + if got.Status != "not_allowed" { + t.Errorf("Status = %q, want %q", got.Status, "not_allowed") + } +} + +func TestCheckDomainStatusBadRequestDecodesBody(t *testing.T) { + t.Parallel() + + want := DomainReport{ + Status: "not_allowed", + HasMX: false, + Blacklisted: true, + DomainAgeInDays: 0, + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(want) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + + got, err := client.CheckDomain(context.Background(), "bad.example.com") + if err != nil { + t.Fatalf("CheckDomain() on 400 error = %v", err) + } + + if got == nil { + t.Fatal("CheckDomain() on 400 = nil, want decoded report") + } + + if !got.Blacklisted { + t.Error("Blacklisted = false, want true") + } +} + +func TestCheckDomainStatusInternalServerError(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on 500 = %v, want error", got) + } +} + +func TestCheckDomainStatusGatewayTimeout(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusGatewayTimeout) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on 504 = %v, want error", got) + } +} + +func TestCheckDomainStatusClientClosedRequest(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(499) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on 499 = %v, want error", got) + } +} + +func TestCheckDomainMalformedJSON(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if _, err := fmt.Fprint(w, `{"status": "allowed" bad json`); err != nil { + t.Errorf("fmt.Fprint: %v", err) + } + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on bad JSON = %v, want error", got) + } +} + +func TestCheckDomainNilClient(t *testing.T) { + t.Parallel() + + var c *Client + + got, err := c.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on nil client = %v, want error", got) + } +} + +func TestCheckDomainEmptyBaseURL(t *testing.T) { + t.Parallel() + + client := &Client{baseURL: "", httpClient: &http.Client{}} + + got, err := client.CheckDomain(context.Background(), "example.com") + if err == nil { + t.Fatalf("CheckDomain() on empty baseURL = %v, want error", got) + } +} + +func TestCheckDomainEmptyDomain(t *testing.T) { + t.Parallel() + + client := NewClient("https://example.com", 5) + + got, err := client.CheckDomain(context.Background(), " ") + if err == nil { + t.Fatalf("CheckDomain() on empty domain = %v, want error", got) + } +} + +func TestCheckDomainNormalized(t *testing.T) { + t.Parallel() + + var gotPath string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(DomainReport{Status: "allowed"}) + })) + defer srv.Close() + + client := NewClient(srv.URL, srv.Client().Timeout) + + _, err := client.CheckDomain(context.Background(), " Example.COM ") + if err != nil { + t.Fatalf("CheckDomain() error = %v", err) + } + + want := "/domain/example.com" + if gotPath != want { + t.Errorf("request path = %q, want %q (domain should be trimmed and lowercased)", gotPath, want) + } +} + +func TestShouldBlockDomainNilReport(t *testing.T) { + t.Parallel() + + if ShouldBlockDomain(nil) { + t.Error("ShouldBlockDomain(nil) = true, want false") + } +} + +// "not_allowed" is the API's verdict on a domain as an email domain. This is its real +// answer for api.github.com (2026-10-12): no MX record, so "not_allowed". Killing on it +// would kill GitHub's API. +func TestShouldBlockDomainNotAllowedDoesNotBlock(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "not_allowed", HasMX: false, Blacklisted: false} + if ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(not_allowed, no MX) = true, want false: that's an email check, not a threat verdict") + } +} + +func TestShouldBlockDomainBlacklisted(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "allowed", Blacklisted: true} + if !ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(blacklisted) = false, want true") + } +} + +func TestShouldBlockDomainBothFalse(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "allowed", Blacklisted: false} + if ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(allowed, not blacklisted) = true, want false") + } +} + +func TestShouldBlockDomainDisposableAloneDoesNotBlock(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "allowed", Disposable: true} + if ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(disposable alone) = true, want false") + } +} + +func TestShouldBlockDomainSpamAloneDoesNotBlock(t *testing.T) { + t.Parallel() + + report := &DomainReport{Status: "allowed", Spam: true} + if ShouldBlockDomain(report) { + t.Error("ShouldBlockDomain(spam alone) = true, want false") + } +} + +func TestCheckIPOK(t *testing.T) { + var gotPath string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _, _ = w.Write([]byte(`{"ip":"203.0.113.9","security":{"is_attacker":true}}`)) + })) + t.Cleanup(srv.Close) + + report, err := NewClient(srv.URL, time.Second).CheckIP(context.Background(), " 203.0.113.9 ") + if err != nil { + t.Fatalf("CheckIP() error = %v", err) + } + + if gotPath != "/ip/203.0.113.9" { + t.Fatalf("request path = %q, want /ip/203.0.113.9", gotPath) + } + + if report == nil || !report.Security.IsAttacker { + t.Fatalf("CheckIP() report = %+v, want is_attacker decoded", report) + } +} + +// The API answers 403 for private and reserved IPs; that is a clean verdict, not an error. +func TestCheckIPForbiddenIsCleanNotError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(srv.Close) + + report, err := NewClient(srv.URL, time.Second).CheckIP(context.Background(), "10.0.0.1") + if err != nil || report != nil { + t.Fatalf("CheckIP() = %+v, %v; want nil, nil", report, err) + } +} + +// Any other failure is an error, never a verdict: a failed lookup must not become a kill. +func TestCheckIPErrors(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ip/203.0.113.1" { + w.WriteHeader(http.StatusInternalServerError) + return + } + + _, _ = w.Write([]byte(`{not json`)) + })) + t.Cleanup(srv.Close) + client := NewClient(srv.URL, time.Second) + + for _, ip := range []string{"203.0.113.1", "203.0.113.2", "not-an-ip", ""} { + if report, err := client.CheckIP(context.Background(), ip); err == nil || report != nil { + t.Errorf("CheckIP(%q) = %+v, %v; want nil report and an error", ip, report, err) + } + } + + var nilClient *Client + if _, err := nilClient.CheckIP(context.Background(), "203.0.113.1"); err == nil { + t.Error("nil client CheckIP() error = nil, want error") + } +} + +func TestShouldKill(t *testing.T) { + tests := []struct { + name string + security SecurityReport + want bool + }{ + {"abuser", SecurityReport{IsAbuser: true}, true}, + {"attacker", SecurityReport{IsAttacker: true}, true}, + {"threat", SecurityReport{IsThreat: true}, true}, + // Anonymity alone is too noisy to kill on: VPN, Tor and cloud users are not attackers. + {"vpn, tor, proxy, cloud", SecurityReport{IsVPN: true, IsTor: true, IsProxy: true, IsCloudProvider: true}, false}, + {"clean", SecurityReport{}, false}, + } + for _, tt := range tests { + if got := ShouldKill(&IPReport{Security: tt.security}); got != tt.want { + t.Errorf("ShouldKill(%s) = %v, want %v", tt.name, got, tt.want) + } + } + + if ShouldKill(nil) { + t.Error("ShouldKill(nil) = true, want false") + } +} diff --git a/internal/detector/detector.go b/internal/detector/detector.go index 86f9678..47a5958 100644 --- a/internal/detector/detector.go +++ b/internal/detector/detector.go @@ -23,19 +23,26 @@ const ( // // Returns an error if neither is found. func FindDBPath() (string, error) { + return findDBPath(physicalDBPath, findDockerDBPath) +} + +// findDBPath is FindDBPath's logic with the physical path and Docker fallback +// injectable, so tests can exercise both branches without touching +// /etc/pihole or requiring a real Docker daemon. +func findDBPath(physicalPath string, dockerFallback func() (string, error)) (string, error) { // Check the default host installation first. - if err := validateDBFile(physicalDBPath); err == nil { - return physicalDBPath, nil + if err := validateDBFile(physicalPath); err == nil { + return physicalPath, nil } else if !errors.Is(err, os.ErrNotExist) { - return "", fmt.Errorf("failed to access physical pi-hole DB at %s: %w", physicalDBPath, err) + return "", fmt.Errorf("failed to access physical pi-hole DB at %s: %w", physicalPath, err) } // Fall back to a running Docker installation. - path, err := findDockerDBPath() + path, err := dockerFallback() if err != nil { return "", fmt.Errorf( "pi-hole not found: no physical installation at %s and no running pi-hole Docker container (%w)", - physicalDBPath, err, + physicalPath, err, ) } diff --git a/internal/detector/detector_test.go b/internal/detector/detector_test.go new file mode 100644 index 0000000..7b17bd7 --- /dev/null +++ b/internal/detector/detector_test.go @@ -0,0 +1,308 @@ +package detector + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// installFakeDocker writes an executable shell script named "docker" into a +// temp directory and prepends it to PATH, mirroring the same trick used for +// somo elsewhere in this project (see internal/connlist/connlist_test.go). +func installFakeDocker(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + + path := filepath.Join(dir, "docker") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake docker: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func writeRegularFile(t *testing.T, path string) { + t.Helper() + + if err := os.WriteFile(path, []byte("fake db"), 0o644); err != nil { + t.Fatalf("write file %s: %v", path, err) + } +} + +// failDockerFallback returns a fallback that fails the test if it is ever +// called, for asserting the physical-path branch short-circuits correctly. +func failDockerFallback(t *testing.T) func() (string, error) { + t.Helper() + + return func() (string, error) { + t.Fatal("docker fallback should not have been called") + return "", nil + } +} + +// ---- validateDBFile -------------------------------------------------- + +func TestValidateDBFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + regular := filepath.Join(dir, "pihole-FTL.db") + writeRegularFile(t, regular) + + if err := validateDBFile(regular); err != nil { + t.Fatalf("validateDBFile(regular file) error = %v, want nil", err) + } + + if err := validateDBFile(dir); err == nil { + t.Fatal("validateDBFile(directory) error = nil, want error") + } else if !strings.Contains(err.Error(), "not a regular file") { + t.Fatalf("validateDBFile(directory) error = %q, want it to mention 'not a regular file'", err) + } + + missing := filepath.Join(dir, "nope.db") + if err := validateDBFile(missing); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("validateDBFile(missing) error = %v, want os.ErrNotExist", err) + } +} + +// ---- findDBPath (physical vs. Docker fallback) ------------------------- + +func TestFindDBPathPhysicalFound(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + physical := filepath.Join(dir, "pihole-FTL.db") + writeRegularFile(t, physical) + + got, err := findDBPath(physical, failDockerFallback(t)) + if err != nil { + t.Fatalf("findDBPath() error = %v", err) + } + + if got != physical { + t.Fatalf("findDBPath() = %q, want %q", got, physical) + } +} + +func TestFindDBPathPhysicalIsDirectoryShortCircuits(t *testing.T) { + t.Parallel() + + dir := t.TempDir() // a directory, not a regular file + + _, err := findDBPath(dir, failDockerFallback(t)) + if err == nil { + t.Fatal("findDBPath() error = nil, want error") + } + + if !strings.Contains(err.Error(), "failed to access physical pi-hole DB") { + t.Fatalf("findDBPath() error = %q, want it to mention accessing the physical DB", err) + } +} + +func TestFindDBPathFallsBackToDockerWhenPhysicalMissing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + missingPhysical := filepath.Join(dir, "nope.db") + dockerPath := filepath.Join(dir, "docker-mounted.db") + + got, err := findDBPath(missingPhysical, func() (string, error) { + return dockerPath, nil + }) + if err != nil { + t.Fatalf("findDBPath() error = %v", err) + } + + if got != dockerPath { + t.Fatalf("findDBPath() = %q, want %q", got, dockerPath) + } +} + +func TestFindDBPathBothMissing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + missingPhysical := filepath.Join(dir, "nope.db") + + _, err := findDBPath(missingPhysical, func() (string, error) { + return "", errors.New("no running pi-hole container found") + }) + if err == nil { + t.Fatal("findDBPath() error = nil, want error") + } + + if !strings.Contains(err.Error(), "pi-hole not found") { + t.Fatalf("findDBPath() error = %q, want it to mention 'pi-hole not found'", err) + } +} + +// ---- findPiholeContainer ------------------------------------------------ + +func TestFindPiholeContainerMatchesByName(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"abc123\tmy-pihole-1\tsomeimage:latest\" ;;\nesac\n") + + id, err := findPiholeContainer() + if err != nil { + t.Fatalf("findPiholeContainer() error = %v", err) + } + + if id != "abc123" { + t.Fatalf("findPiholeContainer() = %q, want %q", id, "abc123") + } +} + +func TestFindPiholeContainerMatchesByImage(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"def456\tunrelated-name\tpihole/pihole:2024.07.0\" ;;\nesac\n") + + id, err := findPiholeContainer() + if err != nil { + t.Fatalf("findPiholeContainer() error = %v", err) + } + + if id != "def456" { + t.Fatalf("findPiholeContainer() = %q, want %q", id, "def456") + } +} + +func TestFindPiholeContainerCaseInsensitive(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"ghi789\tMyPiHoleBox\tSomeImage\" ;;\nesac\n") + + id, err := findPiholeContainer() + if err != nil { + t.Fatalf("findPiholeContainer() error = %v", err) + } + + if id != "ghi789" { + t.Fatalf("findPiholeContainer() = %q, want %q", id, "ghi789") + } +} + +func TestFindPiholeContainerNoneFound(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"xyz999\tunrelated\tother/image\" ;;\nesac\n") + + _, err := findPiholeContainer() + if err == nil { + t.Fatal("findPiholeContainer() error = nil, want error") + } + + if !strings.Contains(err.Error(), "no running pi-hole container found") { + t.Fatalf("findPiholeContainer() error = %q, want it to mention no container found", err) + } +} + +func TestFindPiholeContainerDockerNotRunning(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\necho \"cannot connect to the docker daemon\" >&2\nexit 1\n") + + _, err := findPiholeContainer() + if err == nil { + t.Fatal("findPiholeContainer() error = nil, want error") + } + + if !strings.Contains(err.Error(), "docker ps failed") { + t.Fatalf("findPiholeContainer() error = %q, want it to mention 'docker ps failed'", err) + } +} + +// ---- findDockerDBPath ------------------------------------------------- + +func TestFindDockerDBPathSuccess(t *testing.T) { + mountDir := t.TempDir() + writeRegularFile(t, filepath.Join(mountDir, ftlDBFilename)) + + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo '[{\"Source\":\""+mountDir+"\",\"Destination\":\"/etc/pihole\"}]' ;;\n"+ + "esac\n") + + got, err := findDockerDBPath() + if err != nil { + t.Fatalf("findDockerDBPath() error = %v", err) + } + + want := filepath.Join(mountDir, ftlDBFilename) + if got != want { + t.Fatalf("findDockerDBPath() = %q, want %q", got, want) + } +} + +func TestFindDockerDBPathNoPiholeMount(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo '[{\"Source\":\"/some/other\",\"Destination\":\"/var/lib/other\"}]' ;;\n"+ + "esac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + + if !strings.Contains(err.Error(), "no /etc/pihole mount detected") { + t.Fatalf("findDockerDBPath() error = %q, want it to mention no mount detected", err) + } +} + +func TestFindDockerDBPathMountedButDBMissing(t *testing.T) { + mountDir := t.TempDir() // no pihole-FTL.db written inside + + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo '[{\"Source\":\""+mountDir+"\",\"Destination\":\"/etc/pihole\"}]' ;;\n"+ + "esac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + + if !strings.Contains(err.Error(), "DB not available at expected path") { + t.Fatalf("findDockerDBPath() error = %q, want it to mention DB not available", err) + } +} + +func TestFindDockerDBPathContainerNotFound(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n ps) echo \"cid1\tunrelated\tother/image\" ;;\nesac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + + if !strings.Contains(err.Error(), "no running pi-hole container found") { + t.Fatalf("findDockerDBPath() error = %q, want it to propagate the container-not-found error", err) + } +} + +func TestFindDockerDBPathInspectFails(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo \"no such container\" >&2; exit 1 ;;\n"+ + "esac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + + if !strings.Contains(err.Error(), "docker inspect failed") { + t.Fatalf("findDockerDBPath() error = %q, want it to mention 'docker inspect failed'", err) + } +} + +func TestFindDockerDBPathMalformedInspectJSON(t *testing.T) { + installFakeDocker(t, "#!/bin/sh\ncase \"$1\" in\n"+ + " ps) echo \"cid1\tmy-pihole\tpihole/pihole\" ;;\n"+ + " inspect) echo \"not json at all\" ;;\n"+ + "esac\n") + + _, err := findDockerDBPath() + if err == nil { + t.Fatal("findDockerDBPath() error = nil, want error") + } + + if !strings.Contains(err.Error(), "failed to parse docker inspect output") { + t.Fatalf("findDockerDBPath() error = %q, want it to mention JSON parse failure", err) + } +} diff --git a/internal/enforcer/enforcer.go b/internal/enforcer/enforcer.go new file mode 100644 index 0000000..6032a6d --- /dev/null +++ b/internal/enforcer/enforcer.go @@ -0,0 +1,139 @@ +// Package enforcer terminates blocked connections, preferring to kill just the one +// connection over the whole owning process. +// +// KillConnection is the preferred entry point: it tries a precise, connection-level +// kill via `ss -K` (requires CONFIG_INET_DIAG_DESTROY and CAP_NET_ADMIN) and falls +// back to killing the whole process — KillProcess's behavior — when that isn't +// possible: ss is missing, the kernel doesn't support it, or hexwall isn't running +// with the needed capability. The process-level fallback remains coarser than +// intended: it takes down every other connection that process holds too. +package enforcer + +import ( + "errors" + "fmt" + "log/slog" + "os" + "strconv" + "strings" +) + +// protectedPrograms is the default set of programs that must never be killed. +// All keys MUST be lowercase: isProtected lowercases the input before lookup. +var protectedPrograms = map[string]bool{ + "systemd": true, + "init": true, + "sshd": true, + "networkmanager": true, + "systemd-resolved": true, + "systemd-networkd": true, + "dbus-daemon": true, + "containerd": true, + "dockerd": true, +} + +// SetProtectedPrograms replaces the default protected-program set. +// Keys are normalized to lowercase on entry because isProtected lowercases +// its input, so stored keys must match for the lookup to work. +func SetProtectedPrograms(programs map[string]bool) { + normalized := make(map[string]bool, len(programs)) + for k, v := range programs { + normalized[strings.ToLower(k)] = v + } + + protectedPrograms = normalized +} + +// isProtected reports whether the program name (case-insensitive) is in the protected set. +func isProtected(name string) bool { + lower := strings.ToLower(name) + return protectedPrograms[lower] +} + +// ErrRefused marks a kill the guardrails declined on purpose, as opposed to one that was +// attempted and failed. Test for it with errors.Is. +var ErrRefused = errors.New("kill refused by guardrail") + +// checkGuardrails applies the safety checks shared by both kill paths below, +// before either one is attempted. Every refusal wraps ErrRefused: +// - Refuses any PID <= 0. +// - Refuses PID 1 (init). +// - Refuses hexwall's own PID. +// - Refuses protected programs (case-insensitive match). +func checkGuardrails(pid int, program string) error { + if pid <= 0 { + return fmt.Errorf("%w: invalid PID %d", ErrRefused, pid) + } + + if pid == 1 { + return fmt.Errorf("%w: refusing to signal init (PID 1)", ErrRefused) + } + + if pid == os.Getpid() { + return fmt.Errorf("%w: refusing to signal own process (PID %d)", ErrRefused, pid) + } + + if isProtected(program) { + return fmt.Errorf("%w: refusing to kill protected program %q (PID %d)", ErrRefused, program, pid) + } + + return nil +} + +// KillProcess terminates the process holding a connection. +// +// Guardrails: see checkGuardrails. +// Sends SIGTERM first, waits a grace period, then SIGKILL if still alive. +// The actual signal-sending is platform-specific; see terminate in +// enforcer_unix.go / enforcer_windows.go. +func KillProcess(pid int, program string, remoteAddr string) error { + if err := checkGuardrails(pid, program); err != nil { + return err + } + + slog.Warn("killing connection", "pid", pid, "program", program, "address", remoteAddr) + + return terminate(pid, program) +} + +// KillConnection kills the single connection identified by localPort/remoteIP/remotePort, +// owned by pid/program. It prefers a precise, connection-level kill (see killSocket in +// enforcer_socket.go) that leaves every other connection the process holds untouched, +// falling back to killing the whole process only when that isn't possible. +// +// Guardrails are the same as KillProcess's, checked once before either kill path +// is attempted. +func KillConnection(pid int, program, localPort, remoteIP, remotePort string) error { + if err := checkGuardrails(pid, program); err != nil { + return err + } + + remoteAddr := remoteIP + ":" + remotePort + + socketErr := killSocket(localPort, remoteIP, remotePort) + if socketErr == nil { + slog.Info("killed single connection", "pid", pid, "program", program, "local_port", localPort, "remote", remoteAddr) + return nil + } + + slog.Debug("connection-level kill unavailable, falling back to killing the process", "err", socketErr, "pid", pid, "program", program) + + slog.Warn("killing connection", "pid", pid, "program", program, "address", remoteAddr) + + return terminate(pid, program) +} + +// ParsePID converts a string PID to int, returning an error for invalid input. +func ParsePID(s string) (int, error) { + s = strings.TrimSpace(s) + if s == "" || s == "-" { + return 0, fmt.Errorf("empty or dash PID %q", s) + } + + pid, err := strconv.Atoi(s) + if err != nil { + return 0, fmt.Errorf("invalid PID %q: %w", s, err) + } + + return pid, nil +} diff --git a/internal/enforcer/enforcer_socket.go b/internal/enforcer/enforcer_socket.go new file mode 100644 index 0000000..2917f8b --- /dev/null +++ b/internal/enforcer/enforcer_socket.go @@ -0,0 +1,61 @@ +package enforcer + +import ( + "fmt" + "os/exec" + "strings" +) + +// killSocket attempts to close exactly one TCP connection — identified by its +// local port and the remote IP:port it is talking to — via `ss -K`, which asks +// the kernel to destroy that single socket (CONFIG_INET_DIAG_DESTROY) without +// touching any other connection the owning process holds. +// +// ss -K's own exit code is not a reliable success signal: it has been observed +// to exit 0 even when the kernel refused the destroy request for lack of +// CAP_NET_ADMIN. So after running it, the socket table is re-queried for this +// exact tuple; if it is still there, the kill is treated as failed regardless +// of what ss itself reported. +// +// This needs no build tag: ss is a Linux-only tool (iproute2), but on any +// platform or environment where it is missing, unsupported, or unprivileged, +// this simply returns an error, and the caller falls back to killing the +// whole process. +func killSocket(localPort, remoteIP, remotePort string) error { + // Best-effort: its exit code is not trusted. The recheck below is the + // actual source of truth. + _ = exec.Command("ss", "-K", "dst", remoteIP, "dport", remotePort, "sport", localPort).Run() + + stillEstablished, err := socketStillEstablished(localPort, remoteIP, remotePort) + if err != nil { + return fmt.Errorf("could not verify socket state after ss -K: %w", err) + } + + if stillEstablished { + return fmt.Errorf("ss -K did not close the connection (missing CAP_NET_ADMIN, unsupported kernel, or ss unavailable)") + } + + return nil +} + +// socketStillEstablished reports whether a TCP connection matching this exact +// local-port/remote-IP/remote-port tuple is still in the ESTABLISHED state. +func socketStillEstablished(localPort, remoteIP, remotePort string) (bool, error) { + filter := fmt.Sprintf("( sport = :%s and dst %s and dport = %s )", localPort, remoteIP, remotePort) + + out, err := exec.Command("ss", "-tn", "state", "established", filter).Output() + if err != nil { + return false, fmt.Errorf("ss state query failed: %w", err) + } + + dataLines := 0 + + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if strings.TrimSpace(line) != "" { + dataLines++ + } + } + + // The first line is always the column header; anything past that is a match. + return dataLines > 1, nil +} diff --git a/internal/enforcer/enforcer_socket_test.go b/internal/enforcer/enforcer_socket_test.go new file mode 100644 index 0000000..c846123 --- /dev/null +++ b/internal/enforcer/enforcer_socket_test.go @@ -0,0 +1,251 @@ +package enforcer + +import ( + "errors" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// installFakeSS writes an executable shell script named "ss" into a temp +// directory and prepends it to PATH, mirroring the same trick used for somo +// and docker elsewhere in this project. +func installFakeSS(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + + path := filepath.Join(dir, "ss") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake ss: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +// ---- killSocket, via a fake ss on PATH ---------------------------------- + +func TestKillSocketSucceedsWhenConnectionGoneAfterward(t *testing.T) { + installFakeSS(t, "#!/bin/sh\n"+ + "case \"$1\" in\n"+ + " -K) exit 0 ;;\n"+ + " -tn) echo \"Recv-Q Send-Q Local Address:Port Peer Address:Port\" ;;\n"+ // header only: gone + "esac\n") + + if err := killSocket("51820", "203.0.113.10", "443"); err != nil { + t.Fatalf("killSocket() error = %v, want nil when the recheck shows the socket gone", err) + } +} + +func TestKillSocketFailsWhenConnectionStillThere(t *testing.T) { + installFakeSS(t, "#!/bin/sh\n"+ + "case \"$1\" in\n"+ + " -K) exit 0 ;;\n"+ // ss -K can exit 0 even when it changed nothing (e.g. EPERM) + " -tn) printf \"Recv-Q Send-Q Local Address:Port Peer Address:Port\\n0 0 10.0.0.5:51820 203.0.113.10:443\\n\" ;;\n"+ + "esac\n") + + err := killSocket("51820", "203.0.113.10", "443") + if err == nil { + t.Fatal("killSocket() error = nil, want error when the recheck still shows the connection") + } + + if !strings.Contains(err.Error(), "did not close") { + t.Fatalf("killSocket() error = %q, want it to mention the connection was not closed", err) + } +} + +func TestKillSocketFailsWhenSSUnavailable(t *testing.T) { + // Empty PATH: exec.Command("ss", ...) can't find the binary at all, + // simulating a platform (e.g. macOS) where ss doesn't exist. + t.Setenv("PATH", t.TempDir()) + + err := killSocket("51820", "203.0.113.10", "443") + if err == nil { + t.Fatal("killSocket() error = nil, want error when ss is not on PATH") + } +} + +// ---- socketStillEstablished, against a real loopback connection --------- + +func requireSS(t *testing.T) { + t.Helper() + + if _, err := exec.LookPath("ss"); err != nil { + t.Skip("ss not available in this environment") + } +} + +// openLoopbackConnection sets up a real, live TCP connection over loopback +// and returns the client side plus its own and the server's port, so +// socketStillEstablished can be checked against a socket that genuinely +// exists rather than a fabricated tuple. +func openLoopbackConnection(t *testing.T) (client net.Conn, clientPort, serverPort string) { + t.Helper() + + ln, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen() error = %v", err) + } + + t.Cleanup(func() { _ = ln.Close() }) + + accepted := make(chan net.Conn, 1) + + go func() { + c, err := ln.Accept() + if err == nil { + accepted <- c + } + }() + + conn, err := net.Dial("tcp4", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial() error = %v", err) + } + + t.Cleanup(func() { _ = conn.Close() }) + + server := <-accepted + + t.Cleanup(func() { _ = server.Close() }) + + local := conn.LocalAddr().(*net.TCPAddr) + remote := conn.RemoteAddr().(*net.TCPAddr) + + return conn, strconv.Itoa(local.Port), strconv.Itoa(remote.Port) +} + +func TestSocketStillEstablishedTrueForLiveConnection(t *testing.T) { + requireSS(t) + + _, clientPort, serverPort := openLoopbackConnection(t) + + still, err := socketStillEstablished(clientPort, "127.0.0.1", serverPort) + if err != nil { + t.Fatalf("socketStillEstablished() error = %v", err) + } + + if !still { + t.Fatal("socketStillEstablished() = false, want true for a live connection") + } +} + +func TestSocketStillEstablishedFalseAfterClose(t *testing.T) { + requireSS(t) + + conn, clientPort, serverPort := openLoopbackConnection(t) + if err := conn.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + // Closing the local side takes the socket out of ESTABLISHED essentially + // immediately, but allow a couple of short retries to avoid flakiness. + var ( + still bool + err error + ) + for i := 0; i < 10; i++ { + still, err = socketStillEstablished(clientPort, "127.0.0.1", serverPort) + if err != nil { + t.Fatalf("socketStillEstablished() error = %v", err) + } + + if !still { + break + } + + time.Sleep(20 * time.Millisecond) + } + + if still { + t.Fatal("socketStillEstablished() = true, want false after the connection was closed") + } +} + +func TestSocketStillEstablishedNoMatch(t *testing.T) { + requireSS(t) + + still, err := socketStillEstablished("1", "203.0.113.99", "1") + if err != nil { + t.Fatalf("socketStillEstablished() error = %v", err) + } + + if still { + t.Fatal("socketStillEstablished() = true, want false for a tuple that was never connected") + } +} + +// ---- KillConnection: guardrails, then ss -K, then the process fallback ---- + +// noSuchPID is above the kernel's pid_max (at most 2^22), so no process can have it: if a +// test ever reaches the process fallback, the signal fails with ESRCH instead of hitting +// a real process. +const noSuchPID = 1 << 30 + +// installRecordingSS installs a fake ss that appends each invocation to a log file and +// answers the recheck with stillThere's connection or with the header alone. +func installRecordingSS(t *testing.T, stillThere bool) (callLog string) { + t.Helper() + callLog = filepath.Join(t.TempDir(), "ss-calls") + + answer := `echo "Recv-Q Send-Q Local Address:Port Peer Address:Port"` + if stillThere { + answer = `printf "Recv-Q Send-Q Local Address:Port Peer Address:Port\n0 0 10.0.0.5:51820 203.0.113.10:443\n"` + } + + installFakeSS(t, "#!/bin/sh\n"+ + "echo \"$@\" >> '"+callLog+"'\n"+ + "case \"$1\" in\n"+ + " -K) exit 0 ;;\n"+ + " -tn) "+answer+" ;;\n"+ + "esac\n") + + return callLog +} + +func TestKillConnectionClosesJustTheSocket(t *testing.T) { + callLog := installRecordingSS(t, false) + + if err := KillConnection(noSuchPID, "curl", "51820", "203.0.113.10", "443"); err != nil { + t.Fatalf("KillConnection() error = %v, want nil: ss -K closed the socket, so no process signal was needed", err) + } + + calls, _ := os.ReadFile(callLog) + if !strings.Contains(string(calls), "-K dst 203.0.113.10 dport 443 sport 51820") { + t.Fatalf("ss calls = %q, want an ss -K for exactly this connection", calls) + } +} + +// When ss -K can't close the socket, KillConnection falls back to signalling the process. +// noSuchPID makes that signal fail harmlessly; the error proves the fallback ran and is +// a failure, not a guardrail refusal. +func TestKillConnectionFallsBackToProcessKill(t *testing.T) { + installRecordingSS(t, true) + + err := KillConnection(noSuchPID, "curl", "51820", "203.0.113.10", "443") + if err == nil || !strings.Contains(err.Error(), "SIGTERM") { + t.Fatalf("KillConnection() error = %v, want the SIGTERM fallback's error", err) + } + + if errors.Is(err, ErrRefused) { + t.Fatalf("KillConnection() error = %v is ErrRefused, want a plain failure", err) + } +} + +// Guardrails run before anything else: a refused kill never even calls ss. +func TestKillConnectionRefusedBeforeTouchingSS(t *testing.T) { + callLog := installRecordingSS(t, false) + + if err := KillConnection(1, "curl", "51820", "203.0.113.10", "443"); !errors.Is(err, ErrRefused) { + t.Fatalf("KillConnection(PID 1) error = %v, want ErrRefused", err) + } + + if _, err := os.Stat(callLog); !os.IsNotExist(err) { + t.Fatal("ss was called for a refused kill; guardrails must run first") + } +} diff --git a/internal/enforcer/enforcer_test.go b/internal/enforcer/enforcer_test.go new file mode 100644 index 0000000..b4871fb --- /dev/null +++ b/internal/enforcer/enforcer_test.go @@ -0,0 +1,229 @@ +package enforcer + +import ( + "errors" + "os" + "strings" + "testing" +) + +func TestKillProcessRefusesPIDZero(t *testing.T) { + t.Parallel() + + err := KillProcess(0, "curl", "1.2.3.4:443") + if err == nil { + t.Fatal("KillProcess(0) = nil, want error") + } +} + +func TestKillProcessRefusesNegativePID(t *testing.T) { + t.Parallel() + + err := KillProcess(-1, "curl", "1.2.3.4:443") + if err == nil { + t.Fatal("KillProcess(-1) = nil, want error") + } +} + +func TestKillProcessRefusesPIDOne(t *testing.T) { + t.Parallel() + + err := KillProcess(1, "systemd", "1.2.3.4:443") + if err == nil { + t.Fatal("KillProcess(1) = nil, want error for init") + } +} + +func TestKillProcessRefusesOwnPID(t *testing.T) { + t.Parallel() + + err := KillProcess(os.Getpid(), "test", "1.2.3.4:443") + if err == nil { + t.Fatal("KillProcess(own PID) = nil, want error") + } +} + +func TestKillProcessRefusesProtectedPrograms(t *testing.T) { + t.Parallel() + + protected := []string{ + "systemd", "init", "sshd", "NetworkManager", + "systemd-resolved", "systemd-networkd", "dbus-daemon", + "containerd", "dockerd", + } + + for _, prog := range protected { + // Use a non-init, non-own PID that we know exists won't be matched + // by the other guards. PID 2 is usually kthreadd on Linux. + err := KillProcess(2, prog, "1.2.3.4:443") + if err == nil { + t.Errorf("KillProcess(2, %q) = nil, want error for protected program", prog) + continue + } + + if !strings.Contains(err.Error(), "protected") { + t.Errorf("KillProcess(2, %q) error = %q, want it to mention 'protected'", prog, err) + } + } +} + +func TestKillProcessCaseInsensitiveProtectedMatch(t *testing.T) { + t.Parallel() + + cases := []string{"SSHD", "SystemD", "NETWORKMANAGER", "containerd", "DockerD"} + for _, prog := range cases { + err := KillProcess(2, prog, "1.2.3.4:443") + if err == nil { + t.Errorf("KillProcess(2, %q) = nil, want error for case-insensitive protected match", prog) + continue + } + + if !strings.Contains(err.Error(), "protected") { + t.Errorf("KillProcess(2, %q) error = %q, want it to mention 'protected'", prog, err) + } + } +} + +// ---- KillConnection guardrails (same checkGuardrails as KillProcess) ----- + +func TestKillConnectionRefusesPIDZero(t *testing.T) { + t.Parallel() + + err := KillConnection(0, "curl", "51820", "1.2.3.4", "443") + if err == nil { + t.Fatal("KillConnection(0) = nil, want error") + } +} + +func TestKillConnectionRefusesNegativePID(t *testing.T) { + t.Parallel() + + err := KillConnection(-1, "curl", "51820", "1.2.3.4", "443") + if err == nil { + t.Fatal("KillConnection(-1) = nil, want error") + } +} + +func TestKillConnectionRefusesPIDOne(t *testing.T) { + t.Parallel() + + err := KillConnection(1, "systemd", "51820", "1.2.3.4", "443") + if err == nil { + t.Fatal("KillConnection(1) = nil, want error for init") + } +} + +func TestKillConnectionRefusesOwnPID(t *testing.T) { + t.Parallel() + + err := KillConnection(os.Getpid(), "test", "51820", "1.2.3.4", "443") + if err == nil { + t.Fatal("KillConnection(own PID) = nil, want error") + } +} + +func TestKillConnectionRefusesProtectedPrograms(t *testing.T) { + t.Parallel() + + protected := []string{"sshd", "dockerd", "systemd-resolved"} + for _, prog := range protected { + err := KillConnection(2, prog, "51820", "1.2.3.4", "443") + if err == nil { + t.Errorf("KillConnection(2, %q) = nil, want error for protected program", prog) + continue + } + + if !strings.Contains(err.Error(), "protected") { + t.Errorf("KillConnection(2, %q) error = %q, want it to mention 'protected'", prog, err) + } + } +} + +func TestParsePIDValid(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want int + }{ + {"1234", 1234}, + {" 5678 ", 5678}, + {"0", 0}, + {"1", 1}, + } + + for _, tt := range tests { + got, err := ParsePID(tt.input) + if err != nil { + t.Errorf("ParsePID(%q) error = %v", tt.input, err) + } + + if got != tt.want { + t.Errorf("ParsePID(%q) = %d, want %d", tt.input, got, tt.want) + } + } +} + +func TestParsePIDInvalid(t *testing.T) { + t.Parallel() + + tests := []string{"", "-", "abc", "12.34", "12ab"} + for _, input := range tests { + _, err := ParsePID(input) + if err == nil { + t.Errorf("ParsePID(%q) = nil, want error", input) + } + } +} + +func TestIsProtected(t *testing.T) { + t.Parallel() + + if !isProtected("sshd") { + t.Error("isProtected(\"sshd\") = false, want true") + } + + if !isProtected("SSHD") { + t.Error("isProtected(\"SSHD\") = false, want true (case-insensitive)") + } + + if !isProtected("SystemD") { + t.Error("isProtected(\"SystemD\") = false, want true (case-insensitive)") + } + + if isProtected("curl") { + t.Error("isProtected(\"curl\") = true, want false") + } + + if isProtected("") { + t.Error("isProtected(\"\") = true, want false") + } +} + +func TestIsProtectedMixedCaseAllEntries(t *testing.T) { + t.Parallel() + + for name := range protectedPrograms { + // Construct a mixed-case variant: uppercase first char + lowercase rest. + mixed := strings.ToUpper(name[:1]) + name[1:] + if !isProtected(mixed) { + t.Errorf("isProtected(%q) = false, want true for mixed-case spelling of protected program", mixed) + } + } +} + +// Callers tell "declined on purpose" from "tried and failed" with errors.Is(err, ErrRefused). +func TestGuardrailRefusalsWrapErrRefused(t *testing.T) { + for _, tc := range []struct { + pid int + program string + }{{0, "curl"}, {1, "curl"}, {os.Getpid(), "hexwall"}, {4242, "sshd"}} { + if err := checkGuardrails(tc.pid, tc.program); !errors.Is(err, ErrRefused) { + t.Errorf("checkGuardrails(%d, %q) = %v, want an ErrRefused", tc.pid, tc.program, err) + } + } + + if err := checkGuardrails(4242, "curl"); err != nil { + t.Errorf("checkGuardrails(4242, curl) = %v, want nil", err) + } +} diff --git a/internal/enforcer/enforcer_unix.go b/internal/enforcer/enforcer_unix.go new file mode 100644 index 0000000..e916937 --- /dev/null +++ b/internal/enforcer/enforcer_unix.go @@ -0,0 +1,38 @@ +//go:build !windows + +package enforcer + +import ( + "fmt" + "log/slog" + "syscall" + "time" +) + +// terminate sends SIGTERM, waits a grace period, then escalates to SIGKILL +// if the process is still alive. +func terminate(pid int, program string) error { + // SIGTERM first — allows graceful shutdown. + if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { + return fmt.Errorf("send SIGTERM to PID %d: %w", pid, err) + } + + // Grace period: wait up to 2 seconds for the process to exit. + time.Sleep(2 * time.Second) + + // Check whether the process still exists. + if err := syscall.Kill(pid, 0); err != nil { + // Process is gone — SIGTERM worked. + slog.Info("process terminated by SIGTERM", "pid", pid, "program", program) + return nil + } + + // Process still alive — escalate to SIGKILL. + slog.Warn("process survived SIGTERM, sending SIGKILL", "pid", pid, "program", program) + + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + return fmt.Errorf("send SIGKILL to PID %d: %w", pid, err) + } + + return nil +} diff --git a/internal/enforcer/enforcer_windows.go b/internal/enforcer/enforcer_windows.go new file mode 100644 index 0000000..441d6a4 --- /dev/null +++ b/internal/enforcer/enforcer_windows.go @@ -0,0 +1,13 @@ +//go:build windows + +package enforcer + +import "fmt" + +// terminate is unimplemented on Windows: the SIGTERM-then-SIGKILL grace +// period this package relies on is a POSIX signal semantic that Windows does +// not have. hexwall does not support Windows — CI no longer builds or tests +// this file, so treat it as an unmaintained stub, not a courtesy compile target. +func terminate(pid int, program string) error { + return fmt.Errorf("killing processes is not supported on windows (pid %d, program %q)", pid, program) +} diff --git a/internal/monitor/ladder_test.go b/internal/monitor/ladder_test.go new file mode 100644 index 0000000..900c115 --- /dev/null +++ b/internal/monitor/ladder_test.go @@ -0,0 +1,229 @@ +package monitor + +import ( + "context" + "database/sql" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/hexbytedev/hexwall/internal/connlist" + "github.com/hexbytedev/hexwall/internal/deghost" + "github.com/hexbytedev/hexwall/internal/store" +) + +// These tests walk Run's decision ladder rung by rung for a connection Zeek saw an SNI +// for. Each rung leaves its verdict in sni_observations.outcome; a kill leaves a row in +// killed_connections (PID 1, so the guardrails refuse it and no real process is touched). + +// fakeSNI is an SNISource with fixed answers, keyed like zeek.Client: "lport|ip|rport". +type fakeSNI map[string]string + +func (f fakeSNI) Lookup(localPort, remoteIP, remotePort string) (string, bool) { + sni, ok := f[localPort+"|"+remoteIP+"|"+remotePort] + return sni, ok +} + +const ladderIP = "203.0.113.50" + +// ladderScan runs one enforce-mode scan of a single connection to ladderIP whose SNI is sni. +func ladderScan(t *testing.T, scanner *Scanner, sni string) { + t.Helper() + + conn := connlist.Connection{LPort: "40000", RAddress: ladderIP + ":443", RPort: "443", PID: "1", Program: "curl"} + scanner.Mode = ModeEnforce + scanner.ConnectionLister = func() ([]connlist.Connection, error) { return []connlist.Connection{conn}, nil } + scanner.ZeekClient = fakeSNI{"40000|" + ladderIP + "|443": sni} + scanner.Run(context.Background()) +} + +func sniOutcome(t *testing.T, dbPath, domain string) string { + t.Helper() + + db, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro") + if err != nil { + t.Fatal(err) + } + + defer func() { _ = db.Close() }() + + var outcome string + if err := db.QueryRow(`SELECT outcome FROM sni_observations WHERE ip = ? AND domain = ?`, ladderIP, domain).Scan(&outcome); err != nil { + t.Fatalf("no sni_observations row for %s: %v", domain, err) + } + + return outcome +} + +func killRows(t *testing.T, dbPath string) int { + t.Helper() + return queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, ladderIP) +} + +// Rung 3 outranks rung 4: an allowlisted domain is trusted even though gravity has it. +func TestRunAllowlistedSNIIsTrusted(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + if _, err := db.Exec(`INSERT INTO gravity (domain) VALUES ('ads.example.com')`); err != nil { + t.Fatal(err) + } + + insertDomain(t, db, "ads.example.com", 0) + }) + hexwallStore, dbPath := newTestStore(t) + + ladderScan(t, &Scanner{Checker: checker, Store: hexwallStore}, "ads.example.com") + + if got := sniOutcome(t, dbPath, "ads.example.com"); got != string(store.OutcomePolicyAllowed) { + t.Fatalf("outcome = %q, want %q", got, store.OutcomePolicyAllowed) + } + + if got := killRows(t, dbPath); got != 0 { + t.Fatalf("kill rows = %d, want 0", got) + } +} + +// Rung 4, for each way Pi-hole can block a domain: exact denylist, gravity, regex. +func TestRunBlockedSNIIsKilled(t *testing.T) { + for _, tc := range []struct { + name string + sni string + setup func(t *testing.T, db *sql.DB) + }{ + {"denylist", "evil.example.com", func(t *testing.T, db *sql.DB) { insertDomain(t, db, "evil.example.com", 1) }}, + {"gravity", "ads.example.com", func(t *testing.T, db *sql.DB) { + if _, err := db.Exec(`INSERT INTO gravity (domain) VALUES ('ads.example.com')`); err != nil { + t.Fatal(err) + } + }}, + {"regex", "track.example.com", func(t *testing.T, db *sql.DB) { insertDomain(t, db, `^track\.`, 3) }}, + } { + t.Run(tc.name, func(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { tc.setup(t, db) }) + hexwallStore, dbPath := newTestStore(t) + + ladderScan(t, &Scanner{Checker: checker, Store: hexwallStore}, tc.sni) + + if got := sniOutcome(t, dbPath, tc.sni); got != string(store.OutcomePolicyBlocked) { + t.Fatalf("outcome = %q, want %q", got, store.OutcomePolicyBlocked) + } + + if got := killRows(t, dbPath); got != 1 { + t.Fatalf("kill rows = %d, want 1", got) + } + }) + } +} + +// Rung 5: a domain Pi-hole resolved to this very IP is trusted. +func TestRunSNIKnownForThisIPIsTrusted(t *testing.T) { + checker := newTestChecker(t, nil) + + hexwallStore, dbPath := newTestStore(t) + if err := hexwallStore.UpsertAllowedIPDomain(ladderIP, "cdn.example.com"); err != nil { + t.Fatal(err) + } + + ladderScan(t, &Scanner{Checker: checker, Store: hexwallStore}, "www.cdn.example.com") + + if got := sniOutcome(t, dbPath, "www.cdn.example.com"); got != string(store.OutcomeIPDomainMatch) { + t.Fatalf("outcome = %q, want %q", got, store.OutcomeIPDomainMatch) + } + + if got := killRows(t, dbPath); got != 0 { + t.Fatalf("kill rows = %d, want 0", got) + } +} + +// Rung 6 with deghost off (the default): an unknown domain is recorded, never killed, +// and no data leaves the machine. +func TestRunUnknownSNIWithoutDeghostIsNotKilled(t *testing.T) { + checker := newTestChecker(t, nil) + hexwallStore, dbPath := newTestStore(t) + client := deghost.NewClient(failIfCalled(t).URL, time.Second) + + ladderScan(t, &Scanner{Checker: checker, Store: hexwallStore, DeghostClient: client}, "new.example.com") + + if got := sniOutcome(t, dbPath, "new.example.com"); got != string(store.OutcomeUnknown) { + t.Fatalf("outcome = %q, want %q", got, store.OutcomeUnknown) + } + + if got := killRows(t, dbPath); got != 0 { + t.Fatalf("kill rows = %d, want 0", got) + } +} + +// Rung 6 with deghost on: a blacklisted verdict kills, and is cached so the next scan +// doesn't ask the API again. +func TestRunUnknownSNIDeghostBlockedIsKilledAndCached(t *testing.T) { + var calls atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + + _, _ = w.Write([]byte(`{"status":"allowed","blacklisted":true}`)) + })) + t.Cleanup(srv.Close) + + checker := newTestChecker(t, nil) + hexwallStore, dbPath := newTestStore(t) + scanner := &Scanner{Checker: checker, Store: hexwallStore, DeghostClient: deghost.NewClient(srv.URL, time.Second), DeghostDomain: true} + + ladderScan(t, scanner, "bad.example.com") + ladderScan(t, scanner, "bad.example.com") + + if got := killRows(t, dbPath); got != 2 { + t.Fatalf("kill rows = %d, want 2 (one per scan)", got) + } + + if got := calls.Load(); got != 1 { + t.Fatalf("deghost called %d times, want 1: the second scan must use the cached verdict", got) + } +} + +// Rung 2 → 7: no SNI for the connection means the IP-only path decides, and no SNI +// verdict is recorded. +func TestRunWithoutSNIUsesIPOnlyPath(t *testing.T) { + checker := newTestChecker(t, nil) + hexwallStore, dbPath := newTestStore(t) + + conn := connlist.Connection{LPort: "40000", RAddress: ladderIP + ":443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, ZeekClient: fakeSNI{}, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{conn}, nil }, + } + scanner.Run(context.Background()) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM sni_observations`); got != 0 { + t.Fatalf("sni_observations rows = %d, want 0", got) + } + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, ladderIP); got != 1 { + t.Fatalf("ip_observations rows = %d, want 1: the IP-only path must have run", got) + } +} + +// Item 8: an allow pattern Go can't compile might have allowed this domain, so a +// reputation verdict alone must not kill it. `(a)\1` (a back-reference) is valid in +// Pi-hole's regex flavour but not in Go's. +func TestRunDeghostBlockNotKilledWhenAllowPatternUnreadable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"status":"allowed","blacklisted":true}`)) + })) + t.Cleanup(srv.Close) + + checker := newTestChecker(t, func(db *sql.DB) { insertDomain(t, db, `(a)\1`, 2) }) + if checker.AllowListComplete() { + t.Fatal("AllowListComplete() = true, want false with an unreadable allow pattern") + } + + hexwallStore, dbPath := newTestStore(t) + scanner := &Scanner{Checker: checker, Store: hexwallStore, DeghostClient: deghost.NewClient(srv.URL, time.Second), DeghostDomain: true} + + ladderScan(t, scanner, "bad.example.com") + + if got := killRows(t, dbPath); got != 0 { + t.Fatalf("kill rows = %d, want 0: the user may have allowed this domain", got) + } +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 74265e9..c7fa8af 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -4,6 +4,7 @@ package monitor import ( "context" + "errors" "fmt" "log/slog" "net" @@ -11,9 +12,12 @@ import ( "time" "github.com/hexbytedev/hexwall/internal/allowlist" + "github.com/hexbytedev/hexwall/internal/connlist" "github.com/hexbytedev/hexwall/internal/deghost" - "github.com/hexbytedev/hexwall/internal/somo" + "github.com/hexbytedev/hexwall/internal/enforcer" + "github.com/hexbytedev/hexwall/internal/pihole" "github.com/hexbytedev/hexwall/internal/store" + "github.com/hexbytedev/hexwall/internal/zeek" ) const ( @@ -23,6 +27,41 @@ const ( ModeEnforce = "enforce" ) +// Scanner holds the long-lived collaborators and configuration flags the +// decision ladder needs. Construct it with a struct literal — every field is +// a plain dependency or flag with nothing to validate, so there's no +// NewScanner constructor. +type Scanner struct { + Checker *pihole.Checker + Store *store.Store + DeghostClient *deghost.Client + ZeekClient SNISource + ConnectionLister func() ([]connlist.Connection, error) + Mode string + DeghostIP bool + DeghostDomain bool + Debug bool +} + +// SNISource answers rung 2: which TLS server name did this connection ask for? In +// production that's *zeek.Client, fed by Zeek's ssl.log; tests use a fixed map, so the +// SNI rungs can be exercised without a Zeek log file. +type SNISource interface { + Lookup(localPort, remoteIP, remotePort string) (sni string, ok bool) +} + +// scanSummary tracks per-scan outcome counts for the summary line. +type scanSummary struct { + connections int + allowlisted int + withSNI int + withoutSNI int + trusted int + blocked int + unknown int + errored int +} + func normalizeMode(mode string) string { switch strings.TrimSpace(strings.ToLower(mode)) { case ModeEnforce: @@ -58,33 +97,146 @@ func remoteIP(address string) (net.IP, error) { return ip, nil } -func logScanConnection(debug bool, ip, program, status string) { - if !debug { +// HandleZeekEvent evaluates a Zeek DNS-bypass notice against Pi-hole policy and persists it. +// +// A notice whose SNI is policy-blocked is handled like rung 4: the connection is killed in +// enforce mode and reported in watch mode. hexwall can only close sockets on the machine +// it runs on, so the connection is first looked up in this machine's socket list; a notice +// about another device (Zeek watching a mirrored port, say) is recorded but not acted on. +func (s *Scanner) HandleZeekEvent(event zeek.Event) { + if s.Checker == nil || s.Store == nil { return } - slog.Info("scan connection", "ip", ip, "program", program, "status", status) -} + if event.SNI == "" { + return + } -// RunScan inspects established connections and applies the selected trust and kill policy. -func RunScan(ctx context.Context, hexwallStore *store.Store, deghostClient *deghost.Client, mode string, debug bool) { - if hexwallStore == nil { - slog.Error("scan aborted: nil hexwall store") + selectedMode := normalizeMode(s.Mode) + + blocked, err := s.alertBlockedByPolicy(event) + if err != nil { + slog.Warn("zeek policy lookup failed", "sni", event.SNI, "err", err) return } - if deghostClient == nil { - slog.Error("scan aborted: nil deghost client") + + confidence := "medium" + actionTaken := "logged" + + var conn connlist.Connection + + found := false + + if blocked { + confidence = "high" + + slog.Warn("zeek bypass alert", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI, "mode", selectedMode) + + conn, found = s.findLocalConnection(event) + switch { + case !found: + actionTaken = "not_on_this_host" + + slog.Warn("zeek bypass alert: connection is not on this machine (another device, or already closed); hexwall can only kill its own machine's connections", + "src_ip", event.SrcIP, "src_port", event.SrcPort, "dst_ip", event.DstIP, "sni", event.SNI) + case selectedMode == ModeEnforce: + actionTaken = "kill_attempted" + } + } else { + slog.Info("zeek notice was not policy-blocked", "src_ip", event.SrcIP, "dst_ip", event.DstIP, "dst_port", event.DstPort, "sni", event.SNI) + } + + // Persist before acting, so the alert is on record even if the kill goes wrong. + if err := s.Store.LogZeekAlert(event.SrcIP, event.DstIP, event.DstPort, event.SNI, blocked, confidence, actionTaken); err != nil { + slog.Error("failed to persist zeek alert", "src_ip", event.SrcIP, "sni", event.SNI, "err", err) + } + + if found { + if ip, err := remoteIP(conn.RAddress); err == nil { + s.applyBlockedOutcome(ip.String(), conn) + } + } +} + +// alertBlockedByPolicy applies the ladder's policy rungs to a Zeek alert, in Run's order: +// rung 1 (local network), rung 3 (Pi-hole allowlist, which overrides gravity, as it does +// in Pi-hole itself) and only then rung 4 (denylist, gravity, regex). An allowlist lookup +// error is returned rather than read as "not allowed", so it can never lead to a kill. +func (s *Scanner) alertBlockedByPolicy(event zeek.Event) (bool, error) { + if ip := net.ParseIP(event.DstIP); ip != nil && allowlist.Contains(ip) { + return false, nil + } + + allowed, err := s.Checker.IsAllowedByPolicy(event.SNI) + if err != nil { + return false, fmt.Errorf("allowlist lookup: %w", err) + } + + if allowed { + return false, nil + } + + return s.Checker.IsBlockedByPolicy(event.SNI) +} + +// findLocalConnection looks a Zeek notice's connection up in this machine's socket list. +// Zeek names a connection from the client's side (orig port -> resp IP:port); for a +// connection this machine opened, that is its local port and remote address. +func (s *Scanner) findLocalConnection(event zeek.Event) (connlist.Connection, bool) { + wantIP := net.ParseIP(event.DstIP) + if s.ConnectionLister == nil || event.SrcPort == "" || wantIP == nil { + return connlist.Connection{}, false + } + + connections, err := s.ConnectionLister() + if err != nil { + slog.Warn("zeek bypass alert: could not list connections", "err", err) + return connlist.Connection{}, false + } + + for _, conn := range connections { + if conn.LPort != event.SrcPort || conn.RPort != event.DstPort { + continue + } + + if ip, err := remoteIP(conn.RAddress); err == nil && ip.Equal(wantIP) { + return conn, true + } + } + + return connlist.Connection{}, false +} + +// Run inspects established connections and applies the decision ladder. +// +// The ladder is ordered from cheapest/most-authoritative to most-expensive/least-authoritative: +// 1. Static CIDR allowlist (loopback, RFC1918, cloud metadata) +// 2. Zeek SNI presence — if no SNI, fall back to IP-only evaluation (step 7) +// 3. Pi-hole explicit allowlist (user intent, highest authority) +// 4. Pi-hole explicit denylist / gravity (user intent) +// 5. Inferred trust from DNS history (hexwall's own cache) +// 6. Unknown domain → deghost escalation if enabled; no action if disabled +// 7. IP-only fallback → deghost escalation if enabled; no action if disabled +// +// Safety property: with deghost disabled, only rung 4 (Pi-hole blocklist match) can kill. +func (s *Scanner) Run(ctx context.Context) { + if s.Store == nil { + slog.Error("scan aborted: nil hexwall store") return } - selectedMode := normalizeMode(mode) - if selectedMode != mode { - slog.Warn("invalid scan mode; defaulting to watch", "mode", mode, "fallback", selectedMode) + if s.ConnectionLister == nil { + slog.Warn("connection lister unset; defaulting to somo") + + s.ConnectionLister = connlist.GetEstablishedConnections } - fmt.Printf("[%s] Scanning connections (%s mode)...\n", time.Now().Format("15:04:05"), selectedMode) + selectedMode := normalizeMode(s.Mode) + if selectedMode != s.Mode { + slog.Warn("invalid scan mode; defaulting to watch", "mode", s.Mode, "fallback", selectedMode) + } - connections, err := somo.GetEstablishedConnections() + connections, err := s.ConnectionLister() if err != nil { slog.Error("error fetching connections", "err", err) return @@ -95,108 +247,386 @@ func RunScan(ctx context.Context, hexwallStore *store.Store, deghostClient *degh return } + var summary scanSummary + + summary.connections = len(connections) + for _, conn := range connections { ip, err := remoteIP(conn.RAddress) if err != nil { slog.Warn("invalid IP address", "address", conn.RAddress, "err", err) + + summary.errored++ + continue } ipStr := ip.String() - // Trust allowlisted IPs immediately. + // Rung 1: static CIDR allowlist. if allowlist.Contains(ip) { - logScanConnection(debug, ipStr, conn.Program, "allowed") + summary.allowlisted++ + summary.trusted++ + continue } - allowed, err := hexwallStore.IsAllowed(ipStr) - if err != nil { - slog.Error("store lookup failed", "address", conn.RAddress, "err", err) + // Rung 2: did Zeek see an SNI for this connection? + var sniDomain string + + sniFound := false + + if s.ZeekClient != nil { + if domain, ok := s.ZeekClient.Lookup(conn.LPort, ipStr, conn.RPort); ok { + sniDomain = domain + sniFound = true + summary.withSNI++ + } else { + summary.withoutSNI++ + } + } else { + summary.withoutSNI++ + } + + if !sniFound { + // Rung 7: IP-only fallback — no SNI means non-TLS, pre-existing connection, or ECH. + action := s.evaluateIPOnly(ctx, ipStr, conn) + applyOutcome(&summary, action) + continue } - if allowed { - logScanConnection(debug, ipStr, conn.Program, "allowed") - // Keep long-running connections trusted after their Pi-hole refresh window expires. - if err := hexwallStore.UpdateEstablished(ipStr); err != nil { - slog.Error("failed to update established", "address", conn.RAddress, "err", err) + domain := strings.ToLower(strings.TrimSpace(sniDomain)) + + // Rung 3: Pi-hole explicit allowlist — user intent outranks everything inferred. + allowed, err := s.Checker.IsAllowedByPolicy(domain) + if err != nil { + // Fail-open: if we cannot determine whether a domain is explicitly allowed, + // we must not act on a block from rung 4. Rung 4 is the only rung that can + // kill a connection, and killing a legitimate connection on the basis of + // incomplete policy data is worse than missing one block. + slog.Warn("domain allowlist lookup failed, skipping blocklist check", "domain", domain, "err", err) + } else if allowed { + if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyAllowed); logErr != nil { + slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) } + + summary.trusted++ + continue } - cachedFraudCheck, err := hexwallStore.GetRecentFraudCheck(ipStr) + // Rung 4: Pi-hole explicit denylist / gravity — only rung that can kill. + if err == nil { + if blocked, err := s.Checker.IsBlockedByPolicy(domain); err != nil { + slog.Warn("domain denylist lookup failed", "domain", domain, "err", err) + } else if blocked { + if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, store.OutcomePolicyBlocked); logErr != nil { + slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) + } + + slog.Warn("policy-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + + s.applyBlockedOutcome(ipStr, conn) + + summary.blocked++ + + continue + } + } + + // Rung 5: inferred trust from DNS history — scoped to this IP. + known := false + + knownDomains, err := s.Store.DomainsForIP(ipStr) if err != nil { - slog.Error("fraud cache lookup failed", "ip", ipStr, "program", conn.Program, "err", err) + slog.Debug("sni domain check failed", "domain", domain, "ip", ipStr, "err", err) + } else { + for _, knownDomain := range knownDomains { + if pihole.DomainMatches(domain, knownDomain) { + known = true + break + } + } + } + + outcome := store.OutcomeUnknown + if known { + outcome = store.OutcomeIPDomainMatch + } + + if logErr := s.Store.LogSNIObservation(ipStr, domain, conn.LPort, outcome); logErr != nil { + slog.Debug("failed to log sni observation", "ip", ipStr, "err", logErr) + } + + if known { + summary.trusted++ continue } - if cachedFraudCheck != nil { - if !cachedFraudCheck.ShouldKill { - slog.Info("unrecognized but clean ip", "ip", ipStr, "program", conn.Program, "reason", "cached-fraud-check") - logScanConnection(debug, ipStr, conn.Program, "unrecognized-clean") - continue + // Rung 6: unknown domain — deghost escalation if enabled. + if !s.DeghostDomain { + if s.Debug { + slog.Info("unknown domain, deghost disabled; no action", "domain", domain, "ip", ipStr, "program", conn.Program) } - logScanConnection(debug, ipStr, conn.Program, "vulnerable") - slog.Warn("vulnerable connection detected", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", "cached-fraud-check") - if selectedMode == ModeWatch { - slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + summary.unknown++ + + continue + } + + shouldBlock, reason := false, "" + + if cached, err := s.Store.GetRecentDomainCheck(domain); err != nil { + slog.Error("domain cache lookup failed", "domain", domain, "err", err) + + summary.errored++ + + continue + } else if cached != nil { + shouldBlock = cached.ShouldBlock + reason = "cached-domain-check" + } + + if !shouldBlock && reason == "" { + report, err := s.DeghostClient.CheckDomain(ctx, domain) + if err != nil { + slog.Error("deghost domain check failed", "domain", domain, "program", conn.Program, "err", err) + + summary.errored++ + continue } - if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { - slog.Error("failed to log kill", "address", conn.RAddress, "err", err) + shouldBlock = deghost.ShouldBlockDomain(report) + reason = "live-domain-check" + + if err := s.Store.UpsertDomainCheck(domain, shouldBlock); err != nil { + slog.Error("failed to cache domain check", "domain", domain, "program", conn.Program, "err", err) } - if err := somo.KillConnection(conn.PID); err != nil { - slog.Error("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) - } else { - slog.Info("killed connection", "address", conn.RAddress) + } + + if !shouldBlock { + if s.Debug { + slog.Info("unknown but clean domain", "domain", domain, "ip", ipStr, "program", conn.Program, "reason", reason) } + + summary.unknown++ + continue } - report, err := deghostClient.CheckIP(ctx, ipStr) - if err != nil { - slog.Error("deghost check failed", "ip", ipStr, "program", conn.Program, "err", err) + // Unsure whether the user allowed this domain (a Pi-hole allow pattern couldn't be + // read): report, don't kill. Same rule as a rung-3 lookup error. + if !s.Checker.AllowListComplete() { + slog.Warn("deghost-blocked domain NOT killed: a Pi-hole allow pattern couldn't be read, so it may be allowed", + "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + + summary.unknown++ + continue } - if report == nil { - if err := hexwallStore.UpsertFraudCheck(ipStr, false); err != nil { - slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) + slog.Warn("deghost-blocked domain", "domain", domain, "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "reason", reason) + + s.applyBlockedOutcome(ipStr, conn) + + summary.blocked++ + } + + fmt.Printf("[%s] Scan complete: %d connections, %d trusted, %d blocked, %d unknown (allowlisted=%d sni=%d no-sni=%d errored=%d)\n", + time.Now().Format("15:04:05"), summary.connections, summary.trusted, summary.blocked, summary.unknown, summary.allowlisted, summary.withSNI, summary.withoutSNI, summary.errored) +} + +// evaluateIPOnly handles the IP-only fallback path (rung 7) when no SNI is available. +// It returns "trusted", "blocked", "unknown", or "error". +// Every exit path records an IP observation before returning. +func (s *Scanner) evaluateIPOnly(ctx context.Context, ipStr string, conn connlist.Connection) string { + allowed, err := s.Store.IsAllowed(ipStr) + if err != nil { + slog.Error("store lookup failed", "address", conn.RAddress, "err", err) + + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } + + return "error" + } + + if allowed { + if err := s.Store.UpdateEstablished(ipStr); err != nil { + slog.Error("failed to update established", "address", conn.RAddress, "err", err) + } + + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPTrusted); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } + + return "trusted" + } + + if !s.DeghostIP { + if s.Debug { + slog.Info("unknown IP, deghost disabled; no action", "ip", ipStr, "program", conn.Program) + } + + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } + + return "unknown" + } + + cachedFraudCheck, err := s.Store.GetRecentFraudCheck(ipStr) + if err != nil { + slog.Error("fraud cache lookup failed", "ip", ipStr, "program", conn.Program, "err", err) + + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } + + return "error" + } + + if cachedFraudCheck != nil { + if !cachedFraudCheck.ShouldKill { + if s.Debug { + slog.Info("unknown but clean IP (cached)", "ip", ipStr, "program", conn.Program) } - slog.Info("unrecognized but clean ip", "ip", ipStr, "program", conn.Program, "reason", "403/private-or-reserved") - logScanConnection(debug, ipStr, conn.Program, "unrecognized-clean") - continue + + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } + + return "unknown" } - shouldKill := deghost.ShouldKill(report) - if err := hexwallStore.UpsertFraudCheck(ipStr, shouldKill); err != nil { + slog.Warn("deghost-blocked IP (cached)", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } + + s.applyBlockedOutcome(ipStr, conn) + + return "blocked" + } + + report, err := s.DeghostClient.CheckIP(ctx, ipStr) + if err != nil { + slog.Error("deghost check failed", "ip", ipStr, "program", conn.Program, "err", err) + + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } + + return "error" + } + + if report == nil { + if err := s.Store.UpsertFraudCheck(ipStr, false); err != nil { slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) } - if !shouldKill { - slog.Info("unrecognized but clean ip", "ip", ipStr, "program", conn.Program) - logScanConnection(debug, ipStr, conn.Program, "unrecognized-clean") - continue + if s.Debug { + slog.Info("unknown but clean IP (private/reserved)", "ip", ipStr, "program", conn.Program) } - logScanConnection(debug, ipStr, conn.Program, "vulnerable") + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPReserved); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } - slog.Warn("vulnerable connection detected", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - if selectedMode == ModeWatch { - slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) - continue + return "unknown" + } + + shouldKill := deghost.ShouldKill(report) + if err := s.Store.UpsertFraudCheck(ipStr, shouldKill); err != nil { + slog.Error("failed to cache fraud check", "ip", ipStr, "program", conn.Program, "err", err) + } + + if !shouldKill { + if s.Debug { + slog.Info("unknown but clean IP", "ip", ipStr, "program", conn.Program) } - if err := hexwallStore.LogKill(ipStr, conn.PID, conn.Program); err != nil { - slog.Error("failed to log kill", "address", conn.RAddress, "err", err) + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPUnknown); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) } - if err := somo.KillConnection(conn.PID); err != nil { - slog.Error("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) - } else { - slog.Info("killed connection", "address", conn.RAddress) + + return "unknown" + } + + slog.Warn("deghost-blocked IP", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + + if logErr := s.Store.LogIPObservation(ipStr, conn.Program, store.OutcomeIPBlocked); logErr != nil { + slog.Debug("failed to log ip observation", "ip", ipStr, "err", logErr) + } + + s.applyBlockedOutcome(ipStr, conn) + + return "blocked" +} + +// applyBlockedOutcome enforces the kill/watch policy for a blocked connection. +func (s *Scanner) applyBlockedOutcome(ipStr string, conn connlist.Connection) { + selectedMode := normalizeMode(s.Mode) + if selectedMode == ModeWatch { + slog.Warn("watch mode: would kill connection", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program) + return + } + + // The audit row is written before the attempt, so it exists even if the kill never + // returns, and then updated with what actually happened. + rowID, err := s.Store.LogKill(ipStr, conn.PID, conn.Program) + if err != nil { + slog.Error("failed to log kill", "address", conn.RAddress, "err", err) + } + + record := func(outcome store.KillOutcome) { + if rowID == 0 { + return + } + + if err := s.Store.SetKillOutcome(rowID, outcome); err != nil { + slog.Error("failed to record kill outcome", "address", conn.RAddress, "outcome", outcome, "err", err) } } + + pidInt, err := enforcer.ParsePID(conn.PID) + if err != nil { + slog.Warn("invalid PID, skipping kill", "pid", conn.PID, "address", conn.RAddress, "err", err) + record(store.KillFailed) + + return + } + + // Prefers a connection-level kill (ss -K) so any other connection the same + // process holds is left untouched; falls back to killing the whole process + // when that isn't possible. See enforcer.KillConnection. + err = enforcer.KillConnection(pidInt, conn.Program, conn.LPort, ipStr, conn.RPort) + switch { + case err == nil: + slog.Info("killed connection", "address", conn.RAddress) + record(store.KillKilled) + case errors.Is(err, enforcer.ErrRefused): + slog.Warn("kill refused by guardrail", "address", conn.RAddress, "pid", conn.PID, "program", conn.Program, "err", err) + record(store.KillRefused) + default: + slog.Warn("failed to kill connection", "address", conn.RAddress, "pid", conn.PID, "err", err) + record(store.KillFailed) + } +} + +// applyOutcome increments the appropriate summary counter for a given outcome string. +func applyOutcome(summary *scanSummary, outcome string) { + switch outcome { + case "trusted": + summary.trusted++ + case "blocked": + summary.blocked++ + case "unknown": + summary.unknown++ + case "error": + summary.errored++ + } } diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go new file mode 100644 index 0000000..ec3f1ab --- /dev/null +++ b/internal/monitor/monitor_test.go @@ -0,0 +1,755 @@ +package monitor + +import ( + "context" + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hexbytedev/hexwall/internal/connlist" + "github.com/hexbytedev/hexwall/internal/deghost" + "github.com/hexbytedev/hexwall/internal/pihole" + "github.com/hexbytedev/hexwall/internal/store" + "github.com/hexbytedev/hexwall/internal/zeek" + + _ "modernc.org/sqlite" +) + +// ---- fixtures -------------------------------------------------------- + +// newTestStore creates a real *store.Store backed by a temp SQLite file and +// returns it along with its path, so tests can also open a second read-only +// connection to inspect tables the Store package doesn't expose a reader for +// (e.g. killed_connections, allowed_ips.last_established). +func newTestStore(t *testing.T) (*store.Store, string) { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + s, err := store.NewStore(dbPath) + if err != nil { + t.Fatalf("store.NewStore() error = %v", err) + } + + t.Cleanup(func() { + if err := s.Close(); err != nil { + t.Errorf("Store.Close() error = %v", err) + } + }) + + return s, dbPath +} + +// queryInt opens its own read-only connection to the sqlite file at dbPath and +// scans a single integer column. Used to inspect tables that *store.Store +// doesn't expose an exported reader for. +func queryInt(t *testing.T, dbPath, query string, args ...any) int { + t.Helper() + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: dbPath, RawQuery: "mode=ro"}).String()) + if err != nil { + t.Fatalf("open %s for inspection: %v", dbPath, err) + } + defer func() { + _ = db.Close() + }() + + var got int + if err := db.QueryRow(query, args...).Scan(&got); err != nil { + t.Fatalf("query %q: %v", query, err) + } + + return got +} + +// openTestGravityDB creates a temp gravity.db with the Pi-hole v6 views monitor's +// checker relies on. Mirrors internal/pihole/policy_test.go's helper of the same +// name, duplicated here because it is unexported in its own package. +func openTestGravityDB(t *testing.T) (*sql.DB, string) { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "gravity.db") + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: dbPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open test gravity db: %v", err) + } + + schema := ` + CREATE TABLE domainlist (id INTEGER PRIMARY KEY, type INTEGER, domain TEXT, enabled INTEGER, date_added INTEGER, date_modified INTEGER, comment TEXT, groups TEXT); + CREATE TABLE gravity (id INTEGER PRIMARY KEY, domain TEXT, adlist_id INTEGER); + + CREATE VIEW vw_allowlist AS SELECT domain FROM domainlist WHERE type = 0 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_denylist AS SELECT domain FROM domainlist WHERE type = 1 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_gravity AS SELECT domain FROM gravity; + CREATE VIEW vw_regex_allowlist AS SELECT domain FROM domainlist WHERE type = 2 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_regex_denylist AS SELECT domain FROM domainlist WHERE type = 3 AND enabled = 1 AND TRIM(domain) <> ''; + ` + if _, err := db.Exec(schema); err != nil { + _ = db.Close() + + t.Fatalf("create gravity schema: %v", err) + } + + return db, dbPath +} + +// insertDomain inserts a domain into domainlist. type 0 = allowlist, 1 = denylist. +func insertDomain(t *testing.T, db *sql.DB, domain string, listType int) { + t.Helper() + + if _, err := db.Exec(`INSERT INTO domainlist (type, domain, enabled) VALUES (?, ?, 1)`, listType, domain); err != nil { + t.Fatalf("insert domain %q type %d: %v", domain, listType, err) + } +} + +// newTestChecker builds a real *pihole.Checker backed by temp SQLite files: +// an empty FTL query-history db and a gravity db seeded via gravitySetup. +func newTestChecker(t *testing.T, gravitySetup func(db *sql.DB)) *pihole.Checker { + t.Helper() + + ftlPath := filepath.Join(t.TempDir(), "pihole-FTL.db") + + ftlDB, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("create ftl db: %v", err) + } + + if _, err := ftlDB.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY, timestamp INTEGER, type INTEGER, domain TEXT, status INTEGER, ip TEXT)`); err != nil { + t.Fatalf("create ftl schema: %v", err) + } + + if err := ftlDB.Close(); err != nil { + t.Fatalf("close ftl db: %v", err) + } + + gravDB, gravPath := openTestGravityDB(t) + if gravitySetup != nil { + gravitySetup(gravDB) + } + + if err := gravDB.Close(); err != nil { + t.Fatalf("close gravity db: %v", err) + } + + checker, err := pihole.NewChecker(&pihole.Config{DBPath: ftlPath, GravityDBPath: gravPath}) + if err != nil { + t.Fatalf("pihole.NewChecker() error = %v", err) + } + + t.Cleanup(func() { + if err := checker.Close(); err != nil { + t.Errorf("Checker.Close() error = %v", err) + } + }) + + return checker +} + +// installFakeSomo writes an executable shell script named "somo" into a temp +// directory and prepends it to PATH, mirroring internal/connlist/connlist_test.go's helper. +func installFakeSomo(t *testing.T, script string) { + t.Helper() + + dir := t.TempDir() + + path := filepath.Join(dir, "somo") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fake somo: %v", err) + } + + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +// failIfCalled returns an httptest.Server that fails the test if it receives any request. +// Used to assert a code path never makes a live deghost call. +func failIfCalled(t *testing.T) *httptest.Server { + t.Helper() + + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected HTTP call to deghost: %s", r.URL.Path) + })) + t.Cleanup(srv.Close) + + return srv +} + +// ---- normalizeMode ----------------------------------------------------- + +func TestNormalizeMode(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + mode string + want string + }{ + {"enforce lowercase", "enforce", ModeEnforce}, + {"enforce uppercase", "ENFORCE", ModeEnforce}, + {"enforce padded mixed case", " Enforce ", ModeEnforce}, + {"watch", "watch", ModeWatch}, + {"empty", "", ModeWatch}, + {"garbage", "banana", ModeWatch}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := normalizeMode(tt.mode); got != tt.want { + t.Fatalf("normalizeMode(%q) = %q, want %q", tt.mode, got, tt.want) + } + }) + } +} + +// ---- remoteIP ------------------------------------------------------------ + +func TestRemoteIP(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + address string + wantIP string + wantErr bool + }{ + {"ipv4 with port", "203.0.113.10:443", "203.0.113.10", false}, + {"bare ipv4", "203.0.113.10", "203.0.113.10", false}, + {"bracketed ipv6 with port", "[::1]:443", "::1", false}, + {"bare ipv6", "::1", "::1", false}, + {"invalid host with port", "not-an-ip:443", "", true}, + {"empty", "", "", true}, + {"malformed", "not-an-ip", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ip, err := remoteIP(tt.address) + if tt.wantErr { + if err == nil { + t.Fatalf("remoteIP(%q) error = nil, want error", tt.address) + } + + return + } + + if err != nil { + t.Fatalf("remoteIP(%q) error = %v, want nil", tt.address, err) + } + + if ip.String() != tt.wantIP { + t.Fatalf("remoteIP(%q) = %q, want %q", tt.address, ip.String(), tt.wantIP) + } + }) + } +} + +// ---- applyOutcome -------------------------------------------------------- + +func TestApplyOutcome(t *testing.T) { + t.Parallel() + + tests := []struct { + outcome string + get func(s scanSummary) int + }{ + {"trusted", func(s scanSummary) int { return s.trusted }}, + {"blocked", func(s scanSummary) int { return s.blocked }}, + {"unknown", func(s scanSummary) int { return s.unknown }}, + {"error", func(s scanSummary) int { return s.errored }}, + } + + for _, tt := range tests { + t.Run(tt.outcome, func(t *testing.T) { + t.Parallel() + + var summary scanSummary + applyOutcome(&summary, tt.outcome) + + if got := tt.get(summary); got != 1 { + t.Fatalf("applyOutcome(%q) counter = %d, want 1", tt.outcome, got) + } + }) + } + + t.Run("unrecognized outcome increments nothing", func(t *testing.T) { + t.Parallel() + + var want scanSummary + + got := want + applyOutcome(&got, "garbage") + + if got != want { + t.Fatalf("applyOutcome(garbage) summary = %+v, want unchanged %+v", got, want) + } + }) +} + +// ---- HandleZeekEvent ------------------------------------------------------ + +func TestHandleZeekEventBlockedDomainPersisted(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "evil.example.com", 1) // denylist + }) + hexwallStore, _ := newTestStore(t) + + event := zeek.Event{SrcIP: "10.0.0.5", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"} + scanner := &Scanner{Checker: checker, Store: hexwallStore, Mode: ModeWatch} + scanner.HandleZeekEvent(event) + + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil { + t.Fatalf("RecentZeekAlert() error = %v", err) + } + + if alert == nil { + t.Fatal("RecentZeekAlert() = nil, want persisted alert") + } + + if !alert.Blocked { + t.Fatal("alert.Blocked = false, want true for denylisted domain") + } +} + +func TestHandleZeekEventUnblockedDomainPersisted(t *testing.T) { + checker := newTestChecker(t, nil) + hexwallStore, _ := newTestStore(t) + + event := zeek.Event{SrcIP: "10.0.0.5", DstIP: "203.0.113.20", DstPort: "443", SNI: "clean.example.com"} + scanner := &Scanner{Checker: checker, Store: hexwallStore, Mode: ModeWatch} + scanner.HandleZeekEvent(event) + + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil { + t.Fatalf("RecentZeekAlert() error = %v", err) + } + + if alert == nil { + t.Fatal("RecentZeekAlert() = nil, want persisted alert") + } + + if alert.Blocked { + t.Fatal("alert.Blocked = true, want false for domain not on denylist") + } +} + +func TestHandleZeekEventNoopCases(t *testing.T) { + t.Parallel() + checker := newTestChecker(t, nil) + hexwallStore, _ := newTestStore(t) + + // None of these should panic or write anything. + (&Scanner{Store: hexwallStore, Mode: ModeWatch}).HandleZeekEvent(zeek.Event{SNI: "example.com"}) + (&Scanner{Checker: checker, Mode: ModeWatch}).HandleZeekEvent(zeek.Event{SNI: "example.com"}) + (&Scanner{Checker: checker, Store: hexwallStore, Mode: ModeWatch}).HandleZeekEvent(zeek.Event{SrcIP: "10.0.0.1", SNI: ""}) +} + +// A blocked DNS-bypass alert for a connection this machine owns is killed in enforce mode. +// The connection's PID is 1, which the enforcer's guardrails always refuse, so no real +// process is signalled; the audit row proves the kill path was taken. +func TestHandleZeekEventEnforceKillsLocalConnection(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "evil.example.com", 1) // denylist + }) + hexwallStore, dbPath := newTestStore(t) + + local := connlist.Connection{LPort: "55881", RAddress: "203.0.113.20:443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{local}, nil }, + } + event := zeek.Event{SrcIP: "192.168.0.223", SrcPort: "55881", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"} + scanner.HandleZeekEvent(event) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.20"); got != 1 { + t.Fatalf("killed_connections count = %d, want 1", got) + } + + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil || alert == nil { + t.Fatalf("RecentZeekAlert() = %v, %v; want persisted alert", alert, err) + } + + if alert.ActionTaken != "kill_attempted" { + t.Fatalf("alert.ActionTaken = %q, want %q", alert.ActionTaken, "kill_attempted") + } +} + +// The allowlist exists to override gravity: allowlisting a domain a blocklist wrongly +// caught is its main use. Pi-hole answers such a domain, and the scan spares it at rung 3; +// a Zeek alert about it must not kill it either. +func TestHandleZeekEventAllowlistedDomainNotKilled(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + if _, err := db.Exec(`INSERT INTO gravity (domain) VALUES ('cdn.example.com')`); err != nil { + t.Fatal(err) + } + + insertDomain(t, db, "cdn.example.com", 0) // allowlist + }) + hexwallStore, dbPath := newTestStore(t) + + local := connlist.Connection{LPort: "55881", RAddress: "203.0.113.20:443", RPort: "443", PID: "1", Program: "firefox"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{local}, nil }, + } + event := zeek.Event{SrcIP: "192.168.0.223", SrcPort: "55881", DstIP: "203.0.113.20", DstPort: "443", SNI: "cdn.example.com"} + scanner.HandleZeekEvent(event) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0: the domain is allowlisted", got) + } + + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil || alert == nil { + t.Fatalf("RecentZeekAlert() = %v, %v; want the alert recorded", alert, err) + } + + if alert.Blocked { + t.Fatal("alert recorded as blocked; Pi-hole allows this domain") + } +} + +// Rung 1 applies to Zeek alerts too: a connection to the local network is never killed. +func TestHandleZeekEventLocalNetworkNotKilled(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "nas.example.com", 1) // denylist + }) + hexwallStore, dbPath := newTestStore(t) + + local := connlist.Connection{LPort: "55881", RAddress: "192.168.0.10:443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{local}, nil }, + } + scanner.HandleZeekEvent(zeek.Event{SrcIP: "192.168.0.223", SrcPort: "55881", DstIP: "192.168.0.10", DstPort: "443", SNI: "nas.example.com"}) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0: the destination is on the local network", got) + } +} + +// A connection that isn't in this machine's socket list belongs to another device (or has +// closed). hexwall cannot close a socket on another machine, so it records and warns. +func TestHandleZeekEventEnforceOtherDeviceNotKilled(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "evil.example.com", 1) + }) + hexwallStore, dbPath := newTestStore(t) + + unrelated := connlist.Connection{LPort: "40000", RAddress: "203.0.113.20:443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeEnforce, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{unrelated}, nil }, + } + event := zeek.Event{SrcIP: "192.168.0.50", SrcPort: "55881", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"} + scanner.HandleZeekEvent(event) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0", got) + } + + alert, err := hexwallStore.RecentZeekAlert(event.SrcIP, event.SNI) + if err != nil || alert == nil { + t.Fatalf("RecentZeekAlert() = %v, %v; want persisted alert", alert, err) + } + + if alert.ActionTaken != "not_on_this_host" { + t.Fatalf("alert.ActionTaken = %q, want %q", alert.ActionTaken, "not_on_this_host") + } +} + +func TestHandleZeekEventWatchModeDoesNotKill(t *testing.T) { + checker := newTestChecker(t, func(db *sql.DB) { + insertDomain(t, db, "evil.example.com", 1) + }) + hexwallStore, dbPath := newTestStore(t) + + local := connlist.Connection{LPort: "55881", RAddress: "203.0.113.20:443", RPort: "443", PID: "1", Program: "curl"} + scanner := &Scanner{ + Checker: checker, Store: hexwallStore, Mode: ModeWatch, + ConnectionLister: func() ([]connlist.Connection, error) { return []connlist.Connection{local}, nil }, + } + scanner.HandleZeekEvent(zeek.Event{SrcIP: "192.168.0.223", SrcPort: "55881", DstIP: "203.0.113.20", DstPort: "443", SNI: "evil.example.com"}) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0 in watch mode", got) + } +} + +// ---- applyBlockedOutcome --------------------------------------------------- + +func TestApplyBlockedOutcomeWatchModeNeverLogsKill(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + + conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} + (&Scanner{Store: hexwallStore, Mode: ModeWatch}).applyBlockedOutcome("203.0.113.10", conn) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections`); got != 0 { + t.Fatalf("killed_connections count = %d, want 0 in watch mode", got) + } +} + +func TestApplyBlockedOutcomeEnforceModeLogsKillBeforeAttemptingIt(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + + // PID 1 (init) is always refused by the guardrails. The audit row is still written + // first ("audit survives the kill"), but must then say the kill was refused, not done. + conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.10:443"} + (&Scanner{Store: hexwallStore, Mode: ModeEnforce}).applyBlockedOutcome("203.0.113.10", conn) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ? AND outcome = 'refused'`, "203.0.113.10"); got != 1 { + t.Fatalf("refused rows = %d, want 1: a refused kill must be recorded as refused", got) + } + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE outcome = 'killed'`); got != 0 { + t.Fatalf("killed rows = %d, want 0: nothing was killed", got) + } +} + +func TestApplyBlockedOutcomeInvalidPIDLogsWarningAndReturns(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + + conn := connlist.Connection{PID: "not-a-pid", Program: "curl", RAddress: "203.0.113.11:443"} + (&Scanner{Store: hexwallStore, Mode: ModeEnforce}).applyBlockedOutcome("203.0.113.11", conn) + + // LogKill happens before ParsePID, so the audit row exists, marked failed. + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ? AND outcome = 'failed'`, "203.0.113.11"); got != 1 { + t.Fatalf("failed rows = %d, want 1 (audit before parse failure, then marked failed)", got) + } +} + +// ---- evaluateIPOnly -------------------------------------------------------- + +func TestEvaluateIPOnlyTrustedIPUpdatesEstablished(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + if err := hexwallStore.UpsertAllowedIP("203.0.113.20", "example.com"); err != nil { + t.Fatalf("UpsertAllowedIP() error = %v", err) + } + + srv := failIfCalled(t) + client := deghost.NewClient(srv.URL, time.Second) + + conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.20:443"} + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.20", conn) + if got != "trusted" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "trusted") + } + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM allowed_ips WHERE ip = ? AND last_established IS NOT NULL`, "203.0.113.20"); got != 1 { + t.Fatal("UpdateEstablished was not applied for a trusted IP") + } +} + +func TestEvaluateIPOnlyDeghostDisabledReturnsUnknown(t *testing.T) { + hexwallStore, _ := newTestStore(t) + srv := failIfCalled(t) + client := deghost.NewClient(srv.URL, time.Second) + + conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.21:443"} + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: false, Mode: ModeWatch} + + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.21", conn) + if got != "unknown" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") + } +} + +func TestEvaluateIPOnlyCachedCleanReturnsUnknownNoHTTPCall(t *testing.T) { + hexwallStore, _ := newTestStore(t) + if err := hexwallStore.UpsertFraudCheck("203.0.113.22", false); err != nil { + t.Fatalf("UpsertFraudCheck() error = %v", err) + } + + srv := failIfCalled(t) + client := deghost.NewClient(srv.URL, time.Second) + + conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.22:443"} + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.22", conn) + if got != "unknown" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") + } +} + +func TestEvaluateIPOnlyCachedBlockedReturnsBlockedAndKills(t *testing.T) { + hexwallStore, dbPath := newTestStore(t) + if err := hexwallStore.UpsertFraudCheck("203.0.113.23", true); err != nil { + t.Fatalf("UpsertFraudCheck() error = %v", err) + } + + srv := failIfCalled(t) + client := deghost.NewClient(srv.URL, time.Second) + + // PID 1 keeps the actual kill attempt safely refused by enforcer guardrails. + conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.23:443"} + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeEnforce} + + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.23", conn) + if got != "blocked" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "blocked") + } + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.23"); got != 1 { + t.Fatal("applyBlockedOutcome was not invoked for a cached blocked verdict") + } +} + +func TestEvaluateIPOnlyLiveCheckPrivateReservedCachesClean(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + + client := deghost.NewClient(srv.URL, time.Second) + + hexwallStore, _ := newTestStore(t) + conn := connlist.Connection{Program: "curl", RAddress: "203.0.113.24:443"} + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeWatch} + + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.24", conn) + if got != "unknown" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "unknown") + } + + entry, err := hexwallStore.GetRecentFraudCheck("203.0.113.24") + if err != nil { + t.Fatalf("GetRecentFraudCheck() error = %v", err) + } + + if entry == nil { + t.Fatal("GetRecentFraudCheck() = nil, want a cached clean entry after private/reserved verdict") + } + + if entry.ShouldKill { + t.Fatal("GetRecentFraudCheck().ShouldKill = true, want false for private/reserved verdict") + } +} + +func TestEvaluateIPOnlyLiveCheckShouldKillCachesAndBlocks(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(deghost.IPReport{ + IP: "203.0.113.25", + Security: deghost.SecurityReport{IsAbuser: true}, + }) + })) + defer srv.Close() + + client := deghost.NewClient(srv.URL, time.Second) + + hexwallStore, dbPath := newTestStore(t) + conn := connlist.Connection{PID: "1", Program: "curl", RAddress: "203.0.113.25:443"} + scanner := &Scanner{Store: hexwallStore, DeghostClient: client, DeghostIP: true, Mode: ModeEnforce} + + got := scanner.evaluateIPOnly(context.Background(), "203.0.113.25", conn) + if got != "blocked" { + t.Fatalf("evaluateIPOnly() = %q, want %q", got, "blocked") + } + + entry, err := hexwallStore.GetRecentFraudCheck("203.0.113.25") + if err != nil { + t.Fatalf("GetRecentFraudCheck() error = %v", err) + } + + if entry == nil || !entry.ShouldKill { + t.Fatalf("GetRecentFraudCheck() = %+v, want cached ShouldKill = true", entry) + } + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM killed_connections WHERE ip = ?`, "203.0.113.25"); got != 1 { + t.Fatal("applyBlockedOutcome was not invoked for a live should-kill verdict") + } +} + +// NOTE: store.IsAllowed returning a hard error is not practically reachable with a +// real, healthy *store.Store — it only happens on a corrupted/closed DB handle, which +// would also break every other store call in the same scan. Not covered here; would +// require poking store internals from outside its package, which isn't a safe seam. + +// ---- Scanner.Run ------------------------------------------------------------- + +func TestRunScanZeroConnections(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho '[]'\n") + hexwallStore, _ := newTestStore(t) + + // Must not panic even with nil checker/deghost/zeek dependencies. + (&Scanner{Store: hexwallStore, Mode: ModeWatch, ConnectionLister: connlist.GetEstablishedConnections}).Run(context.Background()) +} + +func TestRunScanFallsBackToSomoWhenConnectionListerUnset(t *testing.T) { + installFakeSomo(t, "#!/bin/sh\necho '[]'\n") + hexwallStore, _ := newTestStore(t) + + // Even when a caller forgets to populate the field, Run should still work. + scanner := &Scanner{Store: hexwallStore, Mode: ModeWatch} + scanner.Run(context.Background()) + + if scanner.ConnectionLister == nil { + t.Fatal("ConnectionLister fallback is nil after Run()") + } +} + +func TestRunScanAllowlistedConnectionSkipsIPFallback(t *testing.T) { + installFakeSomo(t, `#!/bin/sh +echo '[{"proto":"tcp","local_port":"51820","remote_address":"127.0.0.1","remote_port":"443","program":"curl","pid":"1","state":"ESTABLISHED","address_type":"ipv4"}]' +`) + hexwallStore, dbPath := newTestStore(t) + + (&Scanner{Store: hexwallStore, Mode: ModeWatch, ConnectionLister: connlist.GetEstablishedConnections}).Run(context.Background()) + + // Rung 1 (static allowlist) short-circuits before the IP-only fallback, so + // no ip_observations row should exist for a loopback address. + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, "127.0.0.1"); got != 0 { + t.Fatalf("ip_observations count for allowlisted ip = %d, want 0", got) + } +} + +func TestRunScanInvalidAddressDoesNotStopScan(t *testing.T) { + installFakeSomo(t, `#!/bin/sh +echo '[{"proto":"tcp","local_port":"1","remote_address":"not-an-ip","remote_port":"443","program":"curl","pid":"1","state":"ESTABLISHED","address_type":"ipv4"},{"proto":"tcp","local_port":"2","remote_address":"203.0.113.30","remote_port":"443","program":"curl","pid":"2","state":"ESTABLISHED","address_type":"ipv4"}]' +`) + hexwallStore, dbPath := newTestStore(t) + + (&Scanner{Store: hexwallStore, Mode: ModeWatch, ConnectionLister: connlist.GetEstablishedConnections}).Run(context.Background()) + + // The second, valid connection must still be processed through the IP-only + // fallback despite the first connection's address being unparsable. + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, "203.0.113.30"); got != 1 { + t.Fatalf("ip_observations count for valid ip after invalid entry = %d, want 1", got) + } +} + +func TestRunScanUnknownIPRecordsIPObservation(t *testing.T) { + installFakeSomo(t, `#!/bin/sh +echo '[{"proto":"tcp","local_port":"1","remote_address":"203.0.113.31","remote_port":"443","program":"curl","pid":"1","state":"ESTABLISHED","address_type":"ipv4"}]' +`) + hexwallStore, dbPath := newTestStore(t) + + (&Scanner{Store: hexwallStore, Mode: ModeWatch, ConnectionLister: connlist.GetEstablishedConnections}).Run(context.Background()) + + if got := queryInt(t, dbPath, `SELECT COUNT(*) FROM ip_observations WHERE ip = ? AND outcome = 'unknown'`, "203.0.113.31"); got != 1 { + t.Fatalf("ip_observations unknown count = %d, want 1", got) + } +} + +// NOTE on rung 2/3/4 (Zeek SNI -> Pi-hole allow/deny) coverage inside Scanner.Run: +// zeek.Client has no exported constructor or seam that lets an external package +// (monitor_test.go is package monitor, not package zeek) populate its SNI cache +// without going through the real background tailer, which only polls every 5s +// (zeek.tickInterval) with no synchronous first pass. Driving that path from here +// would mean sleeping 5+ seconds per test or poking zeek's unexported fields from +// outside its own package, neither of which fits the constraints for this task. +// Rungs 3/4 themselves (pihole.Checker.IsAllowedByPolicy / IsBlockedByPolicy) are +// exercised directly by internal/pihole's own tests, and HandleZeekEvent above +// exercises the same Checker.IsBlockedByPolicy call against a real gravity db. diff --git a/internal/pihole/cache.go b/internal/pihole/cache.go index 12df0ff..3140906 100644 --- a/internal/pihole/cache.go +++ b/internal/pihole/cache.go @@ -4,7 +4,7 @@ import ( "context" "log/slog" "net" - "sort" + "slices" "sync" "time" @@ -15,60 +15,91 @@ const ( refreshLookback = time.Hour lookupTimeout = time.Second maxConcurrentLookups = 32 + + // hexwall's own lookups go through the host resolver, which may be Pi-hole, so they land + // in the query log hexwall reads. A logged query for a domain hexwall looked up within + // this margin of its lookup is treated as that lookup's echo. + ownLookupSlack = 2 * time.Second + // How long to remember a lookup. FTL writes its log to disk about once a minute, so an + // echo can show up well after the lookup itself. + ownLookupMemory = 10 * time.Minute ) -// IPCache resolves Pi-hole domains to IPs and writes them to the store. +// IPCache resolves the domains Pi-hole answered to IPs and writes them to the store. // The store owns the trusted-IP set, and this type only drives the refresh cycle. +// +// Each logged query is read once, by id, so a domain's IPs are trusted for refreshLookback +// after a client last asked for it. Re-resolving every recent domain on each refresh +// instead would, through a host that uses Pi-hole, log a new query per domain every +// refresh and keep every domain "recent" forever. type IPCache struct { checker *Checker store *store.Store + + answeredQueriesAfter func(afterID, since int64) ([]AnsweredQuery, error) + maxQueryID func() (int64, error) + lookupHost func(context.Context, string) ([]string, error) + now func() time.Time + + mu sync.Mutex + lastID int64 // highest query id already read + ownLookups map[string]time.Time // domain -> when hexwall last resolved it } // NewIPCache creates an IPCache backed by the given Checker and Store. func NewIPCache(checker *Checker, store *store.Store) *IPCache { return &IPCache{ - checker: checker, - store: store, + checker: checker, + store: store, + answeredQueriesAfter: checker.AnsweredQueriesAfter, + maxQueryID: checker.MaxQueryID, + lookupHost: net.DefaultResolver.LookupHost, + now: time.Now, } } -// Refresh queries Pi-hole for recent domains, resolves them with bounded concurrency, -// and upserts the resulting unique IPs into the store. +// Refresh reads the queries Pi-hole answered since the last refresh, resolves their +// domains with bounded concurrency, and upserts the resulting unique IPs into the store. func (c *IPCache) Refresh(ctx context.Context) { if ctx.Err() != nil { return } - since := time.Now().Add(-refreshLookback).Unix() + c.mu.Lock() + defer c.mu.Unlock() - domains, err := c.checker.DomainsSeenSince(since) + domains, err := c.newClientDomains() if err != nil { slog.Error("cache refresh: failed to query pi-hole domains", "err", err) return } + if len(domains) == 0 { - slog.Info("cache refreshed", "domains", 0, "ips", 0) + slog.Debug("cache refreshed", "domains", 0, "ips", 0) return } - sort.Strings(domains) - - type result struct { - domain string - ips []string - } - results := make(chan result, maxConcurrentLookups) sem := make(chan struct{}, maxConcurrentLookups) + // Workers write straight into resolved under mu. Using a shared map instead of a + // results channel keeps a worker from ever blocking on a consumer that has not + // started yet, which would otherwise pin its slot in sem and stall the spawn loop. + var mu sync.Mutex + + resolved := make(map[string][]string, len(domains)) + var wg sync.WaitGroup -spawnLoop: + for _, domain := range domains { select { case <-ctx.Done(): - break spawnLoop case sem <- struct{}{}: } + if ctx.Err() != nil { + break + } + wg.Add(1) go func(d string) { defer wg.Done() @@ -77,31 +108,34 @@ spawnLoop: rctx, cancel := context.WithTimeout(ctx, lookupTimeout) defer cancel() - ips, err := net.DefaultResolver.LookupHost(rctx, d) + mu.Lock() + c.ownLookups[d] = c.now() + mu.Unlock() + + ips, err := c.lookupHost(rctx, d) if err != nil { // DNS failure is normal for blocked domains, expired records, and similar cases. return } - results <- result{domain: d, ips: ips} + + mu.Lock() + resolved[d] = ips + mu.Unlock() }(domain) } - go func() { - wg.Wait() - close(results) - }() - - resolved := make(map[string][]string, len(domains)) - for r := range results { - resolved[r.domain] = r.ips - } + wg.Wait() if ctx.Err() != nil { return } - // The store keeps one domain per IP, so keep the first domain in a stable order. - uniqueIPs := make(map[string]string, len(resolved)) + // A shared CDN edge IP serves many unrelated domains, so keep the full set per IP. + // domains is sorted, so each IP's slice stays in a stable order and its first entry + // is the one recorded as the representative domain in allowed_ips. + ipDomains := make(map[string][]string, len(resolved)) + + seen := make(map[string]struct{}, len(resolved)) for _, domain := range domains { ips, ok := resolved[domain] if !ok { @@ -109,23 +143,98 @@ spawnLoop: } for _, ip := range ips { - if _, exists := uniqueIPs[ip]; exists { + key := ip + "\x00" + domain + if _, exists := seen[key]; exists { continue } - uniqueIPs[ip] = domain + + seen[key] = struct{}{} + + ipDomains[ip] = append(ipDomains[ip], domain) } } - var totalIPs int - for ip, domain := range uniqueIPs { - if err := c.store.UpsertAllowedIP(ip, domain); err != nil { - slog.Error("cache refresh: failed to upsert IP", "ip", ip, "domain", domain, "err", err) + var ( + totalIPs int + totalPairs int + ) + + for ip, ipDomainSet := range ipDomains { + if err := c.store.UpsertAllowedIP(ip, ipDomainSet[0]); err != nil { + slog.Error("cache refresh: failed to upsert IP", "ip", ip, "domain", ipDomainSet[0], "err", err) } else { totalIPs++ } + + for _, domain := range ipDomainSet { + if err := c.store.UpsertAllowedIPDomain(ip, domain); err != nil { + slog.Error("cache refresh: failed to upsert IP domain", "ip", ip, "domain", domain, "err", err) + } else { + totalPairs++ + } + } + } + + slog.Info("cache refreshed", "domains", len(domains), "ips", totalIPs, "pairs", totalPairs) +} + +// newClientDomains returns, sorted and without duplicates, the domains in queries logged +// since the last call, leaving out the echoes of hexwall's own lookups. +func (c *IPCache) newClientDomains() ([]string, error) { + now := c.now() + if c.ownLookups == nil { + c.ownLookups = make(map[string]time.Time) + } + + for domain, at := range c.ownLookups { + if now.Sub(at) > ownLookupMemory { + delete(c.ownLookups, domain) + } + } + + maxID, err := c.maxQueryID() + if err != nil { + return nil, err + } + + if maxID < c.lastID { + slog.Info("cache refresh: pi-hole query log was reset; re-reading the last hour") + + c.lastID = 0 + } + + queries, err := c.answeredQueriesAfter(c.lastID, now.Add(-refreshLookback).Unix()) + if err != nil { + return nil, err + } + + var domains []string + + for _, q := range queries { + c.lastID = max(c.lastID, q.ID) + if c.isOwnLookup(q) { + continue + } + + domains = append(domains, q.Domain) + } + + slices.Sort(domains) + + return slices.Compact(domains), nil +} + +// isOwnLookup reports whether q was logged when hexwall itself resolved q.Domain. A client +// that asked for the same domain in that same moment is skipped too, which costs nothing: +// hexwall's lookup has just refreshed that domain's IPs. +func (c *IPCache) isOwnLookup(q AnsweredQuery) bool { + at, ok := c.ownLookups[q.Domain] + if !ok { + return false } - slog.Info("cache refreshed", "domains", len(domains), "ips", totalIPs) + return !q.Timestamp.Before(at.Add(-ownLookupSlack)) && + !q.Timestamp.After(at.Add(lookupTimeout+ownLookupSlack)) } // RunRefresh calls Refresh on the given interval until ctx is cancelled. diff --git a/internal/pihole/cache_test.go b/internal/pihole/cache_test.go new file mode 100644 index 0000000..1791c47 --- /dev/null +++ b/internal/pihole/cache_test.go @@ -0,0 +1,181 @@ +package pihole + +import ( + "context" + "path/filepath" + "reflect" + "slices" + "testing" + "time" + + "github.com/hexbytedev/hexwall/internal/store" +) + +func TestIPCacheRefreshWritesTrustedIPAndDomains(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := store.NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + log := &fakeQueryLog{now: time.Now()} + log.clientAsks("example.com") + log.clientAsks("example.org") + cache := &IPCache{ + store: hexwallStore, + answeredQueriesAfter: log.answeredQueriesAfter, + maxQueryID: log.maxQueryID, + now: func() time.Time { return log.now }, + lookupHost: func(_ context.Context, host string) ([]string, error) { + switch host { + case "example.com": + return []string{"93.184.216.34", "93.184.216.35"}, nil + case "example.org": + return []string{"93.184.216.34"}, nil + default: + return nil, nil + } + }, + } + + cache.Refresh(context.Background()) + + allowed, err := hexwallStore.IsAllowed("93.184.216.34") + if err != nil { + t.Fatalf("IsAllowed() error = %v", err) + } + + if !allowed { + t.Fatal("IsAllowed() = false, want true") + } + + domains, err := hexwallStore.DomainsForIP("93.184.216.34") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + + want := []string{"example.com", "example.org"} + if !reflect.DeepEqual(domains, want) { + t.Fatalf("DomainsForIP() = %v, want %v", domains, want) + } +} + +// fakeQueryLog stands in for Pi-hole's query log. When hexwall resolves through Pi-hole, +// its own lookups are logged too; lookupHost records them like Pi-hole would. +type fakeQueryLog struct { + now time.Time + queries []AnsweredQuery + lookups []string // domains hexwall resolved, in order +} + +func (l *fakeQueryLog) clientAsks(domain string) { + l.queries = append(l.queries, AnsweredQuery{ID: int64(len(l.queries) + 1), Domain: domain, Timestamp: l.now}) +} + +func (l *fakeQueryLog) answeredQueriesAfter(afterID, since int64) ([]AnsweredQuery, error) { + var out []AnsweredQuery + + for _, q := range l.queries { + if q.ID > afterID && q.Timestamp.Unix() >= since { + out = append(out, q) + } + } + + return out, nil +} + +func (l *fakeQueryLog) maxQueryID() (int64, error) { + if len(l.queries) == 0 { + return 0, nil + } + + return l.queries[len(l.queries)-1].ID, nil +} + +// lookupThroughPihole resolves like a host whose resolver is Pi-hole: the lookup is logged. +func (l *fakeQueryLog) lookupThroughPihole(_ context.Context, host string) ([]string, error) { + l.lookups = append(l.lookups, host) + l.clientAsks(host) + + return []string{"192.0.2.10"}, nil +} + +func newFakeCache(t *testing.T, log *fakeQueryLog) *IPCache { + t.Helper() + + hexwallStore, err := store.NewStore(filepath.Join(t.TempDir(), "hexwall.db")) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + t.Cleanup(func() { _ = hexwallStore.Close() }) + + return &IPCache{ + store: hexwallStore, + answeredQueriesAfter: log.answeredQueriesAfter, + maxQueryID: log.maxQueryID, + lookupHost: log.lookupThroughPihole, + now: func() time.Time { return log.now }, + } +} + +// The feedback loop: hexwall's lookup is logged by Pi-hole, and must not count as a client +// asking again, or every domain stays trusted forever. +func TestIPCacheRefreshIgnoresOwnLookups(t *testing.T) { + log := &fakeQueryLog{now: time.Unix(1_800_000_000, 0)} + cache := newFakeCache(t, log) + + log.clientAsks("example.com") + + for range 5 { + cache.Refresh(context.Background()) + + log.now = log.now.Add(30 * time.Second) + } + + if want := []string{"example.com"}; !slices.Equal(log.lookups, want) { + t.Fatalf("lookups = %q, want %q (one per client query)", log.lookups, want) + } +} + +func TestIPCacheRefreshResolvesRepeatClientQueries(t *testing.T) { + log := &fakeQueryLog{now: time.Unix(1_800_000_000, 0)} + cache := newFakeCache(t, log) + + log.clientAsks("example.com") + cache.Refresh(context.Background()) + + log.now = log.now.Add(time.Minute) + log.clientAsks("example.com") + cache.Refresh(context.Background()) + + if want := []string{"example.com", "example.com"}; !slices.Equal(log.lookups, want) { + t.Fatalf("lookups = %q, want %q", log.lookups, want) + } +} + +// "pihole flush" empties the log and restarts ids; queries after it must still be read. +func TestIPCacheRefreshAfterQueryLogReset(t *testing.T) { + log := &fakeQueryLog{now: time.Unix(1_800_000_000, 0)} + cache := newFakeCache(t, log) + + log.clientAsks("a.example") + log.clientAsks("b.example") + cache.Refresh(context.Background()) + + log.queries = nil + log.now = log.now.Add(time.Minute) + log.clientAsks("c.example") + cache.Refresh(context.Background()) + + if !slices.Contains(log.lookups, "c.example") { + t.Fatalf("lookups = %q, want c.example resolved after the reset", log.lookups) + } +} diff --git a/internal/pihole/domain_match_test.go b/internal/pihole/domain_match_test.go new file mode 100644 index 0000000..902e216 --- /dev/null +++ b/internal/pihole/domain_match_test.go @@ -0,0 +1,42 @@ +package pihole + +import ( + "testing" +) + +func TestDomainMatches(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + observed string + known string + want bool + }{ + {name: "exact match", observed: "github.com", known: "github.com", want: true}, + {name: "subdomain observed", observed: "www.github.com", known: "github.com", want: true}, + {name: "subdomain known", observed: "github.com", known: "www.github.com", want: true}, + {name: "deeper subdomain", observed: "a.b.github.com", known: "github.com", want: true}, + {name: "deeper subdomain reversed", observed: "github.com", known: "a.b.github.com", want: true}, + {name: "lookalike suffix is not a match", observed: "github.com.evil.com", known: "github.com", want: false}, + {name: "lookalike suffix reversed is not a match", observed: "github.com", known: "github.com.evil.com", want: false}, + {name: "partial label is not a match", observed: "notgithub.com", known: "github.com", want: false}, + {name: "unrelated domains", observed: "example.org", known: "github.com", want: false}, + {name: "empty observed", observed: "", known: "github.com", want: false}, + {name: "empty known", observed: "github.com", known: "", want: false}, + {name: "both empty", observed: "", known: "", want: false}, + {name: "whitespace only", observed: " ", known: "github.com", want: false}, + {name: "case and whitespace normalized", observed: " GitHub.COM ", known: "github.com", want: true}, + {name: "case and whitespace normalized subdomain", observed: " WWW.GitHub.com ", known: " GITHUB.com ", want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := DomainMatches(tt.observed, tt.known); got != tt.want { + t.Fatalf("DomainMatches(%q, %q) = %v, want %v", tt.observed, tt.known, got, tt.want) + } + }) + } +} diff --git a/internal/pihole/pihole.go b/internal/pihole/pihole.go index 028e834..40b6e42 100644 --- a/internal/pihole/pihole.go +++ b/internal/pihole/pihole.go @@ -6,9 +6,12 @@ import ( "database/sql" "errors" "fmt" + "log/slog" "net/url" "os" + "path/filepath" "strings" + "sync/atomic" "time" // Register the pure-Go SQLite driver used to read the Pi-hole FTL database. @@ -17,15 +20,18 @@ import ( // Config holds the Pi-hole database configuration. type Config struct { - DBPath string + DBPath string + GravityDBPath string // optional; derived from DBPath when empty } -// Checker reads Pi-hole query history from the FTL database. +// Checker reads Pi-hole query history from the FTL database and policy lists from gravity. type Checker struct { - db *sql.DB -} + db *sql.DB + gravityDB *sql.DB -const domainLookback = time.Hour + // regex is swapped whole by RefreshRegexLists while scans read it from another goroutine. + regex atomic.Pointer[regexLists] +} // NewChecker opens a read-only Checker for the configured Pi-hole database. func NewChecker(config *Config) (*Checker, error) { @@ -50,13 +56,16 @@ func NewChecker(config *Config) (*Checker, error) { if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("pihole-FTL.db not found at %s", config.DBPath) } + return nil, fmt.Errorf("cannot access %s: %w", config.DBPath, err) } + if err := file.Close(); err != nil { return nil, fmt.Errorf("close access check for %s: %w", config.DBPath, err) } dsn := "file:" + (&url.URL{Path: config.DBPath, RawQuery: "mode=ro"}).String() + db, err := sql.Open("sqlite", dsn) if err != nil { return nil, fmt.Errorf("failed to open pihole-db: %w", err) @@ -67,55 +76,238 @@ func NewChecker(config *Config) (*Checker, error) { return nil, fmt.Errorf("failed to connect to pihole-db: %w", err) } - return &Checker{ - db: db, - }, nil + // Open gravity.db for policy list lookups. + gravityPath := config.GravityDBPath + if gravityPath == "" { + gravityPath = filepath.Join(filepath.Dir(config.DBPath), "gravity.db") + } + + var gravityDB *sql.DB + if err := openGravityDB(gravityPath, &gravityDB); err != nil { + slog.Warn("pi-hole gravity database unavailable; policy checks disabled", + "path", gravityPath, "err", err) + } + + checker := &Checker{ + db: db, + gravityDB: gravityDB, + } + if err := checker.RefreshRegexLists(); err != nil { + slog.Warn("pi-hole regex lists unavailable; only exact-match policy checks apply", "err", err) + } + + return checker, nil +} + +// openGravityDB opens the Pi-hole gravity database in read-only mode. +// On success it sets *out; on failure it returns an error but does not touch *out. +func openGravityDB(path string, out **sql.DB) error { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("cannot access %s: %w", path, err) + } + + if err := file.Close(); err != nil { + return fmt.Errorf("close access check for %s: %w", path, err) + } + + dsn := "file:" + (&url.URL{Path: path, RawQuery: "mode=ro"}).String() + + db, err := sql.Open("sqlite", dsn) + if err != nil { + return fmt.Errorf("failed to open gravity-db: %w", err) + } + + if err := db.Ping(); err != nil { + _ = db.Close() + return fmt.Errorf("failed to connect to gravity-db: %w", err) + } + + *out = db + + return nil } -// Close closes the Checker database connection. +// Close closes both database connections. func (c *Checker) Close() error { - return c.db.Close() + if c == nil { + return nil + } + + var ( + errFTL error + errGravity error + ) + + if c.db != nil { + errFTL = c.db.Close() + } + + if c.gravityDB != nil { + errGravity = c.gravityDB.Close() + } + + return errors.Join(errFTL, errGravity) } -// IsDomainKnown reports whether domain appeared in Pi-hole query history within the last hour. -// Domains missing from that history bypassed Pi-hole DNS resolution and are treated as suspicious. -func (c *Checker) IsDomainKnown(domain string) (bool, error) { +// IsAllowedByPolicy reports whether the domain is on Pi-hole's allowlist or matches an allow regex. +func (c *Checker) IsAllowedByPolicy(domain string) (bool, error) { domain = normalizeDomain(domain) if domain == "" { return false, nil } - cutoff := time.Now().Add(-domainLookback).Unix() + if c == nil || c.gravityDB == nil { + return false, nil + } var found int - err := c.db.QueryRow(` - SELECT 1 - FROM queries - WHERE timestamp >= ? - AND TRIM(domain) <> '' - AND domain = ? COLLATE NOCASE + + err := c.gravityDB.QueryRow(` + SELECT 1 FROM vw_allowlist + WHERE domain = ? COLLATE NOCASE LIMIT 1 - `, cutoff, domain).Scan(&found) + `, domain).Scan(&found) if errors.Is(err, sql.ErrNoRows) { + return c.matchesRegex(domain, true), nil + } + + if err != nil { + return false, fmt.Errorf("allowlist lookup failed: %w", err) + } + + return true, nil +} + +// IsBlockedByPolicy reports whether the domain is on Pi-hole's denylist or gravity, or matches a deny regex. +func (c *Checker) IsBlockedByPolicy(domain string) (bool, error) { + domain = normalizeDomain(domain) + if domain == "" { return false, nil } + + if c == nil || c.gravityDB == nil { + return false, nil + } + + var found int + + err := c.gravityDB.QueryRow(` + SELECT 1 FROM vw_denylist + WHERE domain = ? COLLATE NOCASE + LIMIT 1 + `, domain).Scan(&found) + if err == nil { + return true, nil + } + + if !errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf("denylist lookup failed: %w", err) + } + + err = c.gravityDB.QueryRow(` + SELECT 1 FROM vw_gravity + WHERE domain = ? COLLATE NOCASE + LIMIT 1 + `, domain).Scan(&found) + if errors.Is(err, sql.ErrNoRows) { + return c.matchesRegex(domain, false), nil + } + if err != nil { - return false, fmt.Errorf("database query failed: %w", err) + return false, fmt.Errorf("gravity lookup failed: %w", err) } return true, nil } -// DomainsSeenSince returns the distinct domains Pi-hole recorded since the given Unix timestamp. -func (c *Checker) DomainsSeenSince(since int64) ([]string, error) { - rows, err := c.db.Query(` - SELECT DISTINCT LOWER(TRIM(domain)) +// PolicyCounts returns the number of entries in each Pi-hole policy view. +// The returned map is keyed by view name. Returns nil if gravity DB is unavailable. +func (c *Checker) PolicyCounts() (map[string]int64, error) { + if c == nil || c.gravityDB == nil { + return nil, nil + } + + views := []string{"vw_allowlist", "vw_denylist", "vw_gravity", "vw_regex_allowlist", "vw_regex_denylist"} + counts := make(map[string]int64, len(views)) + + for _, view := range views { + var count int64 + + err := c.gravityDB.QueryRow(fmt.Sprintf(`SELECT COUNT(*) FROM %s`, view)).Scan(&count) + if err != nil { + return nil, fmt.Errorf("count %s: %w", view, err) + } + + counts[view] = count + } + + return counts, nil +} + +// ExtraGroups returns how many enabled Pi-hole groups exist besides Default (id 0). +// hexwall doesn't know which group this machine's queries fall into, so it treats a +// block in any group as a block here; with extra groups, that can be stricter than +// what Pi-hole itself would apply to this machine. Returns 0 if gravity is unavailable. +func (c *Checker) ExtraGroups() (int, error) { + if c == nil || c.gravityDB == nil { + return 0, nil + } + + var count int + if err := c.gravityDB.QueryRow(`SELECT COUNT(*) FROM "group" WHERE id != 0 AND enabled = 1`).Scan(&count); err != nil { + return 0, fmt.Errorf("count groups: %w", err) + } + + return count, nil +} + +// answeredStatuses lists the FTL query status codes that count as "Pi-hole answered this +// domain". Only these feed the trusted-IP cache. A blocked query is still written to the +// query log, and without this filter its domain would be re-resolved by the host's own +// resolver and its real IP trusted -- blocking a domain would end up trusting it. +// Status codes: https://docs.pi-hole.net/database/query-database/#query-status +// +// This is an allow-list on purpose: a status code added by a future FTL release is +// untrusted until it is reviewed here, rather than silently trusted. +var answeredStatuses = []int{ + 2, // forwarded to upstream + 3, // answered from cache + 12, // retried + 13, // retried, but ignored + 14, // already forwarded + 17, // answered from stale cache +} + +// AnsweredQuery is one query from Pi-hole's log that Pi-hole answered. +type AnsweredQuery struct { + ID int64 + Domain string + Timestamp time.Time +} + +// AnsweredQueriesAfter returns the queries Pi-hole answered with an id above afterID and a +// timestamp at or after since (Unix seconds), in id order. Queries Pi-hole blocked are +// excluded; see answeredStatuses. +func (c *Checker) AnsweredQueriesAfter(afterID, since int64) ([]AnsweredQuery, error) { + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(answeredStatuses)), ",") + args := make([]any, 0, len(answeredStatuses)+2) + + args = append(args, afterID, since) + for _, status := range answeredStatuses { + args = append(args, status) + } + + rows, err := c.db.Query(fmt.Sprintf(` + SELECT id, LOWER(TRIM(domain)), timestamp FROM queries - WHERE timestamp >= ? + WHERE id > ? + AND timestamp >= ? AND domain IS NOT NULL AND TRIM(domain) <> '' - ORDER BY LOWER(TRIM(domain)) - `, since) + AND status IN (%s) + ORDER BY id + `, placeholders), args...) if err != nil { return nil, fmt.Errorf("database query failed: %w", err) } @@ -123,24 +315,64 @@ func (c *Checker) DomainsSeenSince(since int64) ([]string, error) { _ = rows.Close() }() - var domains []string + var queries []AnsweredQuery + for rows.Next() { - var domain string - if err := rows.Scan(&domain); err != nil { + var q AnsweredQuery + + var timestamp float64 // FTL v6 stores fractional seconds + if err := rows.Scan(&q.ID, &q.Domain, ×tamp); err != nil { return nil, fmt.Errorf("scan failed: %w", err) } - if domain == "" { - continue - } - domains = append(domains, domain) + q.Timestamp = time.Unix(0, int64(timestamp*float64(time.Second))) + queries = append(queries, q) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("rows iteration failed: %w", err) } - return domains, nil + return queries, nil +} + +// MaxQueryID returns the highest id in Pi-hole's query log, or 0 when it is empty. It drops +// below an id seen earlier only when the log was flushed or recreated. +func (c *Checker) MaxQueryID() (int64, error) { + var maxID sql.NullInt64 + if err := c.db.QueryRow(`SELECT MAX(id) FROM queries`).Scan(&maxID); err != nil { + return 0, fmt.Errorf("database query failed: %w", err) + } + + return maxID.Int64, nil +} + +// DomainMatches reports whether an observed SNI domain corresponds to a known domain, +// treating a subdomain as a match for its parent in either direction. +// +// The direction is intentionally symmetric: Zeek reports the SNI exactly as sent on the wire +// (www.github.com) while the store holds whatever Pi-hole logged and the cache resolved +// (github.com), and either can be the more specific name. +// +// The match requires a label boundary, so github.com.evil.com does not match github.com: +// it does not end in ".github.com". A plain suffix test would have accepted it. +// +// A registrable-domain (eTLD+1) comparison via golang.org/x/net/publicsuffix would handle more +// edge cases -- notably unrelated names under a shared public suffix -- but it adds a third-party +// dependency for a marginal gain, so the label-boundary rule is the deliberate v1 tradeoff. +func DomainMatches(observed, known string) bool { + observed = normalizeDomain(observed) + known = normalizeDomain(known) + + if observed == "" || known == "" { + return false + } + + if observed == known { + return true + } + + return strings.HasSuffix(observed, "."+known) || strings.HasSuffix(known, "."+observed) } func normalizeDomain(domain string) string { diff --git a/internal/pihole/pihole_test.go b/internal/pihole/pihole_test.go index b64decc..43edbb3 100644 --- a/internal/pihole/pihole_test.go +++ b/internal/pihole/pihole_test.go @@ -2,13 +2,16 @@ package pihole import ( "net/url" + "slices" "testing" + "time" ) func TestSQLiteReadOnlyDSNRelativePath(t *testing.T) { t.Parallel() dsn := "file:" + (&url.URL{Path: "./pihole-FTL.db", RawQuery: "mode=ro"}).String() + want := "file:./pihole-FTL.db?mode=ro" if dsn != want { t.Fatalf("dsn = %q, want %q", dsn, want) @@ -19,8 +22,66 @@ func TestSQLiteReadOnlyDSNAbsolutePath(t *testing.T) { t.Parallel() dsn := "file:" + (&url.URL{Path: "/etc/pihole/pihole-FTL.db", RawQuery: "mode=ro"}).String() + want := "file:/etc/pihole/pihole-FTL.db?mode=ro" if dsn != want { t.Fatalf("dsn = %q, want %q", dsn, want) } } + +// TestAnsweredQueriesAfterExcludesBlocked pins the trust-cache input: a domain Pi-hole blocked +// is still in the query log, and must not come back as a domain to resolve and trust. +// Statuses 2, 3 and 17 are the ones a real Pi-hole (FTL v6.7.1) logged for answered queries; +// 1 is what it logged for a gravity-blocked domain. +func TestAnsweredQueriesAfterExcludesBlocked(t *testing.T) { + t.Parallel() + + checker := newTestChecker(t, "") + now := time.Now().Unix() + + rows := []struct { + domain string + status int + }{ + {"forwarded.example", 2}, + {"cached.example", 3}, + {"stale-cache.example", 17}, + {"gravity-blocked.example", 1}, + {"regex-blocked.example", 4}, + {"denylist-blocked.example", 5}, + {"cname-blocked.example", 9}, + } + for _, r := range rows { + if _, err := checker.db.Exec( + `INSERT INTO queries (timestamp, type, domain, status) VALUES (?, 1, ?, ?)`, + now, r.domain, r.status, + ); err != nil { + t.Fatalf("insert %q: %v", r.domain, err) + } + } + + queries, err := checker.AnsweredQueriesAfter(0, now-60) + if err != nil { + t.Fatalf("AnsweredQueriesAfter() error: %v", err) + } + + var got []string + for _, q := range queries { + got = append(got, q.Domain) + } + + want := []string{"forwarded.example", "cached.example", "stale-cache.example"} + if !slices.Equal(got, want) { + t.Fatalf("AnsweredQueriesAfter() domains = %v, want %v", got, want) + } + + // The cursor: nothing after the last id read. + maxID, err := checker.MaxQueryID() + if err != nil { + t.Fatalf("MaxQueryID() error: %v", err) + } + + if rest, err := checker.AnsweredQueriesAfter(maxID, now-60); err != nil || len(rest) != 0 { + t.Fatalf("AnsweredQueriesAfter(MaxQueryID()) = %v, %v; want none", rest, err) + } +} diff --git a/internal/pihole/policy_test.go b/internal/pihole/policy_test.go new file mode 100644 index 0000000..32bf9a8 --- /dev/null +++ b/internal/pihole/policy_test.go @@ -0,0 +1,603 @@ +package pihole + +import ( + "database/sql" + "net/url" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestNormalizeDomain(t *testing.T) { + if got := normalizeDomain(" Example.COM "); got != "example.com" { + t.Fatalf("normalizeDomain() = %q, want %q", got, "example.com") + } +} + +// openTestGravityDB creates a temporary SQLite database with Pi-hole-like views +// and returns the opened DB along with its path. The caller must close it. +func openTestGravityDB(t *testing.T) (*sql.DB, string) { + t.Helper() + dir := t.TempDir() + dbPath := filepath.Join(dir, "gravity.db") + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: dbPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open test gravity db: %v", err) + } + + // Create backing tables and views matching Pi-hole v6 shapes. + schema := ` + CREATE TABLE adlist (id INTEGER PRIMARY KEY, address TEXT, enabled INTEGER, date_added INTEGER, date_modified INTEGER, comment TEXT, type INTEGER, number INTEGER); + CREATE TABLE domainlist (id INTEGER PRIMARY KEY, type INTEGER, domain TEXT, enabled INTEGER, date_added INTEGER, date_modified INTEGER, comment TEXT, groups TEXT); + CREATE TABLE gravity (id INTEGER PRIMARY KEY, domain TEXT, adlist_id INTEGER); + CREATE INDEX idx_gravity ON gravity (domain, adlist_id); + + CREATE VIEW vw_allowlist AS SELECT domain FROM domainlist WHERE type = 0 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_denylist AS SELECT domain FROM domainlist WHERE type = 1 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_gravity AS SELECT domain FROM gravity; + CREATE VIEW vw_regex_allowlist AS SELECT domain FROM domainlist WHERE type = 2 AND enabled = 1 AND TRIM(domain) <> ''; + CREATE VIEW vw_regex_denylist AS SELECT domain FROM domainlist WHERE type = 3 AND enabled = 1 AND TRIM(domain) <> ''; + ` + if _, err := db.Exec(schema); err != nil { + _ = db.Close() + + t.Fatalf("create schema: %v", err) + } + + return db, dbPath +} + +// insertDomain inserts a domain into the domainlist table with the given type. +// type 0 = allowlist, type 1 = denylist, type 2 = regex allowlist, type 3 = regex denylist. +func insertDomain(t *testing.T, db *sql.DB, domain string, listType int) { + t.Helper() + + _, err := db.Exec(`INSERT INTO domainlist (type, domain, enabled) VALUES (?, ?, 1)`, listType, domain) + if err != nil { + t.Fatalf("insert domain %q type %d: %v", domain, listType, err) + } +} + +// insertGravity inserts a domain into the gravity table. +func insertGravity(t *testing.T, db *sql.DB, domain string) { + t.Helper() + + _, err := db.Exec(`INSERT INTO gravity (domain, adlist_id) VALUES (?, 1)`, domain) + if err != nil { + t.Fatalf("insert gravity domain %q: %v", domain, err) + } +} + +func newTestChecker(t *testing.T, gravityPath string) *Checker { + t.Helper() + ftlPath := filepath.Join(t.TempDir(), "pihole-FTL.db") + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open test FTL db: %v", err) + } + + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY, timestamp INTEGER, type INTEGER, domain TEXT, status INTEGER, ip TEXT)`); err != nil { + _ = db.Close() + + t.Fatalf("create FTL schema: %v", err) + } + + var gravityDB *sql.DB + if gravityPath != "" { + gravityDB, err = sql.Open("sqlite", "file:"+(&url.URL{Path: gravityPath, RawQuery: "mode=ro"}).String()) + if err != nil { + _ = db.Close() + + t.Fatalf("open gravity db: %v", err) + } + } + + return &Checker{db: db, gravityDB: gravityDB} +} + +func TestIsBlockedByPolicy_Denylist(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "blocked.example.com", 1) // type 1 = denylist + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + blocked, err := checker.IsBlockedByPolicy("blocked.example.com") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true for denylist domain") + } +} + +func TestIsBlockedByPolicy_Gravity(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertGravity(t, gravDB, "malware.example.net") + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + blocked, err := checker.IsBlockedByPolicy("malware.example.net") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true for gravity domain") + } +} + +func TestIsBlockedByPolicy_NoMatch(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertGravity(t, gravDB, "other.example.com") + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + blocked, err := checker.IsBlockedByPolicy("safe.example.org") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + + if blocked { + t.Fatal("IsBlockedByPolicy() = true, want false for unknown domain") + } +} + +func TestIsAllowedByPolicy_Allowlist(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "trusted.example.com", 0) // type 0 = allowlist + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + allowed, err := checker.IsAllowedByPolicy("trusted.example.com") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v", err) + } + + if !allowed { + t.Fatal("IsAllowedByPolicy() = false, want true for allowlist domain") + } +} + +func TestIsAllowedByPolicy_NoMatch(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "trusted.example.com", 0) + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + allowed, err := checker.IsAllowedByPolicy("unknown.example.org") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v", err) + } + + if allowed { + t.Fatal("IsAllowedByPolicy() = true, want false for unknown domain") + } +} + +func TestPolicyLookup_CaseNormalization(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "CaseSensitive.COM", 1) + insertGravity(t, gravDB, "GravityDomain.ORG") + insertDomain(t, gravDB, "AllowCase.COM", 0) + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + // Denylist: case-insensitive match + blocked, err := checker.IsBlockedByPolicy(" casesensitive.com ") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true for case-insensitive denylist match") + } + + // Gravity: case-insensitive match + blocked, err = checker.IsBlockedByPolicy("GRAVITYDOMAIN.ORG") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true for case-insensitive gravity match") + } + + // Allowlist: case-insensitive match + allowed, err := checker.IsAllowedByPolicy("allowcase.com") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v", err) + } + + if !allowed { + t.Fatal("IsAllowedByPolicy() = false, want true for case-insensitive allowlist match") + } +} + +func TestPolicyLookup_EmptyDomain(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "blocked.example.com", 1) + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + blocked, err := checker.IsBlockedByPolicy("") + if err != nil { + t.Fatalf("IsBlockedByPolicy(\"\") error = %v", err) + } + + if blocked { + t.Fatal("IsBlockedByPolicy(\"\") = true, want false") + } + + allowed, err := checker.IsAllowedByPolicy("") + if err != nil { + t.Fatalf("IsAllowedByPolicy(\"\") error = %v", err) + } + + if allowed { + t.Fatal("IsAllowedByPolicy(\"\") = true, want false") + } +} + +func TestPolicyLookup_NilGravityDB(t *testing.T) { + t.Parallel() + + // Create a checker with nil gravityDB (simulates gravity.db unavailable). + checker := &Checker{ + db: nil, // not needed for these methods + gravityDB: nil, + } + + blocked, err := checker.IsBlockedByPolicy("anything.example.com") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v, want nil", err) + } + + if blocked { + t.Fatal("IsBlockedByPolicy() = true, want false for nil gravity DB") + } + + allowed, err := checker.IsAllowedByPolicy("anything.example.com") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v, want nil", err) + } + + if allowed { + t.Fatal("IsAllowedByPolicy() = true, want false for nil gravity DB") + } +} + +func TestPolicyCounts(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "allow1.com", 0) + insertDomain(t, gravDB, "allow2.com", 0) + insertDomain(t, gravDB, "deny1.com", 1) + insertGravity(t, gravDB, "g1.com") + insertGravity(t, gravDB, "g2.com") + insertGravity(t, gravDB, "g3.com") + insertDomain(t, gravDB, "regex_allow", 2) + insertDomain(t, gravDB, "regex_deny1", 3) + insertDomain(t, gravDB, "regex_deny2", 3) + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + counts, err := checker.PolicyCounts() + if err != nil { + t.Fatalf("PolicyCounts() error = %v", err) + } + + if counts == nil { + t.Fatal("PolicyCounts() = nil, want non-nil map") + } + + want := map[string]int64{ + "vw_allowlist": 2, + "vw_denylist": 1, + "vw_gravity": 3, + "vw_regex_allowlist": 1, + "vw_regex_denylist": 2, + } + for view, wantCount := range want { + if got := counts[view]; got != wantCount { + t.Errorf("PolicyCounts()[%q] = %d, want %d", view, got, wantCount) + } + } +} + +func TestPolicyCounts_NilGravityDB(t *testing.T) { + t.Parallel() + + checker := &Checker{gravityDB: nil} + + counts, err := checker.PolicyCounts() + if err != nil { + t.Fatalf("PolicyCounts() error = %v, want nil", err) + } + + if counts != nil { + t.Fatalf("PolicyCounts() = %v, want nil for nil gravity DB", counts) + } +} + +func TestGravityDBPath_Derived(t *testing.T) { + t.Parallel() + + // Create the FTL db in a known directory. + dir := t.TempDir() + ftlPath := filepath.Join(dir, "pihole-FTL.db") + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open FTL db: %v", err) + } + + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY)`); err != nil { + _ = db.Close() + + t.Fatalf("init FTL db: %v", err) + } + + _ = db.Close() + + // Create gravity.db next to it. + gravPath := filepath.Join(dir, "gravity.db") + + gravDB, err := sql.Open("sqlite", "file:"+(&url.URL{Path: gravPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open gravity db: %v", err) + } + // Create minimal schema so gravity opens cleanly. + if _, err := gravDB.Exec(`CREATE TABLE adlist (id INTEGER PRIMARY KEY); CREATE VIEW vw_gravity AS SELECT 1 WHERE 0;`); err != nil { + _ = gravDB.Close() + + t.Fatalf("create gravity schema: %v", err) + } + + _ = gravDB.Close() + + // NewChecker should auto-discover gravity.db next to pihole-FTL.db. + checker, err := NewChecker(&Config{DBPath: ftlPath}) + if err != nil { + t.Fatalf("NewChecker() error = %v", err) + } + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + if checker.gravityDB == nil { + t.Fatal("gravityDB is nil, want it to be auto-discovered") + } +} + +func TestGravityDBPath_Missing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + ftlPath := filepath.Join(dir, "pihole-FTL.db") + + db, err := sql.Open("sqlite", "file:"+(&url.URL{Path: ftlPath, RawQuery: "mode=rwc"}).String()) + if err != nil { + t.Fatalf("open FTL db: %v", err) + } + + if _, err := db.Exec(`CREATE TABLE queries (id INTEGER PRIMARY KEY)`); err != nil { + _ = db.Close() + + t.Fatalf("init FTL db: %v", err) + } + + _ = db.Close() + + // No gravity.db in dir — NewChecker should succeed with nil gravityDB. + checker, err := NewChecker(&Config{DBPath: ftlPath}) + if err != nil { + t.Fatalf("NewChecker() error = %v, want nil when gravity.db is missing", err) + } + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + if checker.gravityDB != nil { + t.Fatal("gravityDB is non-nil, want nil when gravity.db is missing") + } + + // Policy methods should gracefully return false. + blocked, err := checker.IsBlockedByPolicy("anything.com") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + + if blocked { + t.Fatal("IsBlockedByPolicy() = true, want false when gravity unavailable") + } +} + +func TestCloseJoinsErrors(t *testing.T) { + t.Parallel() + + // Close on a checker with both handles should not panic. + gravDB, gravPath := openTestGravityDB(t) + _ = gravDB.Close() // close it first so Close() returns an error + + checker := newTestChecker(t, gravPath) + // Force gravity DB to nil after construction to test the nil path. + _ = checker.gravityDB.Close() + checker.gravityDB = nil + + err := checker.Close() + if err != nil { + // We expect an error from closing the already-closed FTL db or nil gravity. + // Just verify it doesn't panic. + t.Logf("Close() returned (expected for double-close): %v", err) + } +} + +func TestIsBlockedByPolicy_DenylistBeatsGravity(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "both.example.com", 1) // denylist + insertGravity(t, gravDB, "both.example.com") // also in gravity + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + blocked, err := checker.IsBlockedByPolicy("both.example.com") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true when domain is in both denylist and gravity") + } +} + +func TestIsAllowedByPolicy_TakesPrecedenceOverGravity(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertDomain(t, gravDB, "special.example.com", 0) // allowlist + insertGravity(t, gravDB, "special.example.com") // also in gravity + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + // Allowlist match should be independent of gravity presence. + allowed, err := checker.IsAllowedByPolicy("special.example.com") + if err != nil { + t.Fatalf("IsAllowedByPolicy() error = %v", err) + } + + if !allowed { + t.Fatal("IsAllowedByPolicy() = false, want true for allowlisted domain") + } +} + +func TestIsBlockedByPolicy_WhitespaceNormalization(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + insertGravity(t, gravDB, "trim.example.com") + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + defer func() { + if err := checker.Close(); err != nil { + t.Errorf("checker.Close(): %v", err) + } + }() + + blocked, err := checker.IsBlockedByPolicy(" Trim.Example.COM ") + if err != nil { + t.Fatalf("IsBlockedByPolicy() error = %v", err) + } + + if !blocked { + t.Fatal("IsBlockedByPolicy() = false, want true with whitespace around domain") + } +} + +// Only enabled groups other than Default count: those are the setups where a block may +// apply to some devices and not to this machine. +func TestExtraGroups(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + groups string + want int + }{ + {"default only", `(0, 'Default', 1)`, 0}, + {"one extra, one disabled", `(0, 'Default', 1), (1, 'Kids', 1), (2, 'Old', 0)`, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + gravDB, gravPath := openTestGravityDB(t) + if _, err := gravDB.Exec(`CREATE TABLE "group" (id INTEGER PRIMARY KEY, enabled INTEGER, name TEXT); + INSERT INTO "group" (id, name, enabled) VALUES ` + tc.groups); err != nil { + t.Fatal(err) + } + + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + t.Cleanup(func() { _ = checker.Close() }) + + got, err := checker.ExtraGroups() + if err != nil || got != tc.want { + t.Fatalf("ExtraGroups() = %d, %v; want %d, nil", got, err, tc.want) + } + }) + } +} diff --git a/internal/pihole/regex.go b/internal/pihole/regex.go new file mode 100644 index 0000000..2f52a6c --- /dev/null +++ b/internal/pihole/regex.go @@ -0,0 +1,207 @@ +package pihole + +import ( + "errors" + "fmt" + "log/slog" + "regexp" + "slices" + "strings" +) + +// Pi-hole's regex lists hold rules, not names: one pattern such as (^|\.)tracker\.example$ +// covers every subdomain. FTL matches them as case-insensitive POSIX extended regex; Go's +// regexp accepts nearly all of that syntax, and a pattern it cannot compile is skipped with +// a warning. +// +// A pattern can carry FTL options after a semicolon (";invert", ";querytype=AAAA", +// ";reply=..."). hexwall judges a TLS connection, not a DNS query, so an option it cannot +// honour exactly is resolved toward not killing: an allow rule is applied anyway (too broad +// an allow can only miss a block), while a deny rule is skipped (too broad a deny would kill +// traffic Pi-hole itself lets through). + +const ( + regexAllowView = "vw_regex_allowlist" + regexDenyView = "vw_regex_denylist" +) + +// regexRule is one compiled Pi-hole regex list entry. +type regexRule struct { + re *regexp.Regexp + invert bool +} + +func (r regexRule) matches(domain string) bool { + return r.re.MatchString(domain) != r.invert +} + +// regexLists is an immutable snapshot of both regex lists. The raw patterns are kept so a +// refresh can tell whether anything changed, and skip recompiling (and re-warning) if not. +type regexLists struct { + allowRaw []string + denyRaw []string + allow []regexRule + deny []regexRule +} + +// parseRegexRule compiles one regex list entry. It returns an error when the entry must be +// skipped: the pattern does not compile, or it is a deny rule with an option hexwall cannot +// honour for a connection. +func parseRegexRule(raw string, allow bool) (regexRule, error) { + pattern, options, _ := strings.Cut(raw, ";") + + var rule regexRule + + if options != "" { + for _, option := range strings.Split(options, ";") { + switch { + case option == "invert": + rule.invert = true + case strings.HasPrefix(option, "reply="): + // Changes how Pi-hole answers a blocked query, not whether it blocks. + case strings.HasPrefix(option, "querytype="): + if !allow { + return regexRule{}, errors.New("deny rule limited to a DNS query type cannot be applied to a connection") + } + default: + if !allow { + return regexRule{}, fmt.Errorf("unknown option %q", option) + } + } + } + } + + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + return regexRule{}, fmt.Errorf("pattern does not compile in Go: %w", err) + } + + rule.re = re + + return rule, nil +} + +func compileRegexRules(raws []string, allow bool) []regexRule { + rules := make([]regexRule, 0, len(raws)) + for _, raw := range raws { + rule, err := parseRegexRule(raw, allow) + if err != nil { + if allow { + slog.Warn("pi-hole regex allow rule skipped; connections it allows may still be killed by reputation checks", + "rule", raw, "err", err) + } else { + slog.Warn("pi-hole regex deny rule skipped", "rule", raw, "err", err) + } + + continue + } + + rules = append(rules, rule) + } + + return rules +} + +// RefreshRegexLists reloads Pi-hole's regex allow and deny lists from gravity.db, so rules +// edited in Pi-hole take effect without restarting hexwall. On error the previously loaded +// rules stay in use: stale rules are closer to the user's intent than no rules. +func (c *Checker) RefreshRegexLists() error { + if c == nil || c.gravityDB == nil { + return nil + } + + allowRaw, err := c.regexPatterns(regexAllowView) + if err != nil { + return err + } + + denyRaw, err := c.regexPatterns(regexDenyView) + if err != nil { + return err + } + + if current := c.regex.Load(); current != nil && + slices.Equal(current.allowRaw, allowRaw) && slices.Equal(current.denyRaw, denyRaw) { + return nil + } + + lists := ®exLists{ + allowRaw: allowRaw, + denyRaw: denyRaw, + allow: compileRegexRules(allowRaw, true), + deny: compileRegexRules(denyRaw, false), + } + c.regex.Store(lists) + slog.Info("pi-hole regex lists loaded", + "allow", len(lists.allow), "deny", len(lists.deny), + "skipped", len(allowRaw)+len(denyRaw)-len(lists.allow)-len(lists.deny)) + + return nil +} + +func (c *Checker) regexPatterns(view string) ([]string, error) { + rows, err := c.gravityDB.Query(fmt.Sprintf(` + SELECT DISTINCT TRIM(domain) + FROM %s + WHERE TRIM(domain) <> '' + ORDER BY TRIM(domain) + `, view)) + if err != nil { + return nil, fmt.Errorf("%s lookup failed: %w", view, err) + } + defer func() { + _ = rows.Close() + }() + + var patterns []string + + for rows.Next() { + var pattern string + if err := rows.Scan(&pattern); err != nil { + return nil, fmt.Errorf("%s scan failed: %w", view, err) + } + + patterns = append(patterns, pattern) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("%s iteration failed: %w", view, err) + } + + return patterns, nil +} + +// AllowListComplete reports whether every Pi-hole regex allow rule could be applied. +// When one couldn't (a pattern Go can't compile), IsAllowedByPolicy may answer "not +// allowed" for a domain the user did allow, so checks hexwall adds on top of Pi-hole, +// like reputation, must not kill on their own. +func (c *Checker) AllowListComplete() bool { + if c == nil { + return true + } + + lists := c.regex.Load() + + return lists == nil || len(lists.allow) == len(lists.allowRaw) +} + +// matchesRegex reports whether domain matches any loaded allow (or deny) regex rule. +func (c *Checker) matchesRegex(domain string, allow bool) bool { + lists := c.regex.Load() + if lists == nil { + return false + } + + rules := lists.deny + if allow { + rules = lists.allow + } + + for _, rule := range rules { + if rule.matches(domain) { + return true + } + } + + return false +} diff --git a/internal/pihole/regex_test.go b/internal/pihole/regex_test.go new file mode 100644 index 0000000..9bf3f20 --- /dev/null +++ b/internal/pihole/regex_test.go @@ -0,0 +1,160 @@ +package pihole + +import "testing" + +// Expected results below were confirmed against Pi-hole's own engine +// (pihole-FTL v6.7.1 regex-test), not assumed. + +func newRegexTestChecker(t *testing.T, rules map[string]int) *Checker { + t.Helper() + + gravDB, gravPath := openTestGravityDB(t) + for rule, listType := range rules { + insertDomain(t, gravDB, rule, listType) + } + + _ = gravDB.Close() + + checker := newTestChecker(t, gravPath) + t.Cleanup(func() { _ = checker.Close() }) + + if err := checker.RefreshRegexLists(); err != nil { + t.Fatalf("RefreshRegexLists() error = %v", err) + } + + return checker +} + +func TestIsBlockedByPolicy_RegexDenylist(t *testing.T) { + t.Parallel() + checker := newRegexTestChecker(t, map[string]int{ + `(^|\.)tracker\.example$`: 3, // type 3 = regex denylist + `^ad[[:digit:]]+\.`: 3, + `(^|\.)Shouty\.Example$`: 3, + `(^|\.)aaaa-only\.example$;querytype=AAAA`: 3, + `(^|\.)reply\.example$;reply=nxdomain`: 3, + `(a)\1`: 3, // backreference: valid for FTL, not for Go's regexp + }) + + tests := []struct { + domain string + want bool + }{ + {"c2.tracker.example", true}, + {"tracker.example", true}, + {"tracker.example.evil.com", false}, // the $ anchor holds + {"C2.TRACKER.EXAMPLE", true}, // FTL matches case-insensitively + {"ad7.example", true}, // POSIX character class + {"api.shouty.example", true}, // uppercase pattern, lowercase domain + {"x.reply.example", true}, // reply= changes the answer, not the verdict + {"x.aaaa-only.example", false}, // a querytype-limited deny rule is skipped + {"unrelated.example", false}, + } + for _, tt := range tests { + blocked, err := checker.IsBlockedByPolicy(tt.domain) + if err != nil { + t.Fatalf("IsBlockedByPolicy(%q) error = %v", tt.domain, err) + } + + if blocked != tt.want { + t.Errorf("IsBlockedByPolicy(%q) = %v, want %v", tt.domain, blocked, tt.want) + } + } +} + +func TestIsAllowedByPolicy_RegexAllowlist(t *testing.T) { + t.Parallel() + checker := newRegexTestChecker(t, map[string]int{ + `(^|\.)mycompany\.com$`: 2, // type 2 = regex allowlist + `(^|\.)ipv6-app\.example$;querytype=AAAA`: 2, + }) + + tests := []struct { + domain string + want bool + }{ + {"api.mycompany.com", true}, + {"mycompany.com.evil.net", false}, + {"x.ipv6-app.example", true}, // a querytype-limited allow rule is applied anyway + {"unrelated.example", false}, + } + for _, tt := range tests { + allowed, err := checker.IsAllowedByPolicy(tt.domain) + if err != nil { + t.Fatalf("IsAllowedByPolicy(%q) error = %v", tt.domain, err) + } + + if allowed != tt.want { + t.Errorf("IsAllowedByPolicy(%q) = %v, want %v", tt.domain, allowed, tt.want) + } + } +} + +func TestRegexRule_Invert(t *testing.T) { + t.Parallel() + + rule, err := parseRegexRule(`(^|\.)tracker\.example$;invert`, false) + if err != nil { + t.Fatalf("parseRegexRule() error = %v", err) + } + + if !rule.matches("good.example") { + t.Error("inverted rule should match a domain the pattern does not") + } + + if rule.matches("c2.tracker.example") { + t.Error("inverted rule should not match a domain the pattern does") + } +} + +func TestParseRegexRule_UnknownOption(t *testing.T) { + t.Parallel() + + if _, err := parseRegexRule(`^x\.;future=1`, false); err == nil { + t.Error("deny rule with an unknown option should be skipped") + } + + if _, err := parseRegexRule(`^x\.;future=1`, true); err != nil { + t.Errorf("allow rule with an unknown option should still apply, got %v", err) + } +} + +func TestRefreshRegexLists_PicksUpNewRules(t *testing.T) { + t.Parallel() + gravDB, gravPath := openTestGravityDB(t) + t.Cleanup(func() { _ = gravDB.Close() }) + + checker := newTestChecker(t, gravPath) + t.Cleanup(func() { _ = checker.Close() }) + + if err := checker.RefreshRegexLists(); err != nil { + t.Fatalf("RefreshRegexLists() error = %v", err) + } + + if blocked, _ := checker.IsBlockedByPolicy("c2.tracker.example"); blocked { + t.Fatal("blocked before any rule exists") + } + + insertDomain(t, gravDB, `(^|\.)tracker\.example$`, 3) + + if err := checker.RefreshRegexLists(); err != nil { + t.Fatalf("RefreshRegexLists() error = %v", err) + } + + if blocked, _ := checker.IsBlockedByPolicy("c2.tracker.example"); !blocked { + t.Fatal("rule added in Pi-hole not picked up by refresh") + } +} + +func TestRefreshRegexLists_NilGravityDB(t *testing.T) { + t.Parallel() + + checker := &Checker{} + if err := checker.RefreshRegexLists(); err != nil { + t.Fatalf("RefreshRegexLists() error = %v", err) + } + + if checker.matchesRegex("anything.example", false) { + t.Fatal("no rules loaded, nothing should match") + } +} diff --git a/internal/service/service.go b/internal/service/service.go new file mode 100644 index 0000000..6f55a76 --- /dev/null +++ b/internal/service/service.go @@ -0,0 +1,290 @@ +// Package service installs systemd units so hexwall, and the Zeek it reads, survive a reboot. +package service + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + unitDir = "/etc/systemd/system" + hexwallUnit = "hexwall.service" + zeekUnit = "zeek.service" + // The first line of every unit hexwall writes. A zeek.service without it belongs to + // someone else and is never overwritten. + marker = "# Installed by hexwall install-service. Re-run it to update; edits here are overwritten." + // hexwall's state directory; systemd creates it (StateDirectory=hexwall) and starts + // hexwall there, so the default ./hexwall.db lands in it instead of in /. + stateDir = "/var/lib/hexwall" +) + +// Packaged Zeek units live here; one of them means Zeek already has a unit of its own. +var vendorUnitDirs = []string{"/usr/lib/systemd/system", "/lib/systemd/system"} + +// Options describes the units to install. +type Options struct { + Binary string // absolute path of the hexwall binary the unit runs + Args []string // validated hexwall flags, already passed through WithSafeMode + // Zeekctl is the zeekctl to run from zeek.service; empty when hexwall doesn't use Zeek. + Zeekctl string +} + +// Install writes the units, reloads systemd, enables them for boot, and (re)starts hexwall. +func Install(out io.Writer, opts Options) error { + if os.Geteuid() != 0 { + return errors.New("writing systemd units needs root (run with sudo)") + } + + if _, err := exec.LookPath("systemctl"); err != nil { + return errors.New("systemctl not found: install-service only supports systemd") + } + + return install(out, opts, unitDir, vendorUnitDirs, runSystemctl) +} + +func install(out io.Writer, opts Options, dir string, vendorDirs []string, systemctl func(args ...string) error) error { + withZeek := false + + if opts.Zeekctl != "" { + switch owned, err := zeekUnitOwner(dir, vendorDirs); { + case err != nil: + return err + case owned: + changed, err := writeUnit(filepath.Join(dir, zeekUnit), renderZeekUnit(opts.Zeekctl)) + if err != nil { + return err + } + + report(out, zeekUnit, dir, changed) + + withZeek = true + default: + _, _ = fmt.Fprintf(out, "✓ %s already exists and isn't hexwall's; left as is\n", zeekUnit) + withZeek = true + } + } + + changed, err := writeUnit(filepath.Join(dir, hexwallUnit), renderHexwallUnit(opts.Binary, opts.Args, withZeek)) + if err != nil { + return err + } + + report(out, hexwallUnit, dir, changed) + + steps := [][]string{{"daemon-reload"}} + if withZeek { + steps = append(steps, []string{"enable", "--now", zeekUnit}) + } + + steps = append(steps, []string{"enable", hexwallUnit}, []string{"restart", hexwallUnit}) + for _, step := range steps { + if err := systemctl(step...); err != nil { + return fmt.Errorf("systemctl %s: %w", strings.Join(step, " "), err) + } + } + + _, _ = fmt.Fprintf(out, ` +hexwall now starts at boot and restarts if it crashes. + status: systemctl status %[1]s + logs: journalctl -u %[1]s -f + remove: sudo systemctl disable --now %[1]s +`, hexwallUnit) + + return nil +} + +// Uninstall stops, disables and deletes the units hexwall installed. A zeek.service hexwall +// didn't write keeps running untouched. +func Uninstall(out io.Writer) error { + if os.Geteuid() != 0 { + return errors.New("removing systemd units needs root (run with sudo)") + } + + if _, err := exec.LookPath("systemctl"); err != nil { + _, _ = fmt.Fprintln(out, "✓ no systemd here; no units to remove") + return nil + } + + return uninstall(out, unitDir, runSystemctl) +} + +func uninstall(out io.Writer, dir string, systemctl func(args ...string) error) error { + var removed []string + // hexwall first: it is ordered after Zeek, so it stops before Zeek does. + for _, unit := range []string{hexwallUnit, zeekUnit} { + path := filepath.Join(dir, unit) + + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + + if err != nil { + return err + } + + if !bytes.HasPrefix(data, []byte(marker)) { + _, _ = fmt.Fprintf(out, "! %s wasn't written by hexwall; left in place\n", path) + continue + } + + if err := systemctl("disable", "--now", unit); err != nil { + return fmt.Errorf("systemctl disable --now %s: %w", unit, err) + } + + if err := os.Remove(path); err != nil { + return err + } + + _, _ = fmt.Fprintf(out, "✓ stopped and removed %s\n", path) + + removed = append(removed, unit) + } + + if len(removed) == 0 { + _, _ = fmt.Fprintln(out, "✓ no hexwall systemd units installed") + return nil + } + + if err := systemctl("daemon-reload"); err != nil { + return fmt.Errorf("systemctl daemon-reload: %w", err) + } + + return nil +} + +// zeekUnitOwner reports whether zeek.service is free for hexwall to write: either absent, +// or a unit hexwall wrote earlier. +func zeekUnitOwner(dir string, vendorDirs []string) (bool, error) { + for _, vendor := range vendorDirs { + if _, err := os.Stat(filepath.Join(vendor, zeekUnit)); err == nil { + return false, nil + } + } + + data, err := os.ReadFile(filepath.Join(dir, zeekUnit)) + if errors.Is(err, os.ErrNotExist) { + return true, nil + } + + if err != nil { + return false, err + } + + return bytes.HasPrefix(data, []byte(marker)), nil +} + +func writeUnit(path, content string) (changed bool, err error) { + if current, err := os.ReadFile(path); err == nil && string(current) == content { + return false, nil + } + + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + return false, fmt.Errorf("write %s: %w", path, err) + } + + return true, nil +} + +func report(out io.Writer, unit, dir string, changed bool) { + if changed { + _, _ = fmt.Fprintf(out, "✓ wrote %s\n", filepath.Join(dir, unit)) + } else { + _, _ = fmt.Fprintf(out, "✓ %s already up to date\n", filepath.Join(dir, unit)) + } +} + +func runSystemctl(args ...string) error { + cmd := exec.Command("systemctl", args...) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + + return cmd.Run() +} + +func renderHexwallUnit(binary string, args []string, withZeek bool) string { + after, wants := "network-online.target docker.service", "network-online.target" + if withZeek { + // Wants starts Zeek with hexwall; After makes hexwall wait for it, so ssl.log exists. + after += " " + zeekUnit + wants += " " + zeekUnit + } + + execStart := make([]string, 0, len(args)+1) + for _, arg := range append([]string{binary}, args...) { + execStart = append(execStart, quoteExecArg(arg)) + } + + return marker + ` +[Unit] +Description=hexwall outbound connection monitor +After=` + after + ` +Wants=` + wants + ` + +[Service] +ExecStart=` + strings.Join(execStart, " ") + ` +StateDirectory=hexwall +WorkingDirectory=` + stateDir + ` +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +` +} + +func renderZeekUnit(zeekctl string) string { + zeekctl = quoteExecArg(zeekctl) + // zeekctl starts Zeek in the background and exits, so the unit is a oneshot that stays + // "active" until stopped. + return marker + ` +[Unit] +Description=Zeek network monitor (via zeekctl) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=` + zeekctl + ` deploy +ExecStop=` + zeekctl + ` stop + +[Install] +WantedBy=multi-user.target +` +} + +// quoteExecArg makes arg safe in an ExecStart= line, where systemd splits on whitespace, +// treats % as a specifier and expands $ as an environment variable. +func quoteExecArg(arg string) string { + arg = strings.ReplaceAll(arg, "%", "%%") + + arg = strings.ReplaceAll(arg, "$", "$$") + if arg != "" && !strings.ContainsAny(arg, " \t\"'\\") { + return arg + } + + arg = strings.ReplaceAll(arg, `\`, `\\`) + arg = strings.ReplaceAll(arg, `"`, `\"`) + + return `"` + arg + `"` +} + +// WithSafeMode returns the flags the unit runs hexwall with. A service starts at every +// boot, before anyone is watching, so it runs in watch mode unless the user asked for a +// mode explicitly. +func WithSafeMode(args []string) []string { + for _, arg := range args { + // Go's flag package accepts one or two dashes, with the value attached by = or not. + if arg == "--mode" || arg == "-mode" || strings.HasPrefix(arg, "--mode=") || strings.HasPrefix(arg, "-mode=") { + return args + } + } + + return append([]string{"--mode", "watch"}, args...) +} diff --git a/internal/service/service_test.go b/internal/service/service_test.go new file mode 100644 index 0000000..d951c8a --- /dev/null +++ b/internal/service/service_test.go @@ -0,0 +1,180 @@ +package service + +import ( + "bytes" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestWithSafeMode(t *testing.T) { + tests := []struct { + name string + args []string + want []string + }{ + {"no flags", nil, []string{"--mode", "watch"}}, + {"other flags only", []string{"--debug"}, []string{"--mode", "watch", "--debug"}}, + {"explicit enforce", []string{"--mode", "enforce"}, []string{"--mode", "enforce"}}, + {"equals form", []string{"--mode=enforce", "--debug"}, []string{"--mode=enforce", "--debug"}}, + {"single dash", []string{"-mode", "enforce"}, []string{"-mode", "enforce"}}, + {"single dash equals", []string{"-mode=watch"}, []string{"-mode=watch"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := WithSafeMode(tt.args); !slices.Equal(got, tt.want) { + t.Fatalf("WithSafeMode(%q) = %q, want %q", tt.args, got, tt.want) + } + }) + } +} + +func TestQuoteExecArg(t *testing.T) { + tests := map[string]string{ + "/usr/local/bin/hexwall": "/usr/local/bin/hexwall", + "/path with space/x": `"/path with space/x"`, + "100%": "100%%", + "$HOME": "$$HOME", + `a"b`: `"a\"b"`, + "": `""`, + } + for in, want := range tests { + if got := quoteExecArg(in); got != want { + t.Errorf("quoteExecArg(%q) = %q, want %q", in, got, want) + } + } +} + +func TestRenderHexwallUnit(t *testing.T) { + unit := renderHexwallUnit("/usr/local/bin/hexwall", []string{"--mode", "watch", "--zeek-log", "/opt/zeek/logs/current/ssl.log"}, true) + for _, want := range []string{ + marker + "\n", + "ExecStart=/usr/local/bin/hexwall --mode watch --zeek-log /opt/zeek/logs/current/ssl.log\n", + "After=network-online.target docker.service zeek.service\n", + "Wants=network-online.target zeek.service\n", + "WorkingDirectory=/var/lib/hexwall\n", + "Restart=on-failure\n", + } { + if !strings.Contains(unit, want) { + t.Errorf("unit is missing %q:\n%s", want, unit) + } + } + + if strings.Contains(renderHexwallUnit("/x", nil, false), "zeek") { + t.Error("unit without Zeek still references zeek.service") + } +} + +type fakeSystemctl struct{ calls []string } + +func (f *fakeSystemctl) run(args ...string) error { + f.calls = append(f.calls, strings.Join(args, " ")) + return nil +} + +func TestInstall_WritesUnitsAndStartsHexwall(t *testing.T) { + dir := t.TempDir() + systemctl := &fakeSystemctl{} + opts := Options{Binary: "/usr/local/bin/hexwall", Args: []string{"--mode", "watch"}, Zeekctl: "/opt/zeek/bin/zeekctl"} + + var out bytes.Buffer + if err := install(&out, opts, dir, nil, systemctl.run); err != nil { + t.Fatalf("install() error = %v", err) + } + + for _, unit := range []string{hexwallUnit, zeekUnit} { + if _, err := os.Stat(filepath.Join(dir, unit)); err != nil { + t.Errorf("%s not written: %v", unit, err) + } + } + + want := []string{"daemon-reload", "enable --now zeek.service", "enable hexwall.service", "restart hexwall.service"} + if !slices.Equal(systemctl.calls, want) { + t.Fatalf("systemctl calls = %q, want %q", systemctl.calls, want) + } +} + +// A zeek.service someone else wrote (a distro package, or by hand) must survive untouched. +func TestInstall_KeepsForeignZeekUnit(t *testing.T) { + dir := t.TempDir() + + foreign := "[Service]\nExecStart=/usr/bin/zeek -i eth1\n" + if err := os.WriteFile(filepath.Join(dir, zeekUnit), []byte(foreign), 0o644); err != nil { + t.Fatal(err) + } + + opts := Options{Binary: "/x", Zeekctl: "/opt/zeek/bin/zeekctl"} + if err := install(&bytes.Buffer{}, opts, dir, nil, (&fakeSystemctl{}).run); err != nil { + t.Fatalf("install() error = %v", err) + } + + got, _ := os.ReadFile(filepath.Join(dir, zeekUnit)) + if string(got) != foreign { + t.Fatalf("foreign zeek.service was overwritten:\n%s", got) + } + + hexwall, _ := os.ReadFile(filepath.Join(dir, hexwallUnit)) + if !strings.Contains(string(hexwall), "After=network-online.target docker.service zeek.service") { + t.Fatalf("hexwall.service doesn't order itself after the existing zeek.service:\n%s", hexwall) + } +} + +func TestInstall_WithoutZeek(t *testing.T) { + dir := t.TempDir() + + systemctl := &fakeSystemctl{} + if err := install(&bytes.Buffer{}, Options{Binary: "/x"}, dir, nil, systemctl.run); err != nil { + t.Fatalf("install() error = %v", err) + } + + if _, err := os.Stat(filepath.Join(dir, zeekUnit)); !os.IsNotExist(err) { + t.Fatalf("zeek.service written although hexwall doesn't use Zeek (err %v)", err) + } + + if slices.Contains(systemctl.calls, "enable --now zeek.service") { + t.Fatalf("zeek.service enabled although hexwall doesn't use Zeek: %q", systemctl.calls) + } +} + +func TestUninstall_RemovesOnlyHexwallsUnits(t *testing.T) { + dir := t.TempDir() + if err := install(&bytes.Buffer{}, Options{Binary: "/x"}, dir, nil, (&fakeSystemctl{}).run); err != nil { + t.Fatal(err) + } + + foreign := "[Service]\nExecStart=/usr/bin/zeek -i eth1\n" + if err := os.WriteFile(filepath.Join(dir, zeekUnit), []byte(foreign), 0o644); err != nil { + t.Fatal(err) + } + + systemctl := &fakeSystemctl{} + if err := uninstall(&bytes.Buffer{}, dir, systemctl.run); err != nil { + t.Fatalf("uninstall() error = %v", err) + } + + if _, err := os.Stat(filepath.Join(dir, hexwallUnit)); !os.IsNotExist(err) { + t.Fatalf("hexwall.service still there (err %v)", err) + } + + if got, _ := os.ReadFile(filepath.Join(dir, zeekUnit)); string(got) != foreign { + t.Fatalf("foreign zeek.service changed:\n%s", got) + } + + want := []string{"disable --now hexwall.service", "daemon-reload"} + if !slices.Equal(systemctl.calls, want) { + t.Fatalf("systemctl calls = %q, want %q", systemctl.calls, want) + } +} + +func TestUninstall_NothingInstalled(t *testing.T) { + systemctl := &fakeSystemctl{} + if err := uninstall(&bytes.Buffer{}, t.TempDir(), systemctl.run); err != nil { + t.Fatalf("uninstall() error = %v", err) + } + + if len(systemctl.calls) != 0 { + t.Fatalf("systemctl called with nothing installed: %q", systemctl.calls) + } +} diff --git a/internal/somo/somo.go b/internal/somo/somo.go deleted file mode 100644 index 79d9a73..0000000 --- a/internal/somo/somo.go +++ /dev/null @@ -1,78 +0,0 @@ -// Package somo wraps the somo CLI for monitoring and managing network connections. -package somo - -import ( - "encoding/json" - "fmt" - "os/exec" - "strings" -) - -// Connection matches somo's default JSON output. -type Connection struct { - Protocol string `json:"proto"` - LPort string `json:"local_port"` - RAddress string `json:"remote_address"` - RPort string `json:"remote_port"` - Program string `json:"program"` - PID string `json:"pid"` - State string `json:"state"` - AddressType string `json:"address_type"` -} - -// CheckInstalled reports whether the somo CLI is available in PATH. -func CheckInstalled() error { - _, err := exec.LookPath("somo") - if err != nil { - return fmt.Errorf("somo is not installed in the path: %w", err) - } - - return nil -} - -// GetEstablishedConnections returns a list of established TCP/UDP connections. -func GetEstablishedConnections() ([]Connection, error) { - cmd := exec.Command("somo", "-e", "--json", "--no-pager") - output, err := cmd.CombinedOutput() - if err != nil { - return nil, commandError("failed to run somo", err, output) - } - - var result []Connection - if err := json.Unmarshal(output, &result); err != nil { - return nil, fmt.Errorf("failed to parse somo JSON: %w", err) - } - - return result, nil -} - -// KillConnection kills a process by PID via somo. -func KillConnection(pid string) error { - pid = strings.TrimSpace(pid) - if pid == "" || pid == "-" { - return fmt.Errorf("invalid PID %q", pid) - } - - for _, r := range pid { - if r < '0' || r > '9' { - return fmt.Errorf("invalid PID %q", pid) - } - } - - cmd := exec.Command("somo", "-k", "-p", pid) - output, err := cmd.CombinedOutput() - if err != nil { - return commandError(fmt.Sprintf("failed to kill connection with PID %s", pid), err, output) - } - - return nil -} - -func commandError(message string, err error, output []byte) error { - trimmed := strings.TrimSpace(string(output)) - if trimmed == "" { - return fmt.Errorf("%s: %w", message, err) - } - - return fmt.Errorf("%s: %w: %s", message, err, trimmed) -} diff --git a/internal/store/store.go b/internal/store/store.go index 5b03123..a115409 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -7,7 +7,9 @@ import ( "database/sql" "errors" "fmt" + "log/slog" "net/url" + "strings" "time" // Register the pure-Go SQLite driver used for the local hexwall database. @@ -19,6 +21,12 @@ const ( establishedTrustWindow = time.Minute fraudCheckCacheWindow = 6 * time.Hour sqliteBusyTimeout = 5 * time.Second + + // Prune windows: each table is deleted on a different schedule. + pruneSNIWindow = 7 * 24 * time.Hour // SNI observations: operational state data; 7 days is enough for hand-inspection. + pruneKilledAlertsWindow = 90 * 24 * time.Hour // Audit tables (killed_connections, zeek_alerts): rare events, forensically valuable. + // fraud_checks and domain_checks are bounded by upsert-per-key and fraudCheckCacheWindow (6h); + // they never grow beyond one row per IP/domain and are self-cleaning on read, so pruning is unnecessary. ) const schema = ` @@ -30,12 +38,21 @@ CREATE TABLE IF NOT EXISTS allowed_ips ( last_established INTEGER ); +CREATE TABLE IF NOT EXISTS allowed_ip_domains ( + ip TEXT NOT NULL, + domain TEXT NOT NULL, + first_seen INTEGER NOT NULL, + last_refreshed INTEGER NOT NULL, + PRIMARY KEY (ip, domain) +); + CREATE TABLE IF NOT EXISTS killed_connections ( id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, pid TEXT NOT NULL, program TEXT NOT NULL, - killed_at INTEGER NOT NULL + killed_at INTEGER NOT NULL, + outcome TEXT NOT NULL DEFAULT 'unknown' ); CREATE TABLE IF NOT EXISTS fraud_checks ( @@ -43,6 +60,53 @@ CREATE TABLE IF NOT EXISTS fraud_checks ( should_kill INTEGER NOT NULL, checked_at INTEGER NOT NULL ); + +CREATE TABLE IF NOT EXISTS sni_observations ( + ip TEXT NOT NULL, + domain TEXT NOT NULL, + local_port TEXT NOT NULL, + outcome TEXT NOT NULL, + first_seen INTEGER NOT NULL, + last_seen INTEGER NOT NULL, + times_seen INTEGER NOT NULL, + PRIMARY KEY (ip, domain, local_port) +); + +CREATE TABLE IF NOT EXISTS zeek_alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + src_ip TEXT NOT NULL, + dst_ip TEXT NOT NULL, + dst_port TEXT NOT NULL, + sni TEXT NOT NULL, + blocked INTEGER NOT NULL, + confidence TEXT NOT NULL, + detected_at INTEGER NOT NULL, + action_taken TEXT +); + +CREATE TABLE IF NOT EXISTS domain_checks ( + domain TEXT PRIMARY KEY, + should_block INTEGER NOT NULL, + checked_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS ip_observations ( + ip TEXT NOT NULL, + program TEXT NOT NULL, + outcome TEXT NOT NULL, + first_seen INTEGER NOT NULL, + last_seen INTEGER NOT NULL, + times_seen INTEGER NOT NULL, + PRIMARY KEY (ip, program) +); +` + +// backfillAllowedIPDomains seeds the per-IP domain set from the older IP-level trust table. +// It runs on every startup: INSERT OR IGNORE makes it idempotent, and without it an upgraded +// install would treat every Zeek-observed SNI as a mismatch until the first cache refresh. +const backfillAllowedIPDomains = ` +INSERT OR IGNORE INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) +SELECT ip, domain, first_approved, last_refreshed FROM allowed_ips WHERE TRIM(domain) <> ''; ` // FraudCheckCacheEntry stores the cached kill decision for a prior fraud API lookup. @@ -51,12 +115,172 @@ type FraudCheckCacheEntry struct { CheckedAt int64 } +// DomainCheckCacheEntry stores the cached block decision for a prior domain reputation lookup. +type DomainCheckCacheEntry struct { + ShouldBlock bool + CheckedAt int64 +} + +// SNIOutcome records which rung of the decision ladder classified a connection. +type SNIOutcome string + +const ( + // OutcomePolicyAllowed indicates the connection was allowed by Pi-hole policy. + OutcomePolicyAllowed SNIOutcome = "policy-allowed" + // OutcomePolicyBlocked indicates the connection was blocked by Pi-hole policy. + OutcomePolicyBlocked SNIOutcome = "policy-blocked" + // OutcomeIPDomainMatch indicates the SNI domain matched a known domain for the IP. + OutcomeIPDomainMatch SNIOutcome = "ip-domain-match" + // OutcomeUnknown indicates the connection could not be classified. + OutcomeUnknown SNIOutcome = "unknown" +) + +// Valid reports whether the outcome is one of the known values. +func (o SNIOutcome) Valid() bool { + switch o { + case OutcomePolicyAllowed, OutcomePolicyBlocked, OutcomeIPDomainMatch, OutcomeUnknown: + return true + } + + return false +} + +// ParseSNIOutcome converts a string read from the database into a typed SNIOutcome. +// Rows written by older builds may contain unrecognized values; those parse to OutcomeUnknown with false. +func ParseSNIOutcome(s string) (SNIOutcome, bool) { + o := SNIOutcome(s) + if o.Valid() { + return o, true + } + + return OutcomeUnknown, false +} + +// IPOutcome records how a connection was classified on the IP-only fallback path. +type IPOutcome string + +const ( + // OutcomeIPTrusted indicates the IP is present and fresh in allowed_ips. + OutcomeIPTrusted IPOutcome = "ip-trusted" + // OutcomeIPReserved indicates deghost returned 403: private or reserved address. + OutcomeIPReserved IPOutcome = "private-reserved" + // OutcomeIPBlocked indicates deghost verdict said kill. + OutcomeIPBlocked IPOutcome = "deghost-blocked" + // OutcomeIPUnknown indicates no verdict available or check disabled. + OutcomeIPUnknown IPOutcome = "unknown" +) + +// Valid reports whether the outcome is one of the known values. +func (o IPOutcome) Valid() bool { + switch o { + case OutcomeIPTrusted, OutcomeIPReserved, OutcomeIPBlocked, OutcomeIPUnknown: + return true + } + + return false +} + +// ParseIPOutcome converts a string read from the database into a typed IPOutcome. +// Rows written by older builds may contain unrecognized values; those parse to OutcomeIPUnknown with false. +func ParseIPOutcome(s string) (IPOutcome, bool) { + o := IPOutcome(s) + if o.Valid() { + return o, true + } + + return OutcomeIPUnknown, false +} + +// ZeekAlertEntry stores a Zeek-derived alert that was persisted for audit. +type ZeekAlertEntry struct { + ID int64 + SrcIP string + DstIP string + DstPort string + SNI string + Blocked bool + Confidence string + DetectedAt int64 + ActionTaken string +} + // Store wraps the local hexwall database. type Store struct { readWrite *sql.DB readOnly *sql.DB } +// migrateSNITable detects the old event-log schema (identified by a "known" column) +// and drops it so the new state-table schema is applied by the CREATE TABLE IF NOT EXISTS. +// This is safe specifically because nothing in the codebase reads this table; a future reader +// must not assume this reasoning generalises to other tables. +func migrateSNITable(db *sql.DB) error { + cols, err := db.Query(`PRAGMA table_info(sni_observations)`) + if err != nil { + return fmt.Errorf("query table info: %w", err) + } + defer func() { + if err := cols.Close(); err != nil { + slog.Debug("failed to close columns", "err", err) + } + }() + + hasOldSchema := false + + for cols.Next() { + var ( + cid int + name, ctype string + notNull int + dfltValue sql.NullString + pk int + ) + if err := cols.Scan(&cid, &name, &ctype, ¬Null, &dfltValue, &pk); err != nil { + return fmt.Errorf("scan table info: %w", err) + } + + if name == "known" { + hasOldSchema = true + } + } + + if err := cols.Err(); err != nil { + return fmt.Errorf("iterate table info: %w", err) + } + + if !hasOldSchema { + return nil + } + + slog.Warn("dropping old sni_observations table (write-only audit data; no readers)", "reason", "schema migration to state-table format") + + if _, err := db.Exec(`DROP TABLE sni_observations`); err != nil { + return fmt.Errorf("drop old sni_observations: %w", err) + } + + return nil +} + +// migrateKillOutcome adds the outcome column to a killed_connections table created before +// it existed. Rows written before then are left as 'unknown': at the time, a refused or +// failed kill was recorded the same way as a real one. +func migrateKillOutcome(db *sql.DB) error { + var has int + if err := db.QueryRow(`SELECT COUNT(*) FROM pragma_table_info('killed_connections') WHERE name = 'outcome'`).Scan(&has); err != nil { + return fmt.Errorf("query table info: %w", err) + } + + if has > 0 { + return nil + } + + if _, err := db.Exec(`ALTER TABLE killed_connections ADD COLUMN outcome TEXT NOT NULL DEFAULT 'unknown'`); err != nil { + return fmt.Errorf("add outcome column: %w", err) + } + + return nil +} + // NewStore opens or creates the hexwall database at dbPath and applies the schema. func NewStore(dbPath string) (*Store, error) { readWrite, err := sql.Open("sqlite", sqliteDSN(dbPath, "rwc")) @@ -74,6 +298,31 @@ func NewStore(dbPath string) (*Store, error) { return nil, fmt.Errorf("failed to apply schema: %w", err) } + // Migration: detect and replace old sni_observations event-log schema with the new state table. + // The old table used an autoincrement id + known INTEGER, which is structurally incompatible + // with the new (ip, domain, local_port) primary key. This is safe to drop because nothing in + // the codebase reads this table — it was only ever written to and consulted by hand. + if err := migrateSNITable(readWrite); err != nil { + _ = readWrite.Close() + return nil, fmt.Errorf("failed to migrate sni_observations: %w", err) + } + + // Re-apply schema after migration so that any table dropped by migrateSNITable is recreated. + if _, err := readWrite.Exec(schema); err != nil { + _ = readWrite.Close() + return nil, fmt.Errorf("failed to re-apply schema after migration: %w", err) + } + + if err := migrateKillOutcome(readWrite); err != nil { + _ = readWrite.Close() + return nil, fmt.Errorf("failed to migrate killed_connections: %w", err) + } + + if _, err := readWrite.Exec(backfillAllowedIPDomains); err != nil { + _ = readWrite.Close() + return nil, fmt.Errorf("failed to backfill allowed ip domains: %w", err) + } + readOnly, err := sql.Open("sqlite", sqliteDSN(dbPath, "ro")) if err != nil { _ = readWrite.Close() @@ -83,6 +332,7 @@ func NewStore(dbPath string) (*Store, error) { if err := configureConnection(readOnly, true); err != nil { _ = readOnly.Close() _ = readWrite.Close() + return nil, fmt.Errorf("failed to connect to hexwall db read-only connection: %w", err) } @@ -93,7 +343,10 @@ func sqliteDSN(dbPath, mode string) string { query := url.Values{} query.Set("mode", mode) - return "file:" + (&url.URL{Path: dbPath, RawQuery: query.Encode()}).String() + return "file:" + (&url.URL{ + Path: dbPath, + RawQuery: query.Encode(), + }).String() } func configureConnection(db *sql.DB, readOnly bool) error { @@ -125,8 +378,10 @@ func (s *Store) Close() error { return nil } - var errReadWrite error - var errReadOnly error + var ( + errReadWrite error + errReadOnly error + ) if s.readWrite != nil { errReadWrite = s.readWrite.Close() @@ -151,7 +406,6 @@ func (s *Store) UpsertAllowedIP(ip, domain string) error { domain = excluded.domain, last_refreshed = excluded.last_refreshed `, ip, domain, now, now) - if err != nil { return fmt.Errorf("upsert allowed ip %s: %w", ip, err) } @@ -159,13 +413,70 @@ func (s *Store) UpsertAllowedIP(ip, domain string) error { return nil } +// UpsertAllowedIPDomain inserts or refreshes one domain in the set of domains known for an IP. +// Unlike allowed_ips, which keeps a single representative domain per IP, this table records +// every domain observed for the IP so an SNI can be matched against that IP specifically. +// On conflict, it updates last_refreshed while preserving first_seen. +func (s *Store) UpsertAllowedIPDomain(ip, domain string) error { + domain = strings.ToLower(strings.TrimSpace(domain)) + now := time.Now().Unix() + + _, err := s.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + ON CONFLICT(ip, domain) DO UPDATE SET + last_refreshed = excluded.last_refreshed + `, ip, domain, now, now) + if err != nil { + return fmt.Errorf("upsert allowed ip domain %s/%s: %w", ip, domain, err) + } + + return nil +} + +// DomainsForIP returns every domain on file for the IP that was refreshed within the trust window. +// It returns an empty slice when the IP has no fresh domains on record. +func (s *Store) DomainsForIP(ip string) ([]string, error) { + cutoff := time.Now().Add(-refreshTrustWindow).Unix() + + rows, err := s.readOnly.Query(` + SELECT domain + FROM allowed_ip_domains + WHERE ip = ? + AND last_refreshed >= ? + ORDER BY domain + `, ip, cutoff) + if err != nil { + return nil, fmt.Errorf("domains for ip %s: %w", ip, err) + } + defer func() { + _ = rows.Close() + }() + + domains := []string{} + + for rows.Next() { + var domain string + if err := rows.Scan(&domain); err != nil { + return nil, fmt.Errorf("scan domain for ip %s: %w", ip, err) + } + + domains = append(domains, domain) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate domains for ip %s: %w", ip, err) + } + + return domains, nil +} + // UpdateEstablished stamps the current time as last_established for an IP. // The monitor calls it when somo confirms the connection is still active. func (s *Store) UpdateEstablished(ip string) error { _, err := s.readWrite.Exec(` UPDATE allowed_ips SET last_established = ? WHERE ip = ? `, time.Now().Unix(), ip) - if err != nil { return fmt.Errorf("update established %s: %w", ip, err) } @@ -185,6 +496,7 @@ func (s *Store) IsAllowed(ip string) (bool, error) { establishedCutoff := now.Add(-establishedTrustWindow).Unix() var found int + err := s.readOnly.QueryRow(` SELECT 1 FROM allowed_ips WHERE ip = ? @@ -208,8 +520,11 @@ func (s *Store) IsAllowed(ip string) (bool, error) { func (s *Store) GetRecentFraudCheck(ip string) (*FraudCheckCacheEntry, error) { cutoff := time.Now().Add(-fraudCheckCacheWindow).Unix() - var shouldKill int - var checkedAt int64 + var ( + shouldKill int + checkedAt int64 + ) + err := s.readOnly.QueryRow(` SELECT should_kill, checked_at FROM fraud_checks @@ -246,7 +561,6 @@ func (s *Store) UpsertFraudCheck(ip string, shouldKill bool) error { should_kill = excluded.should_kill, checked_at = excluded.checked_at `, ip, shouldKillInt, time.Now().Unix()) - if err != nil { return fmt.Errorf("upsert fraud check %s: %w", ip, err) } @@ -254,16 +568,264 @@ func (s *Store) UpsertFraudCheck(ip string, shouldKill bool) error { return nil } -// LogKill records a killed connection in the audit log. -func (s *Store) LogKill(ip, pid, program string) error { +// GetRecentDomainCheck returns the cached domain-check decision when it was recorded within the cache window. +func (s *Store) GetRecentDomainCheck(domain string) (*DomainCheckCacheEntry, error) { + domain = strings.ToLower(strings.TrimSpace(domain)) + cutoff := time.Now().Add(-fraudCheckCacheWindow).Unix() + + var ( + shouldBlock int + checkedAt int64 + ) + + err := s.readOnly.QueryRow(` + SELECT should_block, checked_at + FROM domain_checks + WHERE domain = ? + AND checked_at >= ? + LIMIT 1 + `, domain, cutoff).Scan(&shouldBlock, &checkedAt) + + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + + if err != nil { + return nil, fmt.Errorf("get recent domain check for %s: %w", domain, err) + } + + return &DomainCheckCacheEntry{ + ShouldBlock: shouldBlock != 0, + CheckedAt: checkedAt, + }, nil +} + +// UpsertDomainCheck stores the current domain-check decision. +func (s *Store) UpsertDomainCheck(domain string, shouldBlock bool) error { + domain = strings.ToLower(strings.TrimSpace(domain)) + shouldBlockInt := 0 + + if shouldBlock { + shouldBlockInt = 1 + } + _, err := s.readWrite.Exec(` - INSERT INTO killed_connections (ip, pid, program, killed_at) - VALUES (?, ?, ?, ?) - `, ip, pid, program, time.Now().Unix()) + INSERT INTO domain_checks (domain, should_block, checked_at) + VALUES (?, ?, ?) + ON CONFLICT(domain) DO UPDATE SET + should_block = excluded.should_block, + checked_at = excluded.checked_at + `, domain, shouldBlockInt, time.Now().Unix()) + if err != nil { + return fmt.Errorf("upsert domain check %s: %w", domain, err) + } + + return nil +} + +// LogKill records a kill about to be attempted, with outcome KillAttempting, and returns +// the row id for SetKillOutcome. Writing first keeps the record even if the kill never +// returns; SetKillOutcome then says whether it happened. +func (s *Store) LogKill(ip, pid, program string) (int64, error) { + result, err := s.readWrite.Exec(` + INSERT INTO killed_connections (ip, pid, program, killed_at, outcome) + VALUES (?, ?, ?, ?, ?) + `, ip, pid, program, time.Now().Unix(), KillAttempting) + if err != nil { + return 0, fmt.Errorf("log kill %s: %w", ip, err) + } + + id, err := result.LastInsertId() + if err != nil { + return 0, fmt.Errorf("log kill %s: read row id: %w", ip, err) + } + + return id, nil +} + +// KillOutcome is what happened to a kill recorded by LogKill. +type KillOutcome string + +const ( + // KillAttempting is written before the kill, so the row survives if hexwall dies mid-kill. + KillAttempting KillOutcome = "attempting" + // KillKilled means the connection, or its process, is gone. + KillKilled KillOutcome = "killed" + // KillRefused means a guardrail declined it on purpose (PID 1, hexwall itself, sshd...). + KillRefused KillOutcome = "refused" + // KillFailed means it couldn't be done: no usable PID, or the kill itself errored. + KillFailed KillOutcome = "failed" +) + +// SetKillOutcome records how the kill logged as row id ended. +func (s *Store) SetKillOutcome(id int64, outcome KillOutcome) error { + if _, err := s.readWrite.Exec(`UPDATE killed_connections SET outcome = ? WHERE id = ?`, outcome, id); err != nil { + return fmt.Errorf("set kill outcome %d: %w", id, err) + } + + return nil +} + +// LogZeekAlert persists a Zeek-derived event for later review. +func (s *Store) LogZeekAlert(srcIP, dstIP, dstPort, sni string, blocked bool, confidence, actionTaken string) error { + blockedInt := 0 + if blocked { + blockedInt = 1 + } + + _, err := s.readWrite.Exec(` + INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, srcIP, dstIP, dstPort, sni, blockedInt, confidence, time.Now().Unix(), actionTaken) + if err != nil { + return fmt.Errorf("log zeek alert %s: %w", srcIP, err) + } + + return nil +} + +// RecentZeekAlert returns the most recent Zeek alert for the given source IP and SNI. +func (s *Store) RecentZeekAlert(srcIP, sni string) (*ZeekAlertEntry, error) { + cutoff := time.Now().Add(-time.Hour).Unix() + + var ( + id int64 + dstIP string + dstPort string + blockedInt int + confidence string + detectedAt int64 + actionTaken string + ) + + err := s.readOnly.QueryRow(` + SELECT id, dst_ip, dst_port, blocked, confidence, detected_at, action_taken + FROM zeek_alerts + WHERE src_ip = ? + AND sni = ? + AND detected_at >= ? + ORDER BY id DESC + LIMIT 1 + `, srcIP, sni, cutoff).Scan(&id, &dstIP, &dstPort, &blockedInt, &confidence, &detectedAt, &actionTaken) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + + if err != nil { + return nil, fmt.Errorf("get recent zeek alert for %s: %w", srcIP, err) + } + + return &ZeekAlertEntry{ + ID: id, + SrcIP: srcIP, + DstIP: dstIP, + DstPort: dstPort, + SNI: sni, + Blocked: blockedInt != 0, + Confidence: confidence, + DetectedAt: detectedAt, + ActionTaken: actionTaken, + }, nil +} + +// LogSNIObservation records an SNI domain decision as a state-table upsert. +// On first sight it inserts the row; on subsequent observations it overwrites the outcome, +// updates last_seen, and increments times_seen while preserving first_seen. +func (s *Store) LogSNIObservation(ip, domain, localPort string, outcome SNIOutcome) error { + domain = strings.ToLower(strings.TrimSpace(domain)) + now := time.Now().Unix() + + _, err := s.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?, 1) + ON CONFLICT(ip, domain, local_port) DO UPDATE SET + outcome = excluded.outcome, + last_seen = excluded.last_seen, + times_seen = sni_observations.times_seen + 1 + `, ip, domain, localPort, string(outcome), now, now) + if err != nil { + return fmt.Errorf("log sni observation %s: %w", ip, err) + } + + return nil +} + +// LogIPObservation records an IP-only fallback decision as a state-table upsert. +// On first sight it inserts the row; on subsequent observations it overwrites the outcome, +// updates last_seen, and increments times_seen while preserving first_seen. +func (s *Store) LogIPObservation(ip, program string, outcome IPOutcome) error { + now := time.Now().Unix() + _, err := s.readWrite.Exec(` + INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, 1) + ON CONFLICT(ip, program) DO UPDATE SET + outcome = excluded.outcome, + last_seen = excluded.last_seen, + times_seen = ip_observations.times_seen + 1 + `, ip, program, string(outcome), now, now) if err != nil { - return fmt.Errorf("log kill %s: %w", ip, err) + return fmt.Errorf("log ip observation %s: %w", ip, err) } return nil } + +// PruneResult reports how many rows were removed per table in a single Prune call. +type PruneResult struct { + AllowedIPDomains int64 + SNIObservations int64 + IPObservations int64 + KilledConnections int64 + ZeekAlerts int64 +} + +// Prune removes stale rows from all pruneable tables. +// It returns the number of rows deleted per table so the caller can log a summary. +func (s *Store) Prune() (*PruneResult, error) { + now := time.Now().Unix() + result := &PruneResult{} + + // allowed_ip_domains: rows older than refreshTrustWindow are already excluded by + // DomainsForIP and can never grant trust again. Safe to remove. + r, err := s.readWrite.Exec(`DELETE FROM allowed_ip_domains WHERE last_refreshed < ?`, now-int64(refreshTrustWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune allowed_ip_domains: %w", err) + } + + result.AllowedIPDomains, _ = r.RowsAffected() + + // sni_observations: operational state data with a 7-day window. + r, err = s.readWrite.Exec(`DELETE FROM sni_observations WHERE last_seen < ?`, now-int64(pruneSNIWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune sni_observations: %w", err) + } + + result.SNIObservations, _ = r.RowsAffected() + + // ip_observations: same 7-day window as sni_observations. + r, err = s.readWrite.Exec(`DELETE FROM ip_observations WHERE last_seen < ?`, now-int64(pruneSNIWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune ip_observations: %w", err) + } + + result.IPObservations, _ = r.RowsAffected() + + // killed_connections and zeek_alerts: audit records of rare, potentially forensically + // valuable events. 90-day window keeps a meaningful history without unbounded growth. + r, err = s.readWrite.Exec(`DELETE FROM killed_connections WHERE killed_at < ?`, now-int64(pruneKilledAlertsWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune killed_connections: %w", err) + } + + result.KilledConnections, _ = r.RowsAffected() + + r, err = s.readWrite.Exec(`DELETE FROM zeek_alerts WHERE detected_at < ?`, now-int64(pruneKilledAlertsWindow.Seconds())) + if err != nil { + return nil, fmt.Errorf("prune zeek_alerts: %w", err) + } + + result.ZeekAlerts, _ = r.RowsAffected() + + return result, nil +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 15acf98..f411dac 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1,6 +1,7 @@ package store import ( + "database/sql" "path/filepath" "testing" "time" @@ -10,6 +11,7 @@ func TestSQLiteDSNRelativePath(t *testing.T) { t.Parallel() dsn := sqliteDSN("./hexwall.db", "rwc") + want := "file:./hexwall.db?mode=rwc" if dsn != want { t.Fatalf("sqliteDSN() = %q, want %q", dsn, want) @@ -20,6 +22,7 @@ func TestSQLiteDSNAbsolutePath(t *testing.T) { t.Parallel() dsn := sqliteDSN("/tmp/hexwall.db", "ro") + want := "file:/tmp/hexwall.db?mode=ro" if dsn != want { t.Fatalf("sqliteDSN() = %q, want %q", dsn, want) @@ -43,6 +46,7 @@ func TestFraudCheckCacheRoundTrip(t *testing.T) { if err != nil { t.Fatalf("GetRecentFraudCheck() error = %v", err) } + if entry != nil { t.Fatalf("GetRecentFraudCheck() = %#v, want nil before insert", entry) } @@ -55,12 +59,15 @@ func TestFraudCheckCacheRoundTrip(t *testing.T) { if err != nil { t.Fatalf("GetRecentFraudCheck() after insert error = %v", err) } + if entry == nil { t.Fatal("GetRecentFraudCheck() = nil, want cached entry") } + if !entry.ShouldKill { t.Fatal("GetRecentFraudCheck().ShouldKill = false, want true") } + if entry.CheckedAt <= 0 { t.Fatalf("GetRecentFraudCheck().CheckedAt = %d, want positive unix timestamp", entry.CheckedAt) } @@ -91,7 +98,1085 @@ func TestFraudCheckCacheExpiresAfterWindow(t *testing.T) { if err != nil { t.Fatalf("GetRecentFraudCheck() error = %v", err) } + if entry != nil { t.Fatalf("GetRecentFraudCheck() = %#v, want nil for stale entry", entry) } } + +func TestAllowedIPDomainsMultiplePerIP(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + for _, domain := range []string{"github.com", "example.com", "EXAMPLE.NET"} { + if err := hexwallStore.UpsertAllowedIPDomain("203.0.113.30", domain); err != nil { + t.Fatalf("UpsertAllowedIPDomain(%q) error = %v", domain, err) + } + } + + domains, err := hexwallStore.DomainsForIP("203.0.113.30") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + + want := []string{"example.com", "example.net", "github.com"} + if len(domains) != len(want) { + t.Fatalf("DomainsForIP() = %v, want %v", domains, want) + } + + for i, domain := range want { + if domains[i] != domain { + t.Fatalf("DomainsForIP()[%d] = %q, want %q", i, domains[i], domain) + } + } +} + +func TestDomainsForIPUnknownIP(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + if err := hexwallStore.UpsertAllowedIPDomain("203.0.113.30", "github.com"); err != nil { + t.Fatalf("UpsertAllowedIPDomain() error = %v", err) + } + + domains, err := hexwallStore.DomainsForIP("203.0.113.99") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + + if len(domains) != 0 { + t.Fatalf("DomainsForIP() = %v, want empty for unknown ip", domains) + } +} + +func TestDomainsForIPExcludesStaleEntries(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + staleRefreshed := time.Now().Add(-refreshTrustWindow - time.Minute).Unix() + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.40", "stale.example.com", staleRefreshed, staleRefreshed); err != nil { + t.Fatalf("insert stale allowed ip domain error = %v", err) + } + + if err := hexwallStore.UpsertAllowedIPDomain("203.0.113.40", "fresh.example.com"); err != nil { + t.Fatalf("UpsertAllowedIPDomain() error = %v", err) + } + + domains, err := hexwallStore.DomainsForIP("203.0.113.40") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + + if len(domains) != 1 || domains[0] != "fresh.example.com" { + t.Fatalf("DomainsForIP() = %v, want only the fresh domain", domains) + } +} + +func TestUpsertAllowedIPDomainRefreshesWithoutDuplicating(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + originalSeen := time.Now().Add(-30 * time.Minute).Unix() + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.50", "example.com", originalSeen, originalSeen); err != nil { + t.Fatalf("insert allowed ip domain error = %v", err) + } + + if err := hexwallStore.UpsertAllowedIPDomain("203.0.113.50", "example.com"); err != nil { + t.Fatalf("UpsertAllowedIPDomain() error = %v", err) + } + + var ( + rowCount int + firstSeen int64 + lastRefreshed int64 + ) + if err := hexwallStore.readOnly.QueryRow(` + SELECT COUNT(*), MIN(first_seen), MIN(last_refreshed) + FROM allowed_ip_domains + WHERE ip = ? + `, "203.0.113.50").Scan(&rowCount, &firstSeen, &lastRefreshed); err != nil { + t.Fatalf("query allowed ip domain error = %v", err) + } + + if rowCount != 1 { + t.Fatalf("row count = %d, want 1 after re-upsert", rowCount) + } + + if firstSeen != originalSeen { + t.Fatalf("first_seen = %d, want preserved value %d", firstSeen, originalSeen) + } + + if lastRefreshed <= originalSeen { + t.Fatalf("last_refreshed = %d, want a value newer than %d", lastRefreshed, originalSeen) + } +} + +func TestNewStoreBackfillsAllowedIPDomains(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + // UpsertAllowedIP writes only the IP-level record, mimicking a database written + // before allowed_ip_domains existed. + if err := hexwallStore.UpsertAllowedIP("203.0.113.60", "example.com"); err != nil { + t.Fatalf("UpsertAllowedIP() error = %v", err) + } + + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + reopened, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() reopen error = %v", err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + domains, err := reopened.DomainsForIP("203.0.113.60") + if err != nil { + t.Fatalf("DomainsForIP() error = %v", err) + } + + if len(domains) != 1 || domains[0] != "example.com" { + t.Fatalf("DomainsForIP() = %v, want [example.com] from backfill", domains) + } +} + +func TestDomainCheckCacheRoundTrip(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + entry, err := hexwallStore.GetRecentDomainCheck("evil.example.com") + if err != nil { + t.Fatalf("GetRecentDomainCheck() error = %v", err) + } + + if entry != nil { + t.Fatalf("GetRecentDomainCheck() = %#v, want nil before insert", entry) + } + + if err := hexwallStore.UpsertDomainCheck("evil.example.com", true); err != nil { + t.Fatalf("UpsertDomainCheck() error = %v", err) + } + + entry, err = hexwallStore.GetRecentDomainCheck("evil.example.com") + if err != nil { + t.Fatalf("GetRecentDomainCheck() after insert error = %v", err) + } + + if entry == nil { + t.Fatal("GetRecentDomainCheck() = nil, want cached entry") + } + + if !entry.ShouldBlock { + t.Fatal("GetRecentDomainCheck().ShouldBlock = false, want true") + } + + if entry.CheckedAt <= 0 { + t.Fatalf("GetRecentDomainCheck().CheckedAt = %d, want positive unix timestamp", entry.CheckedAt) + } +} + +func TestDomainCheckCacheNormalizedKey(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + if err := hexwallStore.UpsertDomainCheck("EXAMPLE.COM", false); err != nil { + t.Fatalf("UpsertDomainCheck() error = %v", err) + } + + entry, err := hexwallStore.GetRecentDomainCheck(" example.com ") + if err != nil { + t.Fatalf("GetRecentDomainCheck() error = %v", err) + } + + if entry == nil { + t.Fatal("GetRecentDomainCheck() = nil, normalized key lookup should match") + } + + if entry.ShouldBlock { + t.Fatal("GetRecentDomainCheck().ShouldBlock = true, want false") + } +} + +func TestDomainCheckCacheExpiresAfterWindow(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + staleCheckedAt := time.Now().Add(-fraudCheckCacheWindow - time.Minute).Unix() + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO domain_checks (domain, should_block, checked_at) + VALUES (?, ?, ?) + `, "stale.example.com", 1, staleCheckedAt); err != nil { + t.Fatalf("insert stale domain check error = %v", err) + } + + entry, err := hexwallStore.GetRecentDomainCheck("stale.example.com") + if err != nil { + t.Fatalf("GetRecentDomainCheck() error = %v", err) + } + + if entry != nil { + t.Fatalf("GetRecentDomainCheck() = %#v, want nil for stale entry", entry) + } +} + +func TestParseSNIOutcomeValid(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want SNIOutcome + }{ + {"policy-allowed", OutcomePolicyAllowed}, + {"policy-blocked", OutcomePolicyBlocked}, + {"ip-domain-match", OutcomeIPDomainMatch}, + {"unknown", OutcomeUnknown}, + } + + for _, tt := range tests { + got, ok := ParseSNIOutcome(tt.input) + if !ok { + t.Errorf("ParseSNIOutcome(%q) ok = false, want true", tt.input) + } + + if got != tt.want { + t.Errorf("ParseSNIOutcome(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestParseSNIOutcomeInvalid(t *testing.T) { + t.Parallel() + + tests := []string{"", "bogus", "true", "false", "1", "0"} + for _, input := range tests { + got, ok := ParseSNIOutcome(input) + if ok { + t.Errorf("ParseSNIOutcome(%q) ok = true, want false", input) + } + + if got != OutcomeUnknown { + t.Errorf("ParseSNIOutcome(%q) = %q, want %q", input, got, OutcomeUnknown) + } + } +} + +func TestSNIOutcomeValid(t *testing.T) { + t.Parallel() + + valid := []SNIOutcome{OutcomePolicyAllowed, OutcomePolicyBlocked, OutcomeIPDomainMatch, OutcomeUnknown} + for _, o := range valid { + if !o.Valid() { + t.Errorf("SNIOutcome(%q).Valid() = false, want true", o) + } + } + + invalid := []SNIOutcome{"bogus", "", "true"} + for _, o := range invalid { + if o.Valid() { + t.Errorf("SNIOutcome(%q).Valid() = true, want false", o) + } + } +} + +func TestLogSNIObservationInsertAndUpsert(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + // First insert. + if err := hexwallStore.LogSNIObservation("203.0.113.10", "example.com", "443", OutcomePolicyAllowed); err != nil { + t.Fatalf("LogSNIObservation() error = %v", err) + } + + var ( + timesSeen int + firstSeen, lastSeen int64 + outcome string + ) + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, first_seen, last_seen, outcome + FROM sni_observations + WHERE ip = ? AND domain = ? AND local_port = ? + `, "203.0.113.10", "example.com", "443").Scan(×Seen, &firstSeen, &lastSeen, &outcome) + if err != nil { + t.Fatalf("query sni_observations error = %v", err) + } + + if timesSeen != 1 { + t.Fatalf("times_seen = %d, want 1 after first insert", timesSeen) + } + + if firstSeen <= 0 { + t.Fatalf("first_seen = %d, want positive unix timestamp", firstSeen) + } + + if lastSeen != firstSeen { + t.Fatalf("last_seen = %d, want equal to first_seen on first insert, got %d", lastSeen, firstSeen) + } + + if outcome != "policy-allowed" { + t.Fatalf("outcome = %q, want %q", outcome, "policy-allowed") + } + + // Second insert with different outcome — should upsert. + if err := hexwallStore.LogSNIObservation("203.0.113.10", "example.com", "443", OutcomeIPDomainMatch); err != nil { + t.Fatalf("LogSNIObservation() upsert error = %v", err) + } + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, first_seen, last_seen, outcome + FROM sni_observations + WHERE ip = ? AND domain = ? AND local_port = ? + `, "203.0.113.10", "example.com", "443").Scan(×Seen, &firstSeen, &lastSeen, &outcome) + if err != nil { + t.Fatalf("query sni_observations after upsert error = %v", err) + } + + if timesSeen != 2 { + t.Fatalf("times_seen = %d, want 2 after upsert", timesSeen) + } + + if outcome != "ip-domain-match" { + t.Fatalf("outcome = %q, want %q after upsert", outcome, "ip-domain-match") + } + + if lastSeen < firstSeen { + t.Fatalf("last_seen = %d should not be older than first_seen = %d", lastSeen, firstSeen) + } + + // Third insert — times_seen should be 3. + if err := hexwallStore.LogSNIObservation("203.0.113.10", "example.com", "443", OutcomePolicyBlocked); err != nil { + t.Fatalf("LogSNIObservation() third insert error = %v", err) + } + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, outcome + FROM sni_observations + WHERE ip = ? AND domain = ? AND local_port = ? + `, "203.0.113.10", "example.com", "443").Scan(×Seen, &outcome) + if err != nil { + t.Fatalf("query sni_observations after third insert error = %v", err) + } + + if timesSeen != 3 { + t.Fatalf("times_seen = %d, want 3 after third insert", timesSeen) + } + + if outcome != "policy-blocked" { + t.Fatalf("outcome = %q, want %q after third insert", outcome, "policy-blocked") + } +} + +func TestLogSNIObservationNormalizesDomain(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + if err := hexwallStore.LogSNIObservation("203.0.113.10", " EXAMPLE.COM ", "443", OutcomePolicyAllowed); err != nil { + t.Fatalf("LogSNIObservation() error = %v", err) + } + + // Lookup by the normalized form. + var outcome string + + err = hexwallStore.readOnly.QueryRow(` + SELECT outcome FROM sni_observations + WHERE ip = ? AND domain = ? AND local_port = ? + `, "203.0.113.10", "example.com", "443").Scan(&outcome) + if err != nil { + t.Fatalf("query sni_observations for normalized domain error = %v", err) + } + + if outcome != "policy-allowed" { + t.Fatalf("outcome = %q, want %q", outcome, "policy-allowed") + } +} + +func TestNewStoreMigratesOldSNITable(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + // Create a store with the old schema manually. + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + // Drop the new table and recreate with the old schema. + if _, err := hexwallStore.readWrite.Exec(`DROP TABLE sni_observations`); err != nil { + t.Fatalf("drop sni_observations error = %v", err) + } + + if _, err := hexwallStore.readWrite.Exec(` + CREATE TABLE sni_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL, domain TEXT NOT NULL, local_port TEXT NOT NULL, + known INTEGER NOT NULL, observed_at INTEGER NOT NULL + ) + `); err != nil { + t.Fatalf("create old sni_observations error = %v", err) + } + // Insert a row in the old format. + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, known, observed_at) + VALUES (?, ?, ?, ?, ?) + `, "203.0.113.20", "old.example.com", "443", 1, time.Now().Unix()); err != nil { + t.Fatalf("insert old sni_observations error = %v", err) + } + + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + // Reopen — migration should drop the old table and create the new one. + reopened, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() reopen error = %v", err) + } + defer func() { + if err := reopened.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + // The old row should be gone. + var count int + + err = reopened.readOnly.QueryRow(`SELECT COUNT(*) FROM sni_observations`).Scan(&count) + if err != nil { + t.Fatalf("query sni_observations count error = %v", err) + } + + if count != 0 { + t.Fatalf("row count = %d, want 0 after migration dropped old table", count) + } + + // The new schema should work. + if err := reopened.LogSNIObservation("203.0.113.20", "new.example.com", "443", OutcomePolicyAllowed); err != nil { + t.Fatalf("LogSNIObservation() after migration error = %v", err) + } + + err = reopened.readOnly.QueryRow(`SELECT COUNT(*) FROM sni_observations`).Scan(&count) + if err != nil { + t.Fatalf("query sni_observations count after insert error = %v", err) + } + + if count != 1 { + t.Fatalf("row count = %d, want 1 after insert with new schema", count) + } +} + +func TestPruneRemovesStaleRows(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + now := time.Now().Unix() + + // Insert stale allowed_ip_domains (older than refreshTrustWindow). + staleRefreshed := now - int64(refreshTrustWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.30", "stale.example.com", staleRefreshed, staleRefreshed); err != nil { + t.Fatalf("insert stale allowed_ip_domains error = %v", err) + } + + // Insert fresh allowed_ip_domains. + freshRefreshed := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.30", "fresh.example.com", freshRefreshed, freshRefreshed); err != nil { + t.Fatalf("insert fresh allowed_ip_domains error = %v", err) + } + + // Insert stale sni_observations (older than 7 days). + staleSNI := now - int64(pruneSNIWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, "203.0.113.30", "stale.example.com", "443", "unknown", staleSNI, staleSNI, 5); err != nil { + t.Fatalf("insert stale sni_observations error = %v", err) + } + + // Insert fresh sni_observations. + freshSNI := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, "203.0.113.30", "fresh.example.com", "443", "unknown", freshSNI, freshSNI, 1); err != nil { + t.Fatalf("insert fresh sni_observations error = %v", err) + } + + // Insert stale killed_connections (older than 90 days). + staleKill := now - int64(pruneKilledAlertsWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO killed_connections (ip, pid, program, killed_at) + VALUES (?, ?, ?, ?) + `, "203.0.113.30", "1234", "curl", staleKill); err != nil { + t.Fatalf("insert stale killed_connections error = %v", err) + } + + // Insert fresh killed_connections. + freshKill := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO killed_connections (ip, pid, program, killed_at) + VALUES (?, ?, ?, ?) + `, "203.0.113.30", "5678", "wget", freshKill); err != nil { + t.Fatalf("insert fresh killed_connections error = %v", err) + } + + // Insert stale zeek_alerts. + staleZeek := now - int64(pruneKilledAlertsWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "10.0.0.1", "203.0.113.30", "443", "old.example.com", 0, "low", staleZeek, "logged"); err != nil { + t.Fatalf("insert stale zeek_alerts error = %v", err) + } + + // Insert fresh zeek_alerts. + freshZeek := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "10.0.0.2", "203.0.113.30", "443", "new.example.com", 1, "high", freshZeek, "logged"); err != nil { + t.Fatalf("insert fresh zeek_alerts error = %v", err) + } + + // Run prune. + result, err := hexwallStore.Prune() + if err != nil { + t.Fatalf("Prune() error = %v", err) + } + + if result.AllowedIPDomains != 1 { + t.Fatalf("Prune().AllowedIPDomains = %d, want 1", result.AllowedIPDomains) + } + + if result.SNIObservations != 1 { + t.Fatalf("Prune().SNIObservations = %d, want 1", result.SNIObservations) + } + + if result.KilledConnections != 1 { + t.Fatalf("Prune().KilledConnections = %d, want 1", result.KilledConnections) + } + + if result.ZeekAlerts != 1 { + t.Fatalf("Prune().ZeekAlerts = %d, want 1", result.ZeekAlerts) + } + + // Verify fresh rows remain. + var count int + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM allowed_ip_domains`).Scan(&count); err != nil { + t.Fatalf("query allowed_ip_domains count error = %v", err) + } + + if count != 1 { + t.Fatalf("allowed_ip_domains count = %d, want 1 fresh row remaining", count) + } + + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM sni_observations`).Scan(&count); err != nil { + t.Fatalf("query sni_observations count error = %v", err) + } + + if count != 1 { + t.Fatalf("sni_observations count = %d, want 1 fresh row remaining", count) + } + + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM killed_connections`).Scan(&count); err != nil { + t.Fatalf("query killed_connections count error = %v", err) + } + + if count != 1 { + t.Fatalf("killed_connections count = %d, want 1 fresh row remaining", count) + } + + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM zeek_alerts`).Scan(&count); err != nil { + t.Fatalf("query zeek_alerts count error = %v", err) + } + + if count != 1 { + t.Fatalf("zeek_alerts count = %d, want 1 fresh row remaining", count) + } +} + +func TestPruneNoopWhenNothingExpired(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + now := time.Now().Unix() + + // Insert fresh rows in all pruneable tables. + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO allowed_ip_domains (ip, domain, first_seen, last_refreshed) + VALUES (?, ?, ?, ?) + `, "203.0.113.40", "fresh.example.com", now, now); err != nil { + t.Fatalf("insert fresh allowed_ip_domains error = %v", err) + } + + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO sni_observations (ip, domain, local_port, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, "203.0.113.40", "fresh.example.com", "443", "unknown", now, now, 1); err != nil { + t.Fatalf("insert fresh sni_observations error = %v", err) + } + + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO killed_connections (ip, pid, program, killed_at) + VALUES (?, ?, ?, ?) + `, "203.0.113.40", "1234", "curl", now); err != nil { + t.Fatalf("insert fresh killed_connections error = %v", err) + } + + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO zeek_alerts (src_ip, dst_ip, dst_port, sni, blocked, confidence, detected_at, action_taken) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "10.0.0.1", "203.0.113.40", "443", "fresh.example.com", 0, "low", now, "logged"); err != nil { + t.Fatalf("insert fresh zeek_alerts error = %v", err) + } + + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?) + `, "203.0.113.40", "curl", "unknown", now, now, 1); err != nil { + t.Fatalf("insert fresh ip_observations error = %v", err) + } + + result, err := hexwallStore.Prune() + if err != nil { + t.Fatalf("Prune() error = %v", err) + } + + total := result.AllowedIPDomains + result.SNIObservations + result.IPObservations + result.KilledConnections + result.ZeekAlerts + if total != 0 { + t.Fatalf("Prune() deleted %d total rows, want 0 when nothing is expired", total) + } +} + +func TestParseIPOutcomeValid(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want IPOutcome + }{ + {"ip-trusted", OutcomeIPTrusted}, + {"private-reserved", OutcomeIPReserved}, + {"deghost-blocked", OutcomeIPBlocked}, + {"unknown", OutcomeIPUnknown}, + } + + for _, tt := range tests { + got, ok := ParseIPOutcome(tt.input) + if !ok { + t.Errorf("ParseIPOutcome(%q) ok = false, want true", tt.input) + } + + if got != tt.want { + t.Errorf("ParseIPOutcome(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestParseIPOutcomeInvalid(t *testing.T) { + t.Parallel() + + tests := []string{"", "bogus", "true", "false", "1", "0"} + for _, input := range tests { + got, ok := ParseIPOutcome(input) + if ok { + t.Errorf("ParseIPOutcome(%q) ok = true, want false", input) + } + + if got != OutcomeIPUnknown { + t.Errorf("ParseIPOutcome(%q) = %q, want %q", input, got, OutcomeIPUnknown) + } + } +} + +func TestIPOutcomeValid(t *testing.T) { + t.Parallel() + + valid := []IPOutcome{OutcomeIPTrusted, OutcomeIPReserved, OutcomeIPBlocked, OutcomeIPUnknown} + for _, o := range valid { + if !o.Valid() { + t.Errorf("IPOutcome(%q).Valid() = false, want true", o) + } + } + + invalid := []IPOutcome{"bogus", "", "true"} + for _, o := range invalid { + if o.Valid() { + t.Errorf("IPOutcome(%q).Valid() = true, want false", o) + } + } +} + +func TestLogIPObservationInsertAndUpsert(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + // First insert. + if err := hexwallStore.LogIPObservation("203.0.113.10", "curl", OutcomeIPTrusted); err != nil { + t.Fatalf("LogIPObservation() error = %v", err) + } + + var ( + timesSeen int + firstSeen, lastSeen int64 + outcome string + ) + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, first_seen, last_seen, outcome + FROM ip_observations + WHERE ip = ? AND program = ? + `, "203.0.113.10", "curl").Scan(×Seen, &firstSeen, &lastSeen, &outcome) + if err != nil { + t.Fatalf("query ip_observations error = %v", err) + } + + if timesSeen != 1 { + t.Fatalf("times_seen = %d, want 1 after first insert", timesSeen) + } + + if firstSeen <= 0 { + t.Fatalf("first_seen = %d, want positive unix timestamp", firstSeen) + } + + if lastSeen != firstSeen { + t.Fatalf("last_seen = %d, want equal to first_seen on first insert, got %d", lastSeen, firstSeen) + } + + if outcome != "ip-trusted" { + t.Fatalf("outcome = %q, want %q", outcome, "ip-trusted") + } + + // Second insert with different outcome — should upsert. + if err := hexwallStore.LogIPObservation("203.0.113.10", "curl", OutcomeIPBlocked); err != nil { + t.Fatalf("LogIPObservation() upsert error = %v", err) + } + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, first_seen, last_seen, outcome + FROM ip_observations + WHERE ip = ? AND program = ? + `, "203.0.113.10", "curl").Scan(×Seen, &firstSeen, &lastSeen, &outcome) + if err != nil { + t.Fatalf("query ip_observations after upsert error = %v", err) + } + + if timesSeen != 2 { + t.Fatalf("times_seen = %d, want 2 after upsert", timesSeen) + } + + if outcome != "deghost-blocked" { + t.Fatalf("outcome = %q, want %q after upsert", outcome, "deghost-blocked") + } + + if lastSeen < firstSeen { + t.Fatalf("last_seen = %d should not be older than first_seen = %d", lastSeen, firstSeen) + } + + // Third insert — times_seen should be 3. + if err := hexwallStore.LogIPObservation("203.0.113.10", "curl", OutcomeIPReserved); err != nil { + t.Fatalf("LogIPObservation() third insert error = %v", err) + } + + err = hexwallStore.readOnly.QueryRow(` + SELECT times_seen, outcome + FROM ip_observations + WHERE ip = ? AND program = ? + `, "203.0.113.10", "curl").Scan(×Seen, &outcome) + if err != nil { + t.Fatalf("query ip_observations after third insert error = %v", err) + } + + if timesSeen != 3 { + t.Fatalf("times_seen = %d, want 3 after third insert", timesSeen) + } + + if outcome != "private-reserved" { + t.Fatalf("outcome = %q, want %q after third insert", outcome, "private-reserved") + } +} + +func TestLogIPObservationDifferentPrograms(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + // Same IP, different programs — should create separate rows. + if err := hexwallStore.LogIPObservation("203.0.113.20", "curl", OutcomeIPTrusted); err != nil { + t.Fatalf("LogIPObservation() error = %v", err) + } + + if err := hexwallStore.LogIPObservation("203.0.113.20", "wget", OutcomeIPBlocked); err != nil { + t.Fatalf("LogIPObservation() error = %v", err) + } + + var count int + + err = hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM ip_observations WHERE ip = ?`, "203.0.113.20").Scan(&count) + if err != nil { + t.Fatalf("query ip_observations count error = %v", err) + } + + if count != 2 { + t.Fatalf("row count = %d, want 2 for same IP with different programs", count) + } +} + +func TestPruneRemovesStaleIPObservations(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + hexwallStore, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + defer func() { + if err := hexwallStore.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }() + + now := time.Now().Unix() + + // Insert stale ip_observations (older than 7 days). + staleIP := now - int64(pruneSNIWindow.Seconds()) - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?) + `, "203.0.113.50", "curl", "unknown", staleIP, staleIP, 5); err != nil { + t.Fatalf("insert stale ip_observations error = %v", err) + } + + // Insert fresh ip_observations. + freshIP := now - 60 + if _, err := hexwallStore.readWrite.Exec(` + INSERT INTO ip_observations (ip, program, outcome, first_seen, last_seen, times_seen) + VALUES (?, ?, ?, ?, ?, ?) + `, "203.0.113.50", "wget", "unknown", freshIP, freshIP, 1); err != nil { + t.Fatalf("insert fresh ip_observations error = %v", err) + } + + // Run prune. + result, err := hexwallStore.Prune() + if err != nil { + t.Fatalf("Prune() error = %v", err) + } + + if result.IPObservations != 1 { + t.Fatalf("Prune().IPObservations = %d, want 1", result.IPObservations) + } + + // Verify fresh row remains. + var count int + if err := hexwallStore.readOnly.QueryRow(`SELECT COUNT(*) FROM ip_observations`).Scan(&count); err != nil { + t.Fatalf("query ip_observations count error = %v", err) + } + + if count != 1 { + t.Fatalf("ip_observations count = %d, want 1 fresh row remaining", count) + } +} + +// A database from before the outcome column existed must gain it on open, and its old +// rows must say 'unknown': back then a refused kill was recorded like a real one. +func TestNewStoreAddsKillOutcomeToOldDatabase(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "hexwall.db") + + old, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + + if _, err := old.Exec(` + CREATE TABLE killed_connections (id INTEGER PRIMARY KEY AUTOINCREMENT, ip TEXT NOT NULL, pid TEXT NOT NULL, program TEXT NOT NULL, killed_at INTEGER NOT NULL); + INSERT INTO killed_connections (ip, pid, program, killed_at) VALUES ('203.0.113.1', '1', 'curl', 1); + `); err != nil { + t.Fatal(err) + } + + _ = old.Close() + + s, err := NewStore(dbPath) + if err != nil { + t.Fatalf("NewStore() on an old database error = %v", err) + } + + t.Cleanup(func() { _ = s.Close() }) + + var outcome string + if err := s.readWrite.QueryRow(`SELECT outcome FROM killed_connections WHERE ip = '203.0.113.1'`).Scan(&outcome); err != nil { + t.Fatalf("read old row: %v", err) + } + + if outcome != "unknown" { + t.Fatalf("old row outcome = %q, want %q", outcome, "unknown") + } +} + +func TestLogKillThenSetOutcome(t *testing.T) { + s, err := NewStore(filepath.Join(t.TempDir(), "hexwall.db")) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = s.Close() }) + + id, err := s.LogKill("203.0.113.2", "4242", "curl") + if err != nil || id == 0 { + t.Fatalf("LogKill() = %d, %v; want a row id", id, err) + } + + read := func() string { + var outcome string + if err := s.readWrite.QueryRow(`SELECT outcome FROM killed_connections WHERE id = ?`, id).Scan(&outcome); err != nil { + t.Fatal(err) + } + + return outcome + } + if got := read(); got != string(KillAttempting) { + t.Fatalf("outcome before the kill = %q, want %q", got, KillAttempting) + } + + if err := s.SetKillOutcome(id, KillKilled); err != nil { + t.Fatal(err) + } + + if got := read(); got != string(KillKilled) { + t.Fatalf("outcome after SetKillOutcome = %q, want %q", got, KillKilled) + } +} diff --git a/internal/zeek/dns-bypass.zeek b/internal/zeek/dns-bypass.zeek new file mode 100644 index 0000000..2aa6b75 --- /dev/null +++ b/internal/zeek/dns-bypass.zeek @@ -0,0 +1,52 @@ +##! Detects TLS connections whose SNI was never seen in a prior DNS query +##! from the same client within the correlation window. +##! +##! Installed by `hexwall zeek-setup`, which embeds this file in the hexwall +##! binary. Re-run that command after upgrading hexwall rather than editing +##! this copy: hexwall reads the notices it writes, so the two must match. + +module DNSBypass; + +export { + redef enum Notice::Type += { + PossibleBypass + }; + + ## How far back to look for a matching DNS query. + const correlation_window = 10min &redef; +} + +# client_ip -> set of domains it has queried recently, with last-seen time +global recent_queries: table[addr] of table[string] of time &create_expire=correlation_window; + +event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) + { + local client = c$id$orig_h; + local domain = to_lower(query); + + if ( client !in recent_queries ) + recent_queries[client] = table(); + + recent_queries[client][domain] = network_time(); + } + +event ssl_established(c: connection) + { + if ( ! c?$ssl || ! c$ssl?$server_name ) + return; + + local client = c$id$orig_h; + local sni = to_lower(c$ssl$server_name); + + if ( client in recent_queries && sni in recent_queries[client] ) + return; + + # hexwall reads the SNI from $sub; $msg is for humans reading notice.log. + NOTICE([$note=PossibleBypass, + $msg=fmt("SNI %s from %s with no prior DNS query", sni, client), + $sub=sni, + $src=client, + $dst=c$id$resp_h, + $p=c$id$resp_p, + $conn=c]); + } diff --git a/internal/zeek/logfile.go b/internal/zeek/logfile.go new file mode 100644 index 0000000..6605cfe --- /dev/null +++ b/internal/zeek/logfile.go @@ -0,0 +1,66 @@ +package zeek + +import ( + "errors" + "io/fs" + "log/slog" +) + +// Zeek creates each log file when it writes the file's first line, so a log can be +// missing for a while: until the first TLS connection for ssl.log, until the first +// alert for notice.log, and briefly after each rotation. The pollers check every few +// seconds; these helpers make each change of state show up in the logs once. + +// fileState is what a poller last found at its log's path. +type fileState int + +const ( + fileUnchecked fileState = iota + fileMissing // not created yet, or between rotations: normal + fileUnreadable // exists but can't be opened, e.g. permissions + fileOpen +) + +// zeekLog names a log for the state messages. +type zeekLog struct { + name string // "ssl.log" + createdBy string // why it may not exist yet + gives string // what hexwall reads from it +} + +var ( + sslLog = zeekLog{name: "ssl.log", createdBy: "Zeek creates it at the first TLS connection it sees", gives: "TLS server names"} + noticeLog = zeekLog{name: "notice log", createdBy: "Zeek creates it at its first alert", gives: "DNS-bypass alerts"} +) + +// reportOpened logs that the file can be read, if that's news. +func (l zeekLog) reportOpened(state *fileState, path string) { + if *state != fileOpen { + slog.Info("zeek: "+l.name+" found, watching for "+l.gives, "path", path) + + *state = fileOpen + } +} + +// reportOpenError logs why the file can't be opened, if that's news, and reports +// whether it is missing. When it is, whatever appears at the path next is a new file, +// which the caller must read from the start. +func (l zeekLog) reportOpenError(state *fileState, path string, err error) (missing bool) { + if errors.Is(err, fs.ErrNotExist) { + if *state != fileMissing { + slog.Info("zeek: waiting for "+l.name+"; "+l.createdBy, "path", path) + + *state = fileMissing + } + + return true + } + + if *state != fileUnreadable { + slog.Warn("zeek: cannot open "+l.name+"; "+l.gives+" are not being read", "path", path, "err", err) + + *state = fileUnreadable + } + + return false +} diff --git a/internal/zeek/notice.go b/internal/zeek/notice.go new file mode 100644 index 0000000..fc01c2c --- /dev/null +++ b/internal/zeek/notice.go @@ -0,0 +1,279 @@ +package zeek + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math" + "os" + "strconv" + "strings" + "time" +) + +// DNSBypassScript is the Zeek script that writes the notices this file reads. It ships +// inside the hexwall binary (installed by `hexwall zeek-setup`) so the script and this +// parser can never drift apart: the script puts the SNI in the notice's sub field, and +// parseLine reads it from there. +// +//go:embed dns-bypass.zeek +var DNSBypassScript []byte + +// bypassNote is the notice type raised by DNSBypassScript. notice.log carries every Zeek +// notice (invalid certificates, scans, ...); only this one is a DNS-bypass alert. +const bypassNote = "DNSBypass::PossibleBypass" + +var ( + // ErrNoticeHeaderMissing indicates a TSV data line arrived before the "#fields" header that names its columns. + ErrNoticeHeaderMissing = errors.New("tsv notice line before its #fields header") + // ErrNoticeNoSNI indicates a DNS-bypass notice with no SNI in its sub field, e.g. one + // written by an older copy of the Zeek script; re-running `hexwall zeek-setup` fixes it. + ErrNoticeNoSNI = errors.New("dns-bypass notice without an SNI in sub; re-run hexwall zeek-setup") +) + +// Event is a parsed DNS bypass notice from Zeek. +type Event struct { + Timestamp time.Time + SrcIP string + SrcPort string + DstIP string + DstPort string + SNI string +} + +// noticeParser turns notice.log lines into Events, in either of Zeek's log formats: TSV +// (the default), read against the column order of the file's "#fields" header, or JSON +// (LogAscii::use_json = T). It remembers the header between calls, so it must be fed one +// file's lines in order. +type noticeParser struct { + tsvFields map[string]int +} + +// parseLine returns ok=false with a nil error for lines that are not DNS-bypass alerts: +// blank lines, header lines, and other notice types. +func (p *noticeParser) parseLine(line string) (event Event, ok bool, err error) { + line = strings.TrimSuffix(line, "\r") + switch { + case strings.TrimSpace(line) == "": + return Event{}, false, nil + case strings.HasPrefix(line, "#"): + if fields := parseTSVFieldsLine(line); fields != nil { + p.tsvFields = fields + } + + return Event{}, false, nil + } + + fields, err := p.fields(line) + if err != nil { + return Event{}, false, err + } + + if fields["note"] != bypassNote { + return Event{}, false, nil + } + + if fields["sub"] == "" { + return Event{}, false, ErrNoticeNoSNI + } + + return Event{ + Timestamp: parseZeekTime(fields["ts"]), + SrcIP: firstNonEmpty(fields["src"], fields["id.orig_h"]), + SrcPort: fields["id.orig_p"], + DstIP: firstNonEmpty(fields["dst"], fields["id.resp_h"]), + DstPort: firstNonEmpty(fields["p"], fields["id.resp_p"]), + SNI: fields["sub"], + }, true, nil +} + +// fields maps column name to value for one data line. Unset values are left out. +func (p *noticeParser) fields(line string) (map[string]string, error) { + if strings.HasPrefix(line, "{") { + decoder := json.NewDecoder(strings.NewReader(line)) + // Keeps ts as "1398362902.726337" rather than a float64 printed as 1.398362902726337e+09. + decoder.UseNumber() + + var payload map[string]any + if err := decoder.Decode(&payload); err != nil { + return nil, fmt.Errorf("parse json notice line: %w", err) + } + + fields := make(map[string]string, len(payload)) + for name, value := range payload { + fields[name] = fmt.Sprint(value) + } + + return fields, nil + } + + if p.tsvFields == nil { + return nil, ErrNoticeHeaderMissing + } + + cols := strings.Split(line, "\t") + + fields := make(map[string]string, len(p.tsvFields)) + for name, i := range p.tsvFields { + // "-" is Zeek's unset-field marker, "(empty)" its empty-set marker. + if i < len(cols) && cols[i] != "-" && cols[i] != "(empty)" { + fields[name] = cols[i] + } + } + + return fields, nil +} + +// parseZeekTime reads Zeek's default epoch-seconds timestamp, or an RFC 3339 one (the +// LogAscii::json_timestamps option). It falls back to now so an event is never undated. +func parseZeekTime(value string) time.Time { + if seconds, err := strconv.ParseFloat(value, 64); err == nil { + whole, frac := math.Modf(seconds) + return time.Unix(int64(whole), int64(frac*1e9)) + } + + if parsed, err := time.Parse(time.RFC3339Nano, value); err == nil { + return parsed + } + + return time.Now() +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + + return "" +} + +// noticeTailer reads notice.log from where it last stopped. +type noticeTailer struct { + path string + offset int64 + parser noticeParser + warnedBadLine bool + state fileState // what was last reported about the file, so each change is logged once +} + +// skipExisting reads the file as it is now without sending anything: the parser still +// learns the TSV "#fields" header at the top, but alerts already in the file are not +// replayed. In enforce mode an old alert would otherwise trigger a kill on every restart. +func (t *noticeTailer) skipExisting() { + t.read(nil) +} + +// poll reads every complete line added since the last poll and sends its DNS-bypass alerts. +func (t *noticeTailer) poll(events chan<- Event) { + t.read(events) +} + +// read consumes the complete lines past t.offset. With a nil events channel it only +// advances the offset and the parser's header. +func (t *noticeTailer) read(events chan<- Event) { + file, err := os.Open(t.path) + if err != nil { + if noticeLog.reportOpenError(&t.state, t.path, err) { + t.offset = 0 + t.parser = noticeParser{} + } + + return + } + defer func() { _ = file.Close() }() + + noticeLog.reportOpened(&t.state, t.path) + + fi, err := file.Stat() + if err != nil { + slog.Debug("zeek: cannot stat notice log", "path", t.path, "err", err) + return + } + + // Smaller than what was already read: the log was rotated. The new file starts with + // its own header, so forget the old one too. + if fi.Size() < t.offset { + t.offset = 0 + t.parser = noticeParser{} + } + + if fi.Size() == t.offset { + return + } + + if _, err := file.Seek(t.offset, io.SeekStart); err != nil { + slog.Debug("zeek: seek notice log failed", "path", t.path, "err", err) + return + } + + data, err := io.ReadAll(file) + if err != nil { + slog.Debug("zeek: read notice log failed", "path", t.path, "err", err) + return + } + + // Zeek may still be writing the last line. Stop at the last newline; the unfinished + // line is read again, whole, on a later poll. + end := bytes.LastIndexByte(data, '\n') + if end == -1 { + return + } + + t.offset += int64(end + 1) + + for _, line := range strings.Split(string(data[:end]), "\n") { + event, ok, err := t.parser.parseLine(line) + if err != nil { + // Warn once: a format mismatch fails every line, and one warning is enough + // to show that alerts are being missed. + if !t.warnedBadLine { + slog.Warn("zeek: notice log line could not be parsed; DNS-bypass alerts may be missed", "path", t.path, "err", err) + t.warnedBadLine = true + } else { + slog.Debug("zeek: notice log line could not be parsed", "path", t.path, "err", err) + } + + continue + } + + if !ok || events == nil { + continue + } + + select { + case events <- event: + default: + slog.Warn("zeek: DNS-bypass alert dropped, alert handler is behind", "sni", event.SNI, "dst_ip", event.DstIP) + } + } +} + +// WatchNoticeLog tails a Zeek notice log and emits parsed DNS-bypass Events. +func WatchNoticeLog(ctx context.Context, path string, events chan<- Event) error { + logPath := strings.TrimSpace(path) + if ctx == nil || logPath == "" { + return nil + } + + tailer := ¬iceTailer{path: logPath} + tailer.skipExisting() + + poller := time.NewTicker(2 * time.Second) + defer poller.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-poller.C: + tailer.poll(events) + } + } +} diff --git a/internal/zeek/notice_test.go b/internal/zeek/notice_test.go new file mode 100644 index 0000000..e4dd313 --- /dev/null +++ b/internal/zeek/notice_test.go @@ -0,0 +1,323 @@ +package zeek + +import ( + "bytes" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// The testdata notice logs are real Zeek 9.0.0 output: this package's dns-bypass.zeek run +// over Zeek's own chrome-34-google.pcap, once with default settings (TSV) and once with +// LogAscii::use_json=T. Regenerate them whenever the script changes: +// +// zeek -C -r /opt/zeek/share/btest/data/pcaps/tls/chrome-34-google.pcap dns-bypass.zeek [LogAscii::use_json=T] + +func readFixtureLines(t *testing.T, name string) []string { + t.Helper() + + data, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("read fixture %s: %v", name, err) + } + + return strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") +} + +func parseAll(t *testing.T, lines []string) []Event { + t.Helper() + + var ( + parser noticeParser + events []Event + ) + + for _, line := range lines { + event, ok, err := parser.parseLine(line) + if err != nil { + t.Fatalf("parseLine(%q) error = %v", line, err) + } + + if ok { + events = append(events, event) + } + } + + return events +} + +func assertGoogleBypassEvent(t *testing.T, events []Event) { + t.Helper() + + if len(events) != 1 { + t.Fatalf("got %d events, want 1: %+v", len(events), events) + } + + event := events[0] + + if event.SNI != "google.de" { + t.Errorf("SNI = %q, want %q", event.SNI, "google.de") + } + + if event.SrcIP != "192.168.4.149" { + t.Errorf("SrcIP = %q, want %q", event.SrcIP, "192.168.4.149") + } + + if event.SrcPort != "55881" { + t.Errorf("SrcPort = %q, want %q", event.SrcPort, "55881") + } + + if event.DstIP != "74.125.239.152" { + t.Errorf("DstIP = %q, want %q", event.DstIP, "74.125.239.152") + } + + if event.DstPort != "443" { + t.Errorf("DstPort = %q, want %q", event.DstPort, "443") + } + + want := time.Unix(1398362902, 726337000) + if diff := event.Timestamp.Sub(want); diff < -time.Microsecond || diff > time.Microsecond { + t.Errorf("Timestamp = %v, want %v (Zeek's ts, not the parse time)", event.Timestamp, want) + } +} + +func TestNoticeParser_RealTSV(t *testing.T) { + assertGoogleBypassEvent(t, parseAll(t, readFixtureLines(t, "notice.tsv"))) +} + +func TestNoticeParser_RealJSON(t *testing.T) { + assertGoogleBypassEvent(t, parseAll(t, readFixtureLines(t, "notice.ndjson"))) +} + +func TestNoticeParser_TSVLineBeforeHeader(t *testing.T) { + var dataLine string + + for _, line := range readFixtureLines(t, "notice.tsv") { + if !strings.HasPrefix(line, "#") { + dataLine = line + } + } + + var parser noticeParser + if _, _, err := parser.parseLine(dataLine); !errors.Is(err, ErrNoticeHeaderMissing) { + t.Fatalf("parseLine() error = %v, want ErrNoticeHeaderMissing", err) + } +} + +func TestNoticeParser_IgnoresOtherNoticeTypes(t *testing.T) { + line := readFixtureLines(t, "notice.ndjson")[0] + line = strings.Replace(line, bypassNote, "SSL::Invalid_Server_Cert", 1) + + var parser noticeParser + + _, ok, err := parser.parseLine(line) + if err != nil || ok { + t.Fatalf("parseLine() = ok %v, err %v; want a silent skip", ok, err) + } +} + +// A notice from an older copy of the script (SNI only in msg, no sub) must fail loudly, +// not be skipped: the warning tells the user to re-run zeek-setup. +func TestNoticeParser_BypassWithoutSub(t *testing.T) { + line := readFixtureLines(t, "notice.ndjson")[0] + line = strings.Replace(line, `"sub":"google.de",`, "", 1) + + var parser noticeParser + if _, _, err := parser.parseLine(line); !errors.Is(err, ErrNoticeNoSNI) { + t.Fatalf("parseLine() error = %v, want ErrNoticeNoSNI", err) + } +} + +func TestNoticeParser_InvalidJSON(t *testing.T) { + var parser noticeParser + if _, _, err := parser.parseLine(`{"note":`); err == nil { + t.Fatal("parseLine() error = nil, want error") + } +} + +// Zeek can be mid-write when hexwall polls. A half-written line must not be parsed (its +// SNI would be cut short); it must be read whole on the next poll. +func TestNoticeTailer_WaitsForCompleteLine(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", "notice.tsv")) + if err != nil { + t.Fatal(err) + } + + cut := strings.Index(string(data), "google.de") + len("goo") + + path := filepath.Join(t.TempDir(), "notice.log") + if err := os.WriteFile(path, data[:cut], 0o600); err != nil { + t.Fatal(err) + } + + events := make(chan Event, 10) + tailer := ¬iceTailer{path: path} + + tailer.poll(events) + + if len(events) != 0 { + t.Fatalf("got %d events from a half-written line, want 0", len(events)) + } + + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + + tailer.poll(events) + + close(events) + + var got []Event + for event := range events { + got = append(got, event) + } + + assertGoogleBypassEvent(t, got) +} + +func TestNoticeTailer_RelearnsHeaderAfterRotation(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", "notice.tsv")) + if err != nil { + t.Fatal(err) + } + + path := filepath.Join(t.TempDir(), "notice.log") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + + events := make(chan Event, 10) + tailer := ¬iceTailer{path: path} + tailer.poll(events) + <-events + + // Rotation: a fresh, smaller file with its own header. + var ( + header strings.Builder + dataLine string + ) + + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "#fields") || strings.HasPrefix(line, "#separator") { + header.WriteString(line + "\n") + } else if line != "" && !strings.HasPrefix(line, "#") { + dataLine = line + } + } + + if err := os.WriteFile(path, []byte(header.String()+dataLine+"\n"), 0o600); err != nil { + t.Fatal(err) + } + + tailer.poll(events) + + if len(events) != 1 { + t.Fatalf("got %d events after rotation, want 1", len(events)) + } +} + +// Alerts already in notice.log when hexwall starts must not be replayed: in enforce mode +// each one would trigger a kill attempt on every restart. Alerts written afterwards must +// still be read, which needs the header learned during the skip. +func TestNoticeTailer_SkipsExistingAlerts(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", "notice.tsv")) + if err != nil { + t.Fatal(err) + } + + path := filepath.Join(t.TempDir(), "notice.log") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + + events := make(chan Event, 10) + tailer := ¬iceTailer{path: path} + tailer.skipExisting() + tailer.poll(events) + + if len(events) != 0 { + t.Fatalf("got %d replayed events, want 0", len(events)) + } + + var dataLine string + + for _, line := range strings.Split(string(data), "\n") { + if line != "" && !strings.HasPrefix(line, "#") { + dataLine = line + } + } + + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + + if _, err := file.WriteString(dataLine + "\n"); err != nil { + t.Fatal(err) + } + + _ = file.Close() + + tailer.poll(events) + + if len(events) != 1 { + t.Fatalf("got %d events for a new alert, want 1", len(events)) + } +} + +func TestDNSBypassScriptEmbedded(t *testing.T) { + if !strings.Contains(string(DNSBypassScript), "$sub=sni") { + t.Fatal("embedded script does not write the SNI to sub, which parseLine reads") + } +} + +// The tailer polls every 2 s. A missing notice.log is normal until Zeek's first alert, so +// "waiting" must be logged once, not on every poll, and the file appearing must be logged. +// Not parallel: it swaps the default logger. +func TestNoticeTailer_LogsFileStateOnce(t *testing.T) { + var logs bytes.Buffer + + previous := slog.Default() + + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + path := filepath.Join(t.TempDir(), "notice.log") + tailer := ¬iceTailer{path: path} + events := make(chan Event, 10) + + tailer.skipExisting() + + for range 5 { + tailer.poll(events) + } + + if got := strings.Count(logs.String(), "waiting for notice log"); got != 1 { + t.Fatalf("logged 'waiting' %d times over 6 polls, want 1:\n%s", got, logs.String()) + } + + data, err := os.ReadFile(filepath.Join("testdata", "notice.tsv")) + if err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + + for range 3 { + tailer.poll(events) + } + + if got := strings.Count(logs.String(), "notice log found"); got != 1 { + t.Fatalf("logged 'found' %d times, want 1:\n%s", got, logs.String()) + } + // Everything in a file that appears after startup is new, so its alert is delivered. + if len(events) != 1 { + t.Fatalf("got %d events from the newly created file, want 1", len(events)) + } +} diff --git a/internal/zeek/setup.go b/internal/zeek/setup.go new file mode 100644 index 0000000..0f573de --- /dev/null +++ b/internal/zeek/setup.go @@ -0,0 +1,354 @@ +package zeek + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "net" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + scriptName = "dns-bypass.zeek" + loadLine = "@load dns-bypass" + loadComment = "# Added by hexwall zeek-setup: DNS-bypass notices that hexwall reads." + // Every version of the script carries this phrase; Teardown only deletes a file with it. + scriptSignature = "hexwall zeek-setup" + // Where Zeek's binary packages install; under sudo, /opt/zeek/bin is usually not on PATH. + defaultZeekConfig = "/opt/zeek/bin/zeek-config" +) + +// Setup installs DNSBypassScript into a local Zeek, loads it from local.zeek, and checks +// that the interfaces Zeek is told to watch exist. It prints what it did, what is still +// wrong, and the hexwall flags that read the resulting logs. It does not restart Zeek. +func Setup(out io.Writer) error { + zeekConfig, err := findZeekConfig() + if err != nil { + return err + } + + siteDir, err := zeekConfigValue(zeekConfig, "--site_dir") + if err != nil { + return err + } + + prefix, err := zeekConfigValue(zeekConfig, "--prefix") + if err != nil { + return err + } + + changed, err := installScript(siteDir, DNSBypassScript) + if err != nil { + return err + } + + if changed { + _, _ = fmt.Fprintf(out, "✓ installed %s\n", filepath.Join(siteDir, scriptName)) + } else { + _, _ = fmt.Fprintf(out, "✓ %s already up to date\n", filepath.Join(siteDir, scriptName)) + } + + localZeek := filepath.Join(siteDir, "local.zeek") + + added, err := ensureLoaded(localZeek) + if err != nil { + return err + } + + if added { + _, _ = fmt.Fprintf(out, "✓ added %q to %s\n", loadLine, localZeek) + } else { + _, _ = fmt.Fprintf(out, "✓ %s already loads the script\n", localZeek) + } + + nodeCfg := filepath.Join(prefix, "etc", "node.cfg") + + missing, err := missingInterfaces(nodeCfg, interfaceExists) + if err != nil { + _, _ = fmt.Fprintf(out, "! could not check %s: %v\n", nodeCfg, err) + } + + for _, name := range missing { + _, _ = fmt.Fprintf(out, "✗ %s tells Zeek to watch %q, which does not exist on this machine (available: %s)\n", + nodeCfg, name, strings.Join(interfaceNames(), ", ")) + } + + logDir := zeekLogDir(prefix) + _, _ = fmt.Fprintf(out, ` +Next steps: + 1. sudo %s deploy (restarts Zeek with the script loaded) + 2. run hexwall with: + --zeek-log %s --enable-zeek --zeek-notice-log %s +`, + filepath.Join(prefix, "bin", "zeekctl"), + filepath.Join(logDir, "current", "ssl.log"), + filepath.Join(logDir, "current", "notice.log")) + + if len(missing) > 0 { + return fmt.Errorf("fix the interface in %s before deploying: Zeek cannot start on an interface that does not exist", nodeCfg) + } + + return nil +} + +// Teardown undoes Setup: it removes the @load lines from local.zeek and deletes the script. +// A dns-bypass.zeek that hexwall didn't write is left in place. A machine without Zeek has +// nothing to undo. +func Teardown(out io.Writer) error { + zeekConfig, err := findZeekConfig() + if err != nil { + _, _ = fmt.Fprintln(out, "✓ Zeek not found; nothing to remove there") + return nil + } + + siteDir, err := zeekConfigValue(zeekConfig, "--site_dir") + if err != nil { + return err + } + + localZeek := filepath.Join(siteDir, "local.zeek") + + removed, err := removeLoad(localZeek) + if err != nil { + return err + } + + if removed { + _, _ = fmt.Fprintf(out, "✓ removed %q from %s\n", loadLine, localZeek) + } + + script := filepath.Join(siteDir, scriptName) + + deleted, err := removeScript(script) + if err != nil { + return err + } + + if deleted { + _, _ = fmt.Fprintf(out, "✓ deleted %s\n", script) + } else if _, statErr := os.Stat(script); statErr == nil { + _, _ = fmt.Fprintf(out, "! %s wasn't written by hexwall; left in place\n", script) + } + + return nil +} + +// removeLoad deletes the lines ensureLoaded added: the @load line, its comment, and the +// blank line before the comment. +func removeLoad(localZeek string) (removed bool, err error) { + data, err := os.ReadFile(localZeek) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + + if err != nil { + return false, permissionHint(fmt.Errorf("read %s: %w", localZeek, err)) + } + + lines := strings.Split(string(data), "\n") + + kept := make([]string, 0, len(lines)) + for i, line := range lines { + switch trimmed := strings.TrimSpace(line); { + case trimmed == loadLine || trimmed == loadComment: + removed = true + case trimmed == "" && i+1 < len(lines) && strings.TrimSpace(lines[i+1]) == loadComment: + // The blank line ensureLoaded put before its comment. + default: + kept = append(kept, line) + } + } + + if !removed { + return false, nil + } + + if err := os.WriteFile(localZeek, []byte(strings.Join(kept, "\n")), 0o644); err != nil { + return false, permissionHint(fmt.Errorf("write %s: %w", localZeek, err)) + } + + return true, nil +} + +// removeScript deletes the script if hexwall wrote it (any version), and reports whether it did. +func removeScript(path string) (deleted bool, err error) { + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + + if err != nil { + return false, permissionHint(fmt.Errorf("read %s: %w", path, err)) + } + + if !bytes.Contains(data, []byte(scriptSignature)) { + return false, nil + } + + if err := os.Remove(path); err != nil { + return false, permissionHint(fmt.Errorf("delete %s: %w", path, err)) + } + + return true, nil +} + +// Zeekctl returns the path of the local Zeek's zeekctl, located through zeek-config. +func Zeekctl() (string, error) { + zeekConfig, err := findZeekConfig() + if err != nil { + return "", err + } + + prefix, err := zeekConfigValue(zeekConfig, "--prefix") + if err != nil { + return "", err + } + + return filepath.Join(prefix, "bin", "zeekctl"), nil +} + +func findZeekConfig() (string, error) { + if path, err := exec.LookPath("zeek-config"); err == nil { + return path, nil + } + + if _, err := os.Stat(defaultZeekConfig); err == nil { + return defaultZeekConfig, nil + } + + return "", errors.New("zeek-config not found on PATH or at " + defaultZeekConfig + "; is Zeek installed?") +} + +func zeekConfigValue(zeekConfig, flag string) (string, error) { + out, err := exec.Command(zeekConfig, flag).Output() + if err != nil { + return "", fmt.Errorf("%s %s: %w", zeekConfig, flag, err) + } + + return strings.TrimSpace(string(out)), nil +} + +// installScript writes script into siteDir unless an identical copy is already there. +func installScript(siteDir string, script []byte) (changed bool, err error) { + path := filepath.Join(siteDir, scriptName) + if current, err := os.ReadFile(path); err == nil && bytes.Equal(current, script) { + return false, nil + } + + if err := os.WriteFile(path, script, 0o644); err != nil { + return false, permissionHint(fmt.Errorf("write %s: %w", path, err)) + } + + return true, nil +} + +// ensureLoaded appends the @load line to local.zeek unless it is already there. +func ensureLoaded(localZeek string) (added bool, err error) { + data, err := os.ReadFile(localZeek) + if err != nil { + return false, permissionHint(fmt.Errorf("read %s: %w", localZeek, err)) + } + + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) == loadLine { + return false, nil + } + } + + addition := "\n" + loadComment + "\n" + loadLine + "\n" + if len(data) > 0 && !bytes.HasSuffix(data, []byte("\n")) { + addition = "\n" + addition + } + + file, err := os.OpenFile(localZeek, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + return false, permissionHint(fmt.Errorf("open %s: %w", localZeek, err)) + } + + if _, err := file.WriteString(addition); err != nil { + _ = file.Close() + return false, fmt.Errorf("append to %s: %w", localZeek, err) + } + + return true, file.Close() +} + +// missingInterfaces returns the interfaces named in node.cfg that exists reports as absent. +// Newer zeekctl accepts a packet-source prefix ("af_packet::eth0"); only the name is checked. +func missingInterfaces(nodeCfg string, exists func(string) bool) ([]string, error) { + file, err := os.Open(nodeCfg) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + + var missing []string + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + key, value, ok := strings.Cut(line, "=") + if !ok || strings.HasPrefix(line, "#") || strings.TrimSpace(key) != "interface" { + continue + } + + name := strings.TrimSpace(value) + if i := strings.LastIndex(name, "::"); i >= 0 { + name = name[i+2:] + } + + if name != "" && !exists(name) { + missing = append(missing, name) + } + } + + return missing, scanner.Err() +} + +func interfaceExists(name string) bool { + _, err := net.InterfaceByName(name) + return err == nil +} + +func interfaceNames() []string { + interfaces, err := net.Interfaces() + if err != nil { + return nil + } + + names := make([]string, 0, len(interfaces)) + for _, iface := range interfaces { + names = append(names, iface.Name) + } + + return names +} + +// zeekLogDir reads LogDir from zeekctl.cfg, falling back to zeekctl's default. +func zeekLogDir(prefix string) string { + if data, err := os.ReadFile(filepath.Join(prefix, "etc", "zeekctl.cfg")); err == nil { + for _, line := range strings.Split(string(data), "\n") { + key, value, ok := strings.Cut(line, "=") + if ok && strings.EqualFold(strings.TrimSpace(key), "LogDir") { + return strings.TrimSpace(value) + } + } + } + + return filepath.Join(prefix, "logs") +} + +func permissionHint(err error) error { + if errors.Is(err, fs.ErrPermission) { + return fmt.Errorf("%w (run with sudo)", err) + } + + return err +} diff --git a/internal/zeek/setup_test.go b/internal/zeek/setup_test.go new file mode 100644 index 0000000..9e68171 --- /dev/null +++ b/internal/zeek/setup_test.go @@ -0,0 +1,281 @@ +package zeek + +import ( + "bytes" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestInstallScript_WritesThenSkipsIdentical(t *testing.T) { + siteDir := t.TempDir() + + changed, err := installScript(siteDir, DNSBypassScript) + if err != nil || !changed { + t.Fatalf("first installScript() = %v, %v; want true, nil", changed, err) + } + + got, err := os.ReadFile(filepath.Join(siteDir, scriptName)) + if err != nil || string(got) != string(DNSBypassScript) { + t.Fatalf("installed script differs from the embedded one (err %v)", err) + } + + changed, err = installScript(siteDir, DNSBypassScript) + if err != nil || changed { + t.Fatalf("second installScript() = %v, %v; want false, nil", changed, err) + } +} + +// An older copy (e.g. one without $sub) must be replaced, or hexwall can't read its notices. +func TestInstallScript_ReplacesOutdatedCopy(t *testing.T) { + siteDir := t.TempDir() + if err := os.WriteFile(filepath.Join(siteDir, scriptName), []byte("# old version\n"), 0o644); err != nil { + t.Fatal(err) + } + + changed, err := installScript(siteDir, DNSBypassScript) + if err != nil || !changed { + t.Fatalf("installScript() = %v, %v; want true, nil", changed, err) + } +} + +func TestEnsureLoaded_AppendsOnce(t *testing.T) { + localZeek := filepath.Join(t.TempDir(), "local.zeek") + + original := "@load misc/detect-traceroute\n@load tuning/defaults" // no trailing newline + if err := os.WriteFile(localZeek, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + for i, want := range []bool{true, false} { + added, err := ensureLoaded(localZeek) + if err != nil || added != want { + t.Fatalf("ensureLoaded() call %d = %v, %v; want %v, nil", i+1, added, err, want) + } + } + + data, _ := os.ReadFile(localZeek) + + lines := strings.Split(string(data), "\n") + if !slices.Contains(lines, "@load tuning/defaults") { + t.Fatalf("last existing line was mangled:\n%s", data) + } + + if strings.Count(string(data), loadLine) != 1 { + t.Fatalf("want exactly one %q line:\n%s", loadLine, data) + } +} + +func TestMissingInterfaces(t *testing.T) { + nodeCfg := filepath.Join(t.TempDir(), "node.cfg") + // Shape of zeekctl's default standalone node.cfg, plus a worker using a packet-source prefix. + content := `# Example ZeekControl node configuration. +[zeek] +type=standalone +host=localhost +interface=eth0 + +[worker-1] +type=worker +interface=af_packet::wlx001ea6c37e44 +#interface=commented0 +` + if err := os.WriteFile(nodeCfg, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + exists := func(name string) bool { return name == "wlx001ea6c37e44" } + + missing, err := missingInterfaces(nodeCfg, exists) + if err != nil { + t.Fatalf("missingInterfaces() error = %v", err) + } + + if !slices.Equal(missing, []string{"eth0"}) { + t.Fatalf("missingInterfaces() = %v, want [eth0]", missing) + } +} + +func TestZeekLogDir(t *testing.T) { + prefix := t.TempDir() + if got, want := zeekLogDir(prefix), filepath.Join(prefix, "logs"); got != want { + t.Fatalf("zeekLogDir() without zeekctl.cfg = %q, want %q", got, want) + } + + if err := os.MkdirAll(filepath.Join(prefix, "etc"), 0o755); err != nil { + t.Fatal(err) + } + + cfg := "# Log rotation\nLogRotationInterval = 3600\nLogDir = /var/log/zeek\n" + if err := os.WriteFile(filepath.Join(prefix, "etc", "zeekctl.cfg"), []byte(cfg), 0o644); err != nil { + t.Fatal(err) + } + + if got := zeekLogDir(prefix); got != "/var/log/zeek" { + t.Fatalf("zeekLogDir() = %q, want %q", got, "/var/log/zeek") + } +} + +// Setup then Teardown must give local.zeek back exactly as it was. +func TestRemoveLoad_UndoesEnsureLoaded(t *testing.T) { + localZeek := filepath.Join(t.TempDir(), "local.zeek") + + original := "@load misc/detect-traceroute\n\n@load tuning/defaults\n" + if err := os.WriteFile(localZeek, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + if _, err := ensureLoaded(localZeek); err != nil { + t.Fatal(err) + } + + removed, err := removeLoad(localZeek) + if err != nil || !removed { + t.Fatalf("removeLoad() = %v, %v; want true, nil", removed, err) + } + + got, _ := os.ReadFile(localZeek) + if string(got) != original { + t.Fatalf("local.zeek after setup+teardown:\n%q\nwant:\n%q", got, original) + } + + if removed, err := removeLoad(localZeek); err != nil || removed { + t.Fatalf("second removeLoad() = %v, %v; want false, nil", removed, err) + } +} + +func TestRemoveScript_OnlyDeletesHexwallsOwn(t *testing.T) { + dir := t.TempDir() + + ours := filepath.Join(dir, "ours.zeek") + if err := os.WriteFile(ours, DNSBypassScript, 0o644); err != nil { + t.Fatal(err) + } + + if deleted, err := removeScript(ours); err != nil || !deleted { + t.Fatalf("removeScript(hexwall's script) = %v, %v; want true, nil", deleted, err) + } + + foreign := filepath.Join(dir, "foreign.zeek") + if err := os.WriteFile(foreign, []byte("# someone's own dns-bypass\n"), 0o644); err != nil { + t.Fatal(err) + } + + if deleted, err := removeScript(foreign); err != nil || deleted { + t.Fatalf("removeScript(foreign) = %v, %v; want false, nil", deleted, err) + } + + if _, err := os.Stat(foreign); err != nil { + t.Fatalf("foreign script was deleted: %v", err) + } + + if deleted, err := removeScript(filepath.Join(dir, "missing.zeek")); err != nil || deleted { + t.Fatalf("removeScript(missing) = %v, %v; want false, nil", deleted, err) + } +} + +// fakeZeek builds a Zeek install in a temp dir (site dir with a local.zeek, node.cfg +// watching iface, zeekctl.cfg) and a fake zeek-config on PATH that points at it. +func fakeZeek(t *testing.T, iface string) (siteDir, prefix string) { + t.Helper() + prefix = t.TempDir() + + siteDir = filepath.Join(prefix, "share", "zeek", "site") + for _, dir := range []string{siteDir, filepath.Join(prefix, "etc"), filepath.Join(prefix, "bin")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + + files := map[string]string{ + filepath.Join(siteDir, "local.zeek"): "@load misc/detect-traceroute\n", + filepath.Join(prefix, "etc", "node.cfg"): "[zeek]\ntype=standalone\ninterface=" + iface + "\n", + filepath.Join(prefix, "etc", "zeekctl.cfg"): "LogDir = " + filepath.Join(prefix, "logs") + "\n", + filepath.Join(prefix, "bin", "zeek-config"): "#!/bin/sh\ncase \"$1\" in\n --site_dir) echo '" + siteDir + "' ;;\n --prefix) echo '" + prefix + "' ;;\nesac\n", + } + for path, content := range files { + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatal(err) + } + } + + t.Setenv("PATH", filepath.Join(prefix, "bin")+string(os.PathListSeparator)+os.Getenv("PATH")) + + return siteDir, prefix +} + +func anyInterface(t *testing.T) string { + t.Helper() + + names := interfaceNames() + if len(names) == 0 { + t.Skip("no network interfaces") + } + + return names[0] +} + +// The whole user journey: zeek-setup installs and loads the script and prints the right +// hexwall flags; uninstall's Teardown gives local.zeek back exactly as it was. +func TestSetupThenTeardown(t *testing.T) { + siteDir, prefix := fakeZeek(t, anyInterface(t)) + original, _ := os.ReadFile(filepath.Join(siteDir, "local.zeek")) + + var out bytes.Buffer + if err := Setup(&out); err != nil { + t.Fatalf("Setup() error = %v\n%s", err, out.String()) + } + + if installed, err := os.ReadFile(filepath.Join(siteDir, scriptName)); err != nil || !bytes.Equal(installed, DNSBypassScript) { + t.Fatalf("script not installed (err %v)", err) + } + + for _, want := range []string{ + filepath.Join(prefix, "bin", "zeekctl") + " deploy", + "--zeek-log " + filepath.Join(prefix, "logs", "current", "ssl.log"), + "--zeek-notice-log " + filepath.Join(prefix, "logs", "current", "notice.log"), + } { + if !strings.Contains(out.String(), want) { + t.Errorf("Setup() output is missing %q:\n%s", want, out.String()) + } + } + + if err := Teardown(&bytes.Buffer{}); err != nil { + t.Fatalf("Teardown() error = %v", err) + } + + if _, err := os.Stat(filepath.Join(siteDir, scriptName)); !os.IsNotExist(err) { + t.Fatalf("script still installed after Teardown (err %v)", err) + } + + if after, _ := os.ReadFile(filepath.Join(siteDir, "local.zeek")); !bytes.Equal(after, original) { + t.Fatalf("local.zeek after Setup+Teardown:\n%q\nwant:\n%q", after, original) + } +} + +// Zeek can't start on an interface that doesn't exist; Setup must say so and fail. +func TestSetupReportsMissingInterface(t *testing.T) { + fakeZeek(t, "hexwall-test-no-such-if0") + + var out bytes.Buffer + + err := Setup(&out) + if err == nil { + t.Fatalf("Setup() error = nil, want an error for a missing interface\n%s", out.String()) + } + + if !strings.Contains(out.String(), `"hexwall-test-no-such-if0", which does not exist`) { + t.Fatalf("Setup() output doesn't name the missing interface:\n%s", out.String()) + } +} + +func TestZeekctlFromZeekConfig(t *testing.T) { + _, prefix := fakeZeek(t, anyInterface(t)) + + got, err := Zeekctl() + if err != nil || got != filepath.Join(prefix, "bin", "zeekctl") { + t.Fatalf("Zeekctl() = %q, %v; want %q", got, err, filepath.Join(prefix, "bin", "zeekctl")) + } +} diff --git a/internal/zeek/testdata/notice.ndjson b/internal/zeek/testdata/notice.ndjson new file mode 100644 index 0000000..ac83674 --- /dev/null +++ b/internal/zeek/testdata/notice.ndjson @@ -0,0 +1 @@ +{"ts":1398362902.726337,"uid":"CCLHJAgTLjzZMS085","id.orig_h":"192.168.4.149","id.orig_p":55881,"id.resp_h":"74.125.239.152","id.resp_p":443,"proto":"tcp","note":"DNSBypass::PossibleBypass","msg":"SNI google.de from 192.168.4.149 with no prior DNS query","sub":"google.de","src":"192.168.4.149","dst":"74.125.239.152","p":443,"actions":["Notice::ACTION_LOG"],"email_dest":[],"suppress_for":3600.0} diff --git a/internal/zeek/testdata/notice.tsv b/internal/zeek/testdata/notice.tsv new file mode 100644 index 0000000..e3bd261 --- /dev/null +++ b/internal/zeek/testdata/notice.tsv @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path notice +#open 2026-10-11-14-47-56 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions email_dest suppress_for remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] set[string] interval string string string double double +1398362902.726337 CNVr1M3g8O7l7ULm2 192.168.4.149 55881 74.125.239.152 443 - - - tcp DNSBypass::PossibleBypass SNI google.de from 192.168.4.149 with no prior DNS query google.de 192.168.4.149 74.125.239.152 443 - - Notice::ACTION_LOG (empty) 3600.000000 - - - - - +#close 2026-10-11-14-47-56 diff --git a/internal/zeek/zeek.go b/internal/zeek/zeek.go new file mode 100644 index 0000000..e1f9a0f --- /dev/null +++ b/internal/zeek/zeek.go @@ -0,0 +1,379 @@ +// Package zeek tails Zeek logs to extract SNI/domain data and DNS-bypass notices. +package zeek + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "os" + "strings" + "sync" + "time" +) + +const ( + tickInterval = 5 * time.Second + entryExpiry = 2 * time.Minute +) + +// Config holds the configuration for the Zeek ssl.log tailer. +type Config struct { + LogPath string +} + +type sniEntry struct { + domain string + observedAt time.Time +} + +// sslRecord is the subset of fields extracted from a Zeek SSL log line when +// that line is JSON (LogAscii::use_json = T). Zeek's default output is +// tab-separated, not JSON — see recordFromLine and parseTSVFieldsLine for +// that path, which is what a stock Zeek install actually produces. +type sslRecord struct { + OrigH string `json:"id.orig_h"` + OrigP any `json:"id.orig_p"` + RespH string `json:"id.resp_h"` + RespP any `json:"id.resp_p"` + ServerName string `json:"server_name"` +} + +// Client tails a Zeek ssl.log and maintains a short-lived in-memory cache +// mapping connection tuples to their observed SNI domain. +type Client struct { + logPath string + + mu sync.RWMutex + cache map[string]sniEntry + + // tsvFields maps a Zeek TSV column name (e.g. "id.orig_h") to its column + // index, learned from a "#fields" header line. nil until one has been + // seen for the current file, in which case non-JSON lines can't yet be + // interpreted and are skipped. + tsvFields map[string]int + + offset int64 + pending []byte // bytes read past the last newline; held until the rest of the line arrives + state fileState // what was last reported about the file, so each change is logged once + stopCh chan struct{} + done chan struct{} +} + +// NewClient opens the ssl.log at the configured path and starts a +// background goroutine that tails new lines, in either of Zeek's log +// formats (TSV, the default; or JSON, if LogAscii::use_json is set). +// If cfg is nil or LogPath is empty, it returns nil without error. +func NewClient(cfg *Config) (*Client, error) { + if cfg == nil || strings.TrimSpace(cfg.LogPath) == "" { + return nil, nil + } + + logPath := strings.TrimSpace(cfg.LogPath) + + c := &Client{ + logPath: logPath, + cache: make(map[string]sniEntry), + stopCh: make(chan struct{}), + done: make(chan struct{}), + } + + // A missing ssl.log is normal (Zeek creates it at the first TLS connection) and + // must not stop hexwall: the scan runs by IP meanwhile, and the tailer reads the + // file from its first line once it appears. Any other error, such as no + // permission, won't fix itself, so it stops startup loudly. + f, err := os.Open(logPath) + if err != nil { + if !sslLog.reportOpenError(&c.state, logPath, err) { + return nil, fmt.Errorf("failed to open zeek ssl.log %s: %w", logPath, err) + } + } else { + fi, err := f.Stat() + + _ = f.Close() + + if err != nil { + return nil, fmt.Errorf("failed to stat zeek ssl.log %s: %w", logPath, err) + } + + // The tailer starts reading from the current end of the file, which is always + // past the "#fields" header Zeek writes once, at the very top, when it creates + // the log. Without this separate, from-the-start read, TSV parsing would never + // learn the column order in the normal case (Zeek already running before + // hexwall starts) and would silently never match anything. + c.offset = fi.Size() + c.tsvFields = readTSVHeader(logPath) + sslLog.reportOpened(&c.state, logPath) + } + + go c.tailLoop(context.Background()) + + slog.Info("zeek log watcher started", "path", logPath, "offset", c.offset, "tsv_header_found", c.tsvFields != nil) + + return c, nil +} + +// Close stops the background tailer and waits for it to finish. +func (c *Client) Close() { + if c == nil { + return + } + + close(c.stopCh) + + <-c.done +} + +// Lookup returns the SNI domain observed for the given connection tuple, +// or ("", false) if no matching record exists in the cache. +func (c *Client) Lookup(localPort, remoteIP, remotePort string) (string, bool) { + // A nil *Client stored in an interface is not a nil interface, so callers holding one + // can't tell it apart from a real client: answer "no SNI" instead of panicking. + if c == nil { + return "", false + } + + key := localPort + "|" + remoteIP + "|" + remotePort + + c.mu.RLock() + entry, ok := c.cache[key] + c.mu.RUnlock() + + if !ok { + return "", false + } + + if time.Since(entry.observedAt) > entryExpiry { + return "", false + } + + return entry.domain, true +} + +// parseTSVFieldsLine parses a Zeek "#fields\t..." header line into a +// column-name -> index map. Returns nil for any other line, including other +// "#"-prefixed header metadata (#separator, #path, #open, ...). +func parseTSVFieldsLine(line string) map[string]int { + if !strings.HasPrefix(line, "#fields\t") { + return nil + } + + cols := strings.Split(line, "\t") + + fields := make(map[string]int, len(cols)-1) + for i, name := range cols[1:] { + fields[name] = i + } + + return fields +} + +// readTSVHeader scans from the start of the log file for a "#fields" header +// line. Returns nil if the file isn't in TSV format (e.g. it's JSON, or +// reading fails), in which case the caller falls back to per-line JSON +// detection only. +func readTSVHeader(logPath string) map[string]int { + f, err := os.Open(logPath) + if err != nil { + return nil + } + defer func() { _ = f.Close() }() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if fields := parseTSVFieldsLine(line); fields != nil { + return fields + } + + if !strings.HasPrefix(line, "#") { + return nil // reached a data line without finding #fields + } + } + + // Stopping on an error looks the same as "no header" to the caller, and without a + // header no TSV line can be read: say so instead of failing silently. + if err := scanner.Err(); err != nil { + slog.Warn("zeek: could not read the ssl.log header; TLS server names can't be read until Zeek starts a new file", + "path", logPath, "err", err) + } + + return nil +} + +// recordFromLine extracts the five ssl.log fields this package needs from a +// single log line, transparently handling both of Zeek's log formats: +// +// - TSV (Zeek's default): a "#fields" header line, seen once per file, +// declares the column order; every following line is read positionally +// against it via c.tsvFields. Any other "#"-prefixed line is header +// metadata and is skipped. A "#fields" line seen here (not just at +// NewClient time) updates c.tsvFields — this is what makes a rotated +// log's fresh header get picked up too. +// - JSON (LogAscii::use_json = T): each line is a self-describing object. +// +// ok is false for header lines, or for a TSV data line seen before its +// header (data missing "-" is normalized to "", matching Zeek's own +// unset-field marker). +func (c *Client) recordFromLine(line string) (origH, origP, respH, respP, serverName string, ok bool) { + switch { + case strings.HasPrefix(line, "#"): + if fields := parseTSVFieldsLine(line); fields != nil { + c.tsvFields = fields + } + + return "", "", "", "", "", false + + case strings.HasPrefix(line, "{"): + var rec sslRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return "", "", "", "", "", false + } + + return rec.OrigH, fmt.Sprint(rec.OrigP), rec.RespH, fmt.Sprint(rec.RespP), rec.ServerName, true + + default: + if c.tsvFields == nil { + return "", "", "", "", "", false + } + + cols := strings.Split(line, "\t") + get := func(name string) string { + i, ok := c.tsvFields[name] + if !ok || i >= len(cols) { + return "" + } + + if cols[i] == "-" { // Zeek's unset-field marker + return "" + } + + return cols[i] + } + + return get("id.orig_h"), get("id.orig_p"), get("id.resp_h"), get("id.resp_p"), get("server_name"), true + } +} + +func (c *Client) tailLoop(ctx context.Context) { + defer close(c.done) + + ticker := time.NewTicker(tickInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-c.stopCh: + return + case <-ticker.C: + c.processNewLines() + } + } +} + +func (c *Client) processNewLines() { + file, err := os.Open(c.logPath) + if err != nil { + if sslLog.reportOpenError(&c.state, c.logPath, err) { + // Whatever appears here next is a new file, with its own "#fields" header + // at the top: read it from the start and learn that header again. + c.offset = 0 + c.pending = nil + c.tsvFields = nil + } + + return + } + + sslLog.reportOpened(&c.state, c.logPath) + + defer func() { + if err := file.Close(); err != nil { + slog.Debug("zeek: failed to close ssl.log", "path", c.logPath, "err", err) + } + }() + + fi, err := file.Stat() + if err != nil { + slog.Debug("zeek: cannot stat ssl.log", "path", c.logPath, "err", err) + return + } + + // File was likely rotated — reset to the beginning and drop any + // partial line held over from the old file. The new file's own + // "#fields" header will be re-learned via recordFromLine below as the + // tailer reads through it, same as any other line. + if fi.Size() < c.offset { + c.offset = 0 + c.pending = nil + } + + if fi.Size() == c.offset { + return + } + + if _, err := file.Seek(c.offset, io.SeekStart); err != nil { + slog.Debug("zeek: seek failed", "path", c.logPath, "err", err) + return + } + + data, err := io.ReadAll(file) + if err != nil { + slog.Debug("zeek: read failed", "path", c.logPath, "err", err) + return + } + + // We've now consumed everything up to fi.Size() from the file, even + // if the trailing bytes don't yet form a complete line — hold them in + // pending rather than losing them, since Zeek may still be mid-write + // on the last line when this tick fires. + c.offset = fi.Size() + + buf := append(c.pending, data...) + + lastNewline := bytes.LastIndexByte(buf, '\n') + if lastNewline == -1 { + c.pending = buf + return + } + + complete := buf[:lastNewline] + c.pending = append([]byte(nil), buf[lastNewline+1:]...) + + lines := strings.Split(string(complete), "\n") + + c.mu.Lock() + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + origH, origP, respH, respP, serverName, ok := c.recordFromLine(line) + if !ok || serverName == "" || origH == "" || respH == "" { + continue + } + + now := time.Now() + entry := sniEntry{domain: serverName, observedAt: now} + + // Store both orientations so Lookup works regardless of which + // side of the connection is the local endpoint. + c.cache[origP+"|"+respH+"|"+respP] = entry + c.cache[respP+"|"+origH+"|"+origP] = entry + } + + // Prune expired entries. + cutoff := time.Now().Add(-entryExpiry) + for k, v := range c.cache { + if v.observedAt.Before(cutoff) { + delete(c.cache, k) + } + } + c.mu.Unlock() +} diff --git a/internal/zeek/zeek_test.go b/internal/zeek/zeek_test.go new file mode 100644 index 0000000..2278cd6 --- /dev/null +++ b/internal/zeek/zeek_test.go @@ -0,0 +1,428 @@ +package zeek + +import ( + "bytes" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// newTestClient builds a Client pointed at path without starting the +// background tailLoop goroutine, so tests can call processNewLines directly +// at controlled points instead of racing a real ticker. +func newTestClient(t *testing.T, path string) *Client { + t.Helper() + + return &Client{ + logPath: path, + cache: make(map[string]sniEntry), + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } +} + +func TestClientProcessNewLines_BasicParse(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}`+"\n") + + c := newTestClient(t, path) + c.processNewLines() + + domain, ok := c.Lookup("51820", "203.0.113.20", "443") + if !ok { + t.Fatal("Lookup() ok = false, want true") + } + + if domain != "example.com" { + t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") + } + + // The reverse orientation must also resolve, since Lookup doesn't know + // in advance which side of the connection is the local endpoint. + domain, ok = c.Lookup("443", "10.0.0.5", "51820") + if !ok { + t.Fatal("Lookup() reverse orientation ok = false, want true") + } + + if domain != "example.com" { + t.Fatalf("Lookup() reverse orientation domain = %q, want %q", domain, "example.com") + } +} + +func TestClientProcessNewLines_SkipsIncompleteRecords(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443}`+"\n"+ + `not json at all`+"\n"+ + `{"id.orig_h":"","id.orig_p":1,"id.resp_h":"203.0.113.21","id.resp_p":443,"server_name":"missing-orig-host.example"}`+"\n") + + c := newTestClient(t, path) + c.processNewLines() + + if len(c.cache) != 0 { + t.Fatalf("cache = %v, want empty (no valid records in input)", c.cache) + } +} + +func TestClientProcessNewLines_PartialLineHeldAcrossTicks(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + full := `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}` + "\n" + + // Simulate Zeek flushing mid-record: only the first half of the line + // lands before the poll fires. + split := len(full) / 2 + writeFile(t, path, full[:split]) + + c := newTestClient(t, path) + c.processNewLines() + + if _, ok := c.Lookup("51820", "203.0.113.20", "443"); ok { + t.Fatal("Lookup() found a record from a partial line, want none yet") + } + + if len(c.pending) == 0 { + t.Fatal("pending is empty, want the partial line to be held over") + } + + // The rest of the line arrives on the next tick. + writeFile(t, path, full) + c.processNewLines() + + domain, ok := c.Lookup("51820", "203.0.113.20", "443") + if !ok { + t.Fatal("Lookup() ok = false after completing the line, want true") + } + + if domain != "example.com" { + t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") + } +} + +func TestClientProcessNewLines_Rotation(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}`+"\n") + + c := newTestClient(t, path) + c.processNewLines() + + if _, ok := c.Lookup("51820", "203.0.113.20", "443"); !ok { + t.Fatal("Lookup() ok = false before rotation, want true") + } + + // Rotation: zeekctl closes the current log and starts a fresh, empty + // file at the same path — simulate the two ticks that straddle that. + writeFile(t, path, "") + c.processNewLines() + writeFile(t, path, `{"id.orig_h":"10.0.0.9","id.orig_p":9000,"id.resp_h":"203.0.113.30","id.resp_p":443,"server_name":"post-rotation.example"}`+"\n") + c.processNewLines() + + if domain, ok := c.Lookup("9000", "203.0.113.30", "443"); !ok || domain != "post-rotation.example" { + t.Fatalf("Lookup() after rotation = (%q, %v), want (%q, true)", domain, ok, "post-rotation.example") + } +} + +func TestClientLookup_Expiry(t *testing.T) { + c := newTestClient(t, "") + c.cache["51820|203.0.113.20|443"] = sniEntry{ + domain: "example.com", + observedAt: time.Now().Add(-entryExpiry - time.Second), + } + + if _, ok := c.Lookup("51820", "203.0.113.20", "443"); ok { + t.Fatal("Lookup() ok = true for an expired entry, want false") + } +} + +// ---- TSV format (Zeek's actual default output) -------------------------- + +// realZeekTSVHeader and realZeekTSVDataLine are captured verbatim from a +// live Zeek 9.0.0 instance's ssl.log (default config, no LogAscii::use_json). +// This is the format a stock Zeek install actually produces — the JSON +// tests above only exercise the opt-in LogAscii::use_json = T path. +const realZeekTSVHeader = "#separator \\x09\n" + + "#set_separator\t,\n" + + "#empty_field\t(empty)\n" + + "#unset_field\t-\n" + + "#path\tssl\n" + + "#open\t2026-09-23-16-59-25\n" + + "#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tversion\tcipher\tcurve\tserver_name\tresumed\tlast_alert\tnext_protocol\testablished\tssl_history\tcert_chain_fps\tclient_cert_chain_fps\tsni_matches_cert\n" + + "#types\ttime\tstring\taddr\tport\taddr\tport\tstring\tstring\tstring\tstring\tbool\tstring\tstring\tbool\tstring\tvector[string]\tvector[string]\tbool\n" + +const realZeekTSVDataLine = "1790163565.389393\tC5MrwN15Qi19OF5Y2g\t192.168.0.223\t37526\t172.66.147.243\t443\tTLSv13\tTLS_AES_256_GCM_SHA384\tx25519\texample.com\tF\t-\t-\tT\tCsiI\t-\t-\t-\n" + +func TestParseTSVFieldsLine(t *testing.T) { + t.Parallel() + + lines := strings.Split(strings.TrimRight(realZeekTSVHeader, "\n"), "\n") + + var fieldsLine string + + for _, l := range lines { + if strings.HasPrefix(l, "#fields\t") { + fieldsLine = l + } + } + + if fieldsLine == "" { + t.Fatal("test fixture has no #fields line") + } + + fields := parseTSVFieldsLine(fieldsLine) + + want := map[string]int{ + "ts": 0, "uid": 1, "id.orig_h": 2, "id.orig_p": 3, "id.resp_h": 4, "id.resp_p": 5, + "server_name": 9, + } + for name, idx := range want { + if fields[name] != idx { + t.Errorf("parseTSVFieldsLine()[%q] = %d, want %d", name, fields[name], idx) + } + } + + if parseTSVFieldsLine("#types\ttime\tstring") != nil { + t.Error("parseTSVFieldsLine(#types line) = non-nil, want nil (only #fields should match)") + } +} + +func TestReadTSVHeader(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine) + + fields := readTSVHeader(path) + if fields == nil { + t.Fatal("readTSVHeader() = nil, want a field map for a real TSV file") + } + + if fields["server_name"] != 9 { + t.Fatalf("readTSVHeader()[server_name] = %d, want 9", fields["server_name"]) + } +} + +func TestReadTSVHeaderNilForJSON(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, `{"id.orig_h":"10.0.0.5","id.orig_p":51820,"id.resp_h":"203.0.113.20","id.resp_p":443,"server_name":"example.com"}`+"\n") + + if fields := readTSVHeader(path); fields != nil { + t.Fatalf("readTSVHeader() on a JSON file = %v, want nil", fields) + } +} + +// newTestClientTSV builds a Client the way NewClient would for a TSV file: +// with tsvFields pre-populated from the file's own header, since the real +// tailer always starts reading from the file's current end (past the +// header) and depends on this separate, from-the-start read to learn the +// column order at all. +func newTestClientTSV(t *testing.T, path string) *Client { + t.Helper() + + return &Client{ + logPath: path, + cache: make(map[string]sniEntry), + tsvFields: readTSVHeader(path), + } +} + +func TestClientProcessNewLines_TSVRealFixture(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine) + + c := newTestClientTSV(t, path) + c.processNewLines() + + domain, ok := c.Lookup("37526", "172.66.147.243", "443") + if !ok { + t.Fatal("Lookup() ok = false, want true for the real captured TSV line") + } + + if domain != "example.com" { + t.Fatalf("Lookup() domain = %q, want %q", domain, "example.com") + } + + // Reverse orientation too. + if domain, ok := c.Lookup("443", "192.168.0.223", "37526"); !ok || domain != "example.com" { + t.Fatalf("Lookup() reverse orientation = (%q, %v), want (%q, true)", domain, ok, "example.com") + } +} + +func TestClientProcessNewLines_TSVUnsetServerNameSkipped(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + // server_name column (index 9) is "-": Zeek's unset-field marker, e.g. a + // resumed session where the name wasn't re-sent. + line := "1790163565.389393\tCabc\t192.168.0.223\t40000\t172.66.147.243\t443\tTLSv13\tTLS_AES_256_GCM_SHA384\tx25519\t-\tT\t-\t-\tT\tCsiI\t-\t-\t-\n" + writeFile(t, path, realZeekTSVHeader+line) + + c := newTestClientTSV(t, path) + c.processNewLines() + + if len(c.cache) != 0 { + t.Fatalf("cache = %v, want empty (unset server_name must be skipped)", c.cache) + } +} + +func TestClientProcessNewLines_TSVShortLineSkipped(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + // Far fewer columns than the header declares — must not panic (index + // out of range) and must not produce a cache entry. + writeFile(t, path, realZeekTSVHeader+"1790163565.389393\tCabc\t192.168.0.223\n") + + c := newTestClientTSV(t, path) + c.processNewLines() + + if len(c.cache) != 0 { + t.Fatalf("cache = %v, want empty for a too-short TSV line", c.cache) + } +} + +func TestClientProcessNewLines_TSVDataBeforeHeaderSkipped(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVDataLine) // no header at all + + // Deliberately NOT using newTestClientTSV: this simulates tsvFields + // never having been learned (e.g. the header lived entirely before this + // client's starting offset in some edge case). + c := newTestClient(t, path) + c.processNewLines() + + if len(c.cache) != 0 { + t.Fatalf("cache = %v, want empty when no #fields header has been seen", c.cache) + } +} + +func TestClientProcessNewLines_TSVRotationRelearnsHeader(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine) + + c := newTestClientTSV(t, path) + c.processNewLines() + + if _, ok := c.Lookup("37526", "172.66.147.243", "443"); !ok { + t.Fatal("Lookup() ok = false before rotation, want true") + } + + // Rotation: a fresh, smaller file appears at the same path, with its own + // header re-written from scratch (a real Zeek restart does exactly this). + newHeader := "#separator \\x09\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tserver_name\n" + newLine := "1790163999.0\tCnew\t10.0.0.5\t51820\t203.0.113.30\t443\tpost-rotation.example\n" + + writeFile(t, path, "") // straddling tick, matches the JSON rotation test above + c.processNewLines() + writeFile(t, path, newHeader+newLine) + c.processNewLines() + + domain, ok := c.Lookup("51820", "203.0.113.30", "443") + if !ok || domain != "post-rotation.example" { + t.Fatalf("Lookup() after rotation = (%q, %v), want (%q, true)", domain, ok, "post-rotation.example") + } +} + +// A nil *Client can sit inside the scanner's SNISource interface; it must answer +// "no SNI", not panic. +func TestNilClientLookup(t *testing.T) { + var c *Client + if sni, ok := c.Lookup("40000", "203.0.113.1", "443"); ok || sni != "" { + t.Fatalf("nil Client.Lookup() = %q, %v; want \"\", false", sni, ok) + } +} + +// Zeek creates ssl.log at the first TLS connection it sees, so it can be missing when +// hexwall starts (a quiet server, right after boot) and between rotations. hexwall must +// keep running, say "waiting" once, and read the file from its first line when it +// appears. Not parallel: it swaps the default logger. +func TestClientWaitsForMissingSSLLog(t *testing.T) { + var logs bytes.Buffer + + previous := slog.Default() + + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + path := filepath.Join(t.TempDir(), "ssl.log") + + c, err := NewClient(&Config{LogPath: path}) + if err != nil || c == nil { + t.Fatalf("NewClient() with no ssl.log yet = %v, %v; want a client that waits", c, err) + } + + c.Close() // stop the 5 s ticker; the test drives each poll itself + + for range 3 { + c.processNewLines() + } + + if got := strings.Count(logs.String(), "waiting for ssl.log"); got != 1 { + t.Fatalf("logged 'waiting' %d times, want 1:\n%s", got, logs.String()) + } + + // The file appears: read from its first line, header included. + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine) + c.processNewLines() + + if got := strings.Count(logs.String(), "ssl.log found"); got != 1 { + t.Fatalf("logged 'found' %d times, want 1:\n%s", got, logs.String()) + } + + if domain, ok := c.Lookup("37526", "172.66.147.243", "443"); !ok || domain != "example.com" { + t.Fatalf("Lookup() = %q, %v; want example.com from the file that appeared", domain, ok) + } +} + +// A rotation removes ssl.log and Zeek creates a new one later. The new file must be +// read from the start, with its own header, even if it grows past the old offset before +// the next poll. +func TestClientReadsNewFileAfterRotation(t *testing.T) { + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, realZeekTSVHeader+strings.Repeat("# padding so the old file is long\n", 20)) + + c := newTestClientTSV(t, path) + c.processNewLines() + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + + c.processNewLines() // sees the gap + + // Longer than the old file: an offset kept from the old file would land past the + // data line and skip it. + writeFile(t, path, realZeekTSVHeader+realZeekTSVDataLine+strings.Repeat("# more traffic\n", 60)) + c.processNewLines() + + if domain, ok := c.Lookup("37526", "172.66.147.243", "443"); !ok || domain != "example.com" { + t.Fatalf("Lookup() after rotation = %q, %v; want example.com", domain, ok) + } +} + +// A read error while looking for the header must be reported, not look like "no header": +// without a header, hexwall can't read any TSV line. A line longer than bufio.Scanner's +// limit (64 KiB) is an easy way to make the read fail. Not parallel: it swaps the logger. +func TestReadTSVHeaderReportsReadError(t *testing.T) { + var logs bytes.Buffer + + previous := slog.Default() + + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + path := filepath.Join(t.TempDir(), "ssl.log") + writeFile(t, path, "#"+strings.Repeat("x", 70*1024)+"\n") + + if fields := readTSVHeader(path); fields != nil { + t.Fatalf("readTSVHeader() = %v, want nil", fields) + } + + if !strings.Contains(logs.String(), "could not read the ssl.log header") { + t.Fatalf("no warning logged for the read error:\n%s", logs.String()) + } +} diff --git a/main.go b/main.go index 7199497..3926b50 100644 --- a/main.go +++ b/main.go @@ -3,22 +3,27 @@ package main import ( "context" + "errors" "flag" "fmt" + "io/fs" "log/slog" "os" "os/signal" + "path/filepath" "runtime" "strings" "syscall" "time" + "github.com/hexbytedev/hexwall/internal/connlist" "github.com/hexbytedev/hexwall/internal/deghost" "github.com/hexbytedev/hexwall/internal/detector" "github.com/hexbytedev/hexwall/internal/monitor" "github.com/hexbytedev/hexwall/internal/pihole" - "github.com/hexbytedev/hexwall/internal/somo" + "github.com/hexbytedev/hexwall/internal/service" "github.com/hexbytedev/hexwall/internal/store" + "github.com/hexbytedev/hexwall/internal/zeek" ) const ( @@ -28,65 +33,181 @@ const ( connectionScanInterval = 10 * time.Second ) -var version = "dev" -var platform = runtime.GOOS + "/" + runtime.GOARCH +var ( + version = "dev" + platform = runtime.GOOS + "/" + runtime.GOARCH +) + +type runConfig struct { + dbPath string + hexwallDBPath string + zeekLogPath string + zeekNoticeLog string + enableZeek bool + enableDeghostIP bool + enableDeghostDomain bool + mode string + debug bool + showVersion bool +} func main() { os.Exit(run()) } +// parseFlags parses hexwall's run flags from args (os.Args[1:] in normal use). +func parseFlags(args []string) (*runConfig, error) { + flags := flag.NewFlagSet("hexwall", flag.ContinueOnError) + dbPath := flags.String("db", "", "path to pihole-FTL.db (auto-detected if not set)") + hexwallDB := flags.String("hexwall-db", "./hexwall.db", "path to local hexwall database") + zeekLogPath := flags.String("zeek-log", "", "path to zeek ssl.log for SNI-based domain verification (empty to skip)") + zeekNoticeLog := flags.String("zeek-notice-log", "/opt/zeek/logs/current/notice.log", "path to Zeek notice.log for DNS-bypass alerts") + enableZeek := flags.Bool("enable-zeek", false, "enable Zeek-based DNS-bypass detection") + enableDeghostIP := flags.Bool("enable-deghost-ip", false, "send unrecognized IPs to the third-party reputation API (deghostapi.hexbyte.dev) on the IP-only fallback path") + enableDeghostDomain := flags.Bool("enable-deghost-domain", false, "send unrecognized SNI domains to the third-party reputation API (deghostapi.hexbyte.dev) at rung 6") + mode := flags.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") + debug := flags.Bool("debug", false, "enable debug-level logging, including verbose per-connection scan logging") + + showVersion := flags.Bool("version", false, "print version and exit") + if err := flags.Parse(args); err != nil { + return nil, err + } + + cfg := &runConfig{ + dbPath: strings.TrimSpace(*dbPath), + hexwallDBPath: strings.TrimSpace(*hexwallDB), + zeekLogPath: strings.TrimSpace(*zeekLogPath), + zeekNoticeLog: strings.TrimSpace(*zeekNoticeLog), + enableZeek: *enableZeek, + enableDeghostIP: *enableDeghostIP, + enableDeghostDomain: *enableDeghostDomain, + mode: strings.TrimSpace(*mode), + debug: *debug, + showVersion: *showVersion, + } + + if cfg.showVersion { + return cfg, nil + } + + selectedMode := strings.ToLower(cfg.mode) + if selectedMode != monitor.ModeWatch && selectedMode != monitor.ModeEnforce { + return nil, fmt.Errorf("invalid --mode value %q: allowed %q or %q", cfg.mode, monitor.ModeWatch, monitor.ModeEnforce) + } + + cfg.mode = selectedMode + + if cfg.hexwallDBPath == "" { + return nil, fmt.Errorf("invalid --hexwall-db value %q", *hexwallDB) + } + + return cfg, nil +} + func run() int { - dbPath := flag.String("db", "", "path to pihole-FTL.db (auto-detected if not set)") - hexwallDB := flag.String("hexwall-db", "./hexwall.db", "path to local hexwall database") - mode := flag.String("mode", monitor.ModeWatch, "monitor mode: watch (detect only) or enforce (kill + log)") - debug := flag.Bool("debug", false, "enable verbose per-connection scan logging") - showVersion := flag.Bool("version", false, "print version and exit") - flag.Parse() - - if *showVersion { + // `hexwall zeek-setup` installs the Zeek script whose notices hexwall reads, so the + // script always matches this binary's parser. + if len(os.Args) > 1 && os.Args[1] == "zeek-setup" { + if err := zeek.Setup(os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "zeek-setup:", err) + return 1 + } + + return 0 + } + + if len(os.Args) > 1 && os.Args[1] == "install-service" { + if err := installService(os.Args[2:]); err != nil { + fmt.Fprintln(os.Stderr, "install-service:", err) + return 1 + } + + return 0 + } + + if len(os.Args) > 1 && os.Args[1] == "uninstall" { + if err := uninstall(); err != nil { + fmt.Fprintln(os.Stderr, "uninstall:", err) + return 1 + } + + return 0 + } + + cfg, err := parseFlags(os.Args[1:]) + if errors.Is(err, flag.ErrHelp) { + return 0 + } + + if err != nil { + slog.Error(err.Error()) + return 1 + } + + if cfg == nil { + return 1 + } + + // Install the log handler first so every later slog call honours --debug. + logLevel := slog.LevelInfo + if cfg.debug { + logLevel = slog.LevelDebug + } + + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))) + + if cfg.showVersion { fmt.Printf("%s (%s)\n", version, platform) return 0 } - selectedMode := strings.ToLower(strings.TrimSpace(*mode)) - if selectedMode != monitor.ModeWatch && selectedMode != monitor.ModeEnforce { - slog.Error("invalid --mode value", "mode", *mode, "allowed", []string{monitor.ModeWatch, monitor.ModeEnforce}) + // Windows is not supported at all, checked before any tool-availability probing. + if runtime.GOOS == "windows" { + slog.Error("windows is not supported by hexwall") return 1 } - resolvedDBPath := strings.TrimSpace(*dbPath) - hexwallDBPath := strings.TrimSpace(*hexwallDB) - if hexwallDBPath == "" { - slog.Error("invalid --hexwall-db value", "path", *hexwallDB) + slog.Info("detected OS", "os", runtime.GOOS) + + if fatal, msg := privilegeCheck(cfg.mode, os.Geteuid()); fatal { + slog.Error(msg) return 1 + } else if msg != "" { + slog.Warn(msg) } // Cancel background work cleanly on Ctrl+C. ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - // 1. Verify that somo is available before starting any monitoring. - if err := somo.CheckInstalled(); err != nil { - slog.Error("somo is not installed", "err", err, "tip", "install somo from https://github.com/theopfr/somo") + // 1. Pick a connection-listing tool: ss (iproute2) if available, else somo. + connectionLister, toolName, err := connlist.SelectBackend() + if err != nil { + fmt.Println("No connection-listing tool available:", err) + slog.Error("no connection-listing tool available", "err", err) + return 1 } + fmt.Printf("> Using %s for connection listing.\n", toolName) + slog.Info("using connection-listing tool", "tool", toolName) + // 2. Resolve the Pi-hole database path from --db or auto-detection. - if resolvedDBPath == "" { + if cfg.dbPath == "" { detected, err := detector.FindDBPath() if err != nil { slog.Error("could not find pi-hole installation", "err", err) return 1 } - resolvedDBPath = detected - slog.Info("pi-hole database auto-detected", "path", resolvedDBPath) + cfg.dbPath = detected + slog.Info("pi-hole database auto-detected", "path", cfg.dbPath) } // 3. Open the Pi-hole database in read-only mode. - checker, err := pihole.NewChecker(&pihole.Config{DBPath: resolvedDBPath}) + checker, err := pihole.NewChecker(&pihole.Config{DBPath: cfg.dbPath}) if err != nil { - slog.Error("failed to open pi-hole database", "path", resolvedDBPath, "err", err) + slog.Error("failed to open pi-hole database", "path", cfg.dbPath, "err", err) return 1 } defer func() { @@ -95,10 +216,38 @@ func run() int { } }() + // 3b. Log policy list summary from gravity.db. + if counts, err := checker.PolicyCounts(); err != nil { + slog.Warn("failed to read pi-hole policy counts", "err", err) + } else if counts != nil { + slog.Info("pihole policy lists loaded", + "allow", counts["vw_allowlist"], + "deny", counts["vw_denylist"], + "gravity", counts["vw_gravity"], + "regex_allow", counts["vw_regex_allowlist"], + "regex_deny", counts["vw_regex_denylist"]) + + if counts["vw_gravity"] == 0 { + slog.Warn("pi-hole gravity list is empty; blocklist may be broken or not yet downloaded") + } + } + + if groups, err := checker.ExtraGroups(); err != nil { + slog.Debug("could not count pi-hole groups", "err", err) + } else if groups > 0 { + slog.Warn("pi-hole uses groups; hexwall treats a block in any group as a block for this machine, which can be stricter than Pi-hole's own rules for it", + "extra_groups", groups) + } + // 4. Open the local hexwall database, creating it if needed. - hexwallStore, err := store.NewStore(hexwallDBPath) + hexwallStore, err := store.NewStore(cfg.hexwallDBPath) if err != nil { - slog.Error("failed to open hexwall database", "path", hexwallDBPath, "err", err) + slog.Error("failed to open hexwall database", "path", cfg.hexwallDBPath, "err", err) + + if strings.Contains(err.Error(), "readonly database") || errors.Is(err, fs.ErrPermission) { + slog.Error("the database file belongs to another user (often root, from an earlier sudo run): run with sudo, or pick another --hexwall-db") + } + return 1 } defer func() { @@ -107,9 +256,87 @@ func run() int { } }() - slog.Info("hexwall database ready", "path", hexwallDBPath) + slog.Info("hexwall database ready", "path", cfg.hexwallDBPath) + + // 4a. Prune stale rows from prior runs at startup. + if result, err := hexwallStore.Prune(); err != nil { + slog.Warn("initial prune failed", "err", err) + } else if result != nil && (result.AllowedIPDomains+result.SNIObservations+result.IPObservations+result.KilledConnections+result.ZeekAlerts) > 0 { + slog.Info("pruned stale rows", + "allowed_ip_domains", result.AllowedIPDomains, + "sni_observations", result.SNIObservations, + "ip_observations", result.IPObservations, + "killed_connections", result.KilledConnections, + "zeek_alerts", result.ZeekAlerts) + } + + // 4b. Start the Zeek ssl.log tailer when a path is provided. + var zeekClient *zeek.Client + + if cfg.zeekLogPath != "" { + zc, err := zeek.NewClient(&zeek.Config{ + LogPath: cfg.zeekLogPath, + }) + if err != nil { + slog.Error("failed to start zeek log watcher", "path", cfg.zeekLogPath, "err", err) + return 1 + } - deghostClient := deghost.NewClient(deghostBaseURL, deghostTimeout) + defer zc.Close() + + zeekClient = zc + } + + // deghost is opt-in: it sends every unrecognized domain to a third-party API, + // which is a real privacy cost. Only construct the client when at least one check is enabled. + var deghostClient *deghost.Client + if cfg.enableDeghostIP || cfg.enableDeghostDomain { + deghostClient = deghost.NewClient(deghostBaseURL, deghostTimeout) + } else { + slog.Info("deghost disabled; unrecognized connections will not be escalated to third-party API") + } + + // Without an SNI source every connection takes the IP-only path, and with IP deghost off + // that path never kills: enforce mode would silently protect nothing. + if cfg.mode == monitor.ModeEnforce && cfg.zeekLogPath == "" && !cfg.enableDeghostIP { + slog.Warn("enforce mode cannot kill any connection: no --zeek-log for Pi-hole rule matching and --enable-deghost-ip is off") + } + + if cfg.mode == monitor.ModeEnforce && cfg.enableDeghostIP { + slog.Warn("enforce mode with IP deghost enabled: an external reputation verdict can now terminate a local process; shared cloud provider ranges are known to produce false positives on such feeds") + } + + scanner := &monitor.Scanner{ + Checker: checker, + Store: hexwallStore, + DeghostClient: deghostClient, + ZeekClient: zeekClient, + ConnectionLister: connectionLister, + Mode: cfg.mode, + DeghostIP: cfg.enableDeghostIP, + DeghostDomain: cfg.enableDeghostDomain, + Debug: cfg.debug, + } + + var zeekEvents = make(chan zeek.Event, 100) + + if cfg.enableZeek && cfg.zeekNoticeLog != "" { + go func() { + if err := zeek.WatchNoticeLog(ctx, cfg.zeekNoticeLog, zeekEvents); err != nil { + slog.Error("zeek notice watcher stopped", "path", cfg.zeekNoticeLog, "err", err) + } + }() + go func() { + for { + select { + case <-ctx.Done(): + return + case event := <-zeekEvents: + scanner.HandleZeekEvent(event) + } + } + }() + } // 5. Refresh trusted IPs before starting the monitor so the first tick does not kill legitimate connections. cache := pihole.NewIPCache(checker, hexwallStore) @@ -126,13 +353,42 @@ func run() int { return case <-ticker.C: cache.Refresh(ctx) + + if err := checker.RefreshRegexLists(); err != nil { + slog.Warn("pi-hole regex list refresh failed; keeping previous rules", "err", err) + } + } + } + }() + + // 6b. Periodically prune stale rows from the hexwall database. + go func() { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if result, err := hexwallStore.Prune(); err != nil { + slog.Warn("periodic prune failed", "err", err) + } else if result != nil && (result.AllowedIPDomains+result.SNIObservations+result.IPObservations+result.KilledConnections+result.ZeekAlerts) > 0 { + slog.Info("pruned stale rows", + "allowed_ip_domains", result.AllowedIPDomains, + "sni_observations", result.SNIObservations, + "ip_observations", result.IPObservations, + "killed_connections", result.KilledConnections, + "zeek_alerts", result.ZeekAlerts) + } } } }() fmt.Println("Starting network monitor...") - fmt.Printf("> Connections will be checked every %s in %s mode.\n", connectionScanInterval, selectedMode) - if *debug { + fmt.Printf("> Connections will be checked every %s in %s mode.\n", connectionScanInterval, cfg.mode) + + if cfg.debug { fmt.Println("> Debug logging is enabled for every scanned connection.") } @@ -146,7 +402,92 @@ func run() int { slog.Info("shutting down") return 0 case <-ticker.C: - monitor.RunScan(ctx, hexwallStore, deghostClient, selectedMode, *debug) + scanner.Run(ctx) } } } + +// installService installs the systemd units that run hexwall with args at every boot. +func installService(args []string) error { + if runtime.GOOS != "linux" { + return errors.New("only Linux with systemd is supported") + } + + args = service.WithSafeMode(args) + // Reject bad flags now: in a unit they would crash-loop at every boot instead. + cfg, err := parseFlags(args) + if err != nil { + return err + } + + binary, err := os.Executable() + if err != nil { + return err + } + + if binary, err = filepath.EvalSymlinks(binary); err != nil { + return err + } + + if strings.Contains(binary, "go-build") { + return errors.New("this binary is temporary (go run); build hexwall and run the built binary instead") + } + + opts := service.Options{Binary: binary, Args: args} + if cfg.enableZeek || cfg.zeekLogPath != "" { + if opts.Zeekctl, err = zeek.Zeekctl(); err != nil { + return err + } + } + + if err := service.Install(os.Stdout, opts); err != nil { + return err + } + + if cfg.mode == monitor.ModeWatch { + fmt.Println("\nRunning in watch mode: nothing is killed. Re-run with --mode enforce once the logs look right.") + } + + return nil +} + +// uninstall removes what install-service and zeek-setup added. The database (the kill and +// alert audit trail) and the binary are kept: hexwall didn't install the binary, and +// deleting evidence should be a separate, deliberate step. +func uninstall() error { + if runtime.GOOS != "linux" { + return errors.New("only Linux with systemd is supported") + } + // Services first, so Zeek stops before its script is removed. + if err := service.Uninstall(os.Stdout); err != nil { + return err + } + + if err := zeek.Teardown(os.Stdout); err != nil { + return err + } + + fmt.Println(` +Kept on purpose: + /var/lib/hexwall the service's database, with its kill and alert history + (delete it with: sudo rm -r /var/lib/hexwall) + this binary delete it yourself if you installed it (e.g. /usr/local/bin/hexwall)`) + + return nil +} + +// privilegeCheck says what running as euid means for mode. Without root, ss can't name +// other users' programs, ss -K can't close sockets (it needs CAP_NET_ADMIN), and other +// users' processes can't be signalled. Enforce mode would then report kills it can't +// make, so it refuses to start; watch mode still sees connections and only warns. +func privilegeCheck(mode string, euid int) (fatal bool, msg string) { + if euid == 0 { + return false, "" + } + + if mode == monitor.ModeEnforce { + return true, "enforce mode needs root: without it hexwall can't close connections or stop other users' programs (run with sudo)" + } + + return false, "not running as root: other users' programs show without a name, and Zeek's logs may be unreadable (run with sudo for the full picture)" +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..9b23676 --- /dev/null +++ b/main_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" + + "github.com/hexbytedev/hexwall/internal/monitor" +) + +func TestPrivilegeCheck(t *testing.T) { + tests := []struct { + name string + mode string + euid int + wantFatal bool + wantMsg bool + }{ + {"root enforce", monitor.ModeEnforce, 0, false, false}, + {"root watch", monitor.ModeWatch, 0, false, false}, + {"user enforce refuses", monitor.ModeEnforce, 1000, true, true}, + {"user watch warns", monitor.ModeWatch, 1000, false, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fatal, msg := privilegeCheck(tt.mode, tt.euid) + if fatal != tt.wantFatal || (msg != "") != tt.wantMsg { + t.Fatalf("privilegeCheck(%q, %d) = %v, %q; want fatal %v, message %v", tt.mode, tt.euid, fatal, msg, tt.wantFatal, tt.wantMsg) + } + }) + } +}