diff --git a/README.md b/README.md index 4e1117321..dfc4df7f7 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,15 @@ throwaway pod that carries the label of the service, so the NetworkPolicy applie Run it at each bump of the image digest. The token never appears in the output. Exits 1 when a target is served, a request without the token is not refused, or the pod opens a connection. | +| | `devops k8s searxng-check [OPTIONS]` | Check the points of the SearXNG service that a bump of the image digest can change. It sends +six search requests with format=json through a local port-forward: one for each category of +the provider, a query that starts with ! and names an engine that is off, a query that starts +with !! (the redirect is recorded and never followed), and a category that no kept engine +serves. It judges the keys of each reply, the hits, and the engines that must stay off. It +also tries three connects from two throwaway pods, so the NetworkPolicy applies to them: a +pod that is not a client of the service to port 8080, and a pod with the label of the service +to the metadata address and to a private ClusterIP. Run it at each bump of the image digest. +Exits 1 when a row fails or a pod opens a connection. | | | `devops k8s create-tls-secret [OPTIONS] ` | Create or update a kubernetes.io/tls secret from certificate and private key files. | | | `devops k8s enable-tls [OPTIONS]` | Generate Homelab certificates and apply TLS secrets across Kubernetes cluster namespaces. | | | `devops k8s validate [OPTIONS] ` | Validate Kubernetes YAML manifests against OpenAPI schemas using Kubeconform. | diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 1d6310c0d..ca0baaaf9 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -1304,6 +1304,30 @@ devops k8s crawl4ai-check [OPTIONS] | `--context` | `string` | - | Kubernetes cluster context name. | | `--dry-run` | `boolean` | - | Preview execution plan without mutating external state. | +### `devops k8s searxng-check` + +**Check the points of the SearXNG service that a bump of the image digest can change. It sends +six search requests with format=json through a local port-forward: one for each category of +the provider, a query that starts with ! and names an engine that is off, a query that starts +with !! (the redirect is recorded and never followed), and a category that no kept engine +serves. It judges the keys of each reply, the hits, and the engines that must stay off. It +also tries three connects from two throwaway pods, so the NetworkPolicy applies to them: a +pod that is not a client of the service to port 8080, and a pod with the label of the service +to the metadata address and to a private ClusterIP. Run it at each bump of the image digest. +Exits 1 when a row fails or a pod opens a connection.** + +```bash +devops k8s searxng-check [OPTIONS] +``` + +**Options:** + +| Option / Flag | Type | Default | Description | +|---|---|---|---| +| `--local-port` | `integer` | `8081` | Local port of the kubectl port-forward to Service searxng. Use a port that no program holds. | +| `--context` | `string` | - | Kubernetes cluster context name. | +| `--dry-run` | `boolean` | - | Preview execution plan without mutating external state. | + ### `devops k8s create-tls-secret` **Create or update a kubernetes.io/tls secret from certificate and private key files.** diff --git a/docs/ERRORS.md b/docs/ERRORS.md index 82daefafe..1947a4618 100644 --- a/docs/ERRORS.md +++ b/docs/ERRORS.md @@ -121,7 +121,7 @@ Each exception class declares its `error_code` and `exit_code`. A command that a | `SecurityError` | `SECURITY_ERROR` | `1` | Security | Base exception for all security, policy, and egress violations. | | `ServiceAddressError` | `K8S_ERROR` | `1` | Devops_cli.k8s.service_proxy | Raised when a cluster service address cannot be resolved. | | `ServiceNotReachableError` | `DEVOPS_CLI_ERROR` | `1` | Devops_cli.k8s.node_port | A service cannot be reached from outside the cluster. | -| `ServiceProbeError` | `K8S_SERVICE_PROBE_ERROR` | `1` | K8s | Raised when `devops k8s crawl4ai-check` cannot probe a service: the forward, the throwaway pod | +| `ServiceProbeError` | `K8S_SERVICE_PROBE_ERROR` | `1` | K8s | Raised when `devops k8s crawl4ai-check` or `devops k8s searxng-check` cannot probe a service: | | `ServiceStatusError` | `SERVICE_STATUS_ERROR` | `1` | Telemetry | Raised when querying upstream published service status fails. | | `ServiceStoppingError` | `SERVICE_STOPPING` | `1` | Roadmap | Raised when a request ends because the Service is stopping (#2753). | | `StackCredentialError` | `K8S_STACK_CREDENTIAL_ERROR` | `1` | K8s | Raised when a stack credential cannot be read, checked, minted, revoked or stored (#2156). | diff --git a/docs/commands/k8s.md b/docs/commands/k8s.md index 51b0aaf0c..ce0ec3383 100644 --- a/docs/commands/k8s.md +++ b/docs/commands/k8s.md @@ -557,6 +557,32 @@ devops k8s crawl4ai-check [OPTIONS] --- +## `devops k8s searxng-check` + +**Check the points of the SearXNG service that a bump of the image digest can change. It sends +six search requests with format=json through a local port-forward: one for each category of +the provider, a query that starts with ! and names an engine that is off, a query that starts +with !! (the redirect is recorded and never followed), and a category that no kept engine +serves. It judges the keys of each reply, the hits, and the engines that must stay off. It +also tries three connects from two throwaway pods, so the NetworkPolicy applies to them: a +pod that is not a client of the service to port 8080, and a pod with the label of the service +to the metadata address and to a private ClusterIP. Run it at each bump of the image digest. +Exits 1 when a row fails or a pod opens a connection.** + +```bash +devops k8s searxng-check [OPTIONS] +``` + +**Options:** + +| Option / Flag | Type | Default | Description | +|---|---|---|---| +| `--local-port` | `integer` | `8081` | Local port of the kubectl port-forward to Service searxng. Use a port that no program holds. | +| `--context` | `string` | - | Kubernetes cluster context name. | +| `--dry-run` | `boolean` | - | Preview execution plan without mutating external state. | + +--- + ## `devops k8s create-tls-secret` **Create or update a kubernetes.io/tls secret from certificate and private key files.** diff --git a/k8s/README.md b/k8s/README.md index 31a01b70f..ade755df3 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -841,7 +841,8 @@ The rule group `crawl4ai` in `monitoring/prometheus-values.yaml` watches the pag 1. Choose the newest tag that is at least 14 days old (the tag list of `https://hub.docker.com/r/searxng/searxng/tags`). 2. Set the tag and the digest in `deployment.yaml`. 3. Refresh `tests/fixtures/searxng/default_engine_names.txt` (the first line names the tag): `docker run --rm --entrypoint cat /usr/local/searxng/searx/settings.yml | python3 -c "import sys, yaml; print(*sorted(e['name'] for e in yaml.safe_load(sys.stdin)['engines']), sep='\n')"`. If a name of `keep_only` is gone, a test fails: read the notes of the release for the new name. - 4. Run `devops ci test tests/invariants/test_k8s_searxng.py`, then start the image with the controls of the pod and query it once with `format=json`. + 4. Run `devops ci test tests/invariants/test_k8s_searxng.py`, then run `devops k8s searxng-check` against the new image (see the next bullet) and set `SEARXNG_IMAGE` in `src/devops_cli/k8s/searxng_check.py`. +- **The check.** `devops k8s searxng-check` tests the running service for the points that a new digest can change. Through a `kubectl port-forward` it sends six `GET /search` requests with `format=json` and no credential: one for each category of the provider (`general`, `it`, `science`), a query `!google ...` that must stay text, a query `!!g ...` whose redirect is recorded and never followed, and a category that no kept engine serves. It fails a row when a reply lacks one of the seven keys (`answers`, `corrections`, `infoboxes`, `query`, `results`, `suggestions`, `unresponsive_engines`), when HTTP 403 shows that `json` is not in `search.formats`, when a category has no hit, or when an engine that must stay off (Brave, Google, Bing, Startpage, Mojeek, Qwant) shows in `results` or in `unresponsive_engines`. It also starts two throwaway pods and tries three connects: a pod that is not a client of the service must not reach port 8080, and a pod with the label of the service must reach neither the metadata address nor a private ClusterIP. The report names each pod and target by a role. A person runs the command at each bump of the digest. Until `SEARXNG_IMAGE` names the new image, the `k8s` row of `devops ci` fails. The query count stays at six, below the about 30 queries after which the upstream sites suspend an engine. If a row fails on the pinned image, do not add code for it: file the evidence as an issue. `devops k8s searxng-check --dry-run` lists every call and makes none. - **The log.** `kubectl -n devops logs deploy/searxng` shows the engines that failed. A line `CAPTCHA` from `duckduckgo` or `Suspended: access denied` from `github` means that the engine refused the pod address for a while. The pod answers with the other engines, and the search passes to the next provider when no engine gives a hit. The start prints a warning that `/var/cache/searxng` is not owned by `searxng:searxng`: the group of the volume is 977 (`fsGroup`), and the pod writes to it. - **The limits.** Measured with the pinned image under the controls of the pod: it idles at 114 MiB, 20 queries one after another peak at 117 MiB, and 10 at once peak at 122 MiB and 0.8 CPU. The Deployment asks for 192 Mi and 50m, with the limits 384 Mi and 1 CPU. Measure again on the cluster after the first sync. diff --git a/openspec/changes/archive/2026-11-08-3929-searxng-check-command/.openspec.yaml b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/.openspec.yaml new file mode 100644 index 000000000..c0ad5ed4d --- /dev/null +++ b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-11-08 diff --git a/openspec/changes/archive/2026-11-08-3929-searxng-check-command/design.md b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/design.md new file mode 100644 index 000000000..3e170cbce --- /dev/null +++ b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/design.md @@ -0,0 +1,28 @@ +# Design + +## Decisions + +- **A `devops k8s` command, not a live pytest test.**: The test floor fakes the network and `docs/TESTING.md` forbids a skip on the state of the host, so a live test cannot reach the service. The owner chose a command on 2026-11-07. The tests of the command use the process and HTTP fakes, and a person runs the command at each digest bump. +- **Two modules in `src/devops_cli/k8s/`, one of them the existing shared one.**: `service_probe.py` (#4108) holds the forward and the throwaway pod and stays as it is. `searxng_check.py` holds the rows, `reply_problems`, the engines that stay off and `SEARXNG_IMAGE`. The command module in `commands/k8s/` is a thin adapter: options, tables and exit code. +- **Six search rows with fixed names.**: `json-general`, `json-it`, `json-science`, `bang-off-engine`, `bang-external` and `unused-category`. The names are the keys of the report and of the tests. Six requests stay far under the limit of about 30 queries after which the upstream sites suspend an engine (`k8s/README.md`). +- **The categories come from `DEFAULT_SEARXNG_CATEGORIES`.**: The provider sends this value, so the check holds the provider to the pinned image. If a kept engine stops answering in a category, the row of that category has no hit and fails, and the fix is to change the default. The check reads the default and does not copy it. +- **`reply_problems(status, body, *, expect_hits=True, also_off=())` returns a list of texts.**: An empty list is a pass. `expect_hits` is false for the rows that need no hit, and `also_off` adds the engine that a bang names. The same function judges the three `json-` rows and the off-engine bang, so a missing key, a bad `unresponsive_engines`, a 403 (named as `search.formats`), a missing hit and an engine that stays off each have one place. A text names the key, the category or the engine, and the report prints it. +- **The engines that stay off are a prefix list.**: `brave`, `braveapi`, `google`, `bing`, `startpage`, `mojeek` and `qwant`, matched by prefix, because `brave` also covers `brave.news`. The list lives once in `searxng_check.py`, and `tests/invariants/test_k8s_searxng.py` imports it. The check reads both `results[].engine` and `unresponsive_engines[][0]`: an engine that is on and suspended shows only in the second. +- **The external bang is recorded and never followed.**: `http_client` follows redirects by default, so the command passes `follow_redirects=False`. SearXNG answers `!!g` with a redirect to the page of the external site, and the command must not send a request there: the request would leave the cluster under the name of a person. The row records the status and whether a `Location` header exists, and fails only on a fault. The provider already refuses such a query before it sends it. +- **The unused category is recorded and not asserted as an error.**: The source (`parse_category_form`) falls back to `general` for a category that is not loaded, so 200 with hits is the expected answer. The row fails on a status of 500 or more or on no answer. +- **Two pods for three connects.**: One pod with a neutral label tries port 8080 of the Service. One pod with the label `app.kubernetes.io/name=searxng` tries the metadata address and the ClusterIP of `roadmap-service`. The first proves that the ingress rule selects the client label and no other. The second proves the egress rule. The pods never become ready (`probe_from_pod`), so the Service sends them no traffic. The neutral label is not the label of `roadmap-service`, the only allowed client, and not the label of `searxng`. +- **The report names pods and targets by role.**: ``, ``, ``, `` and ``. The ClusterIPs are read at run time and never printed (rule 10 of AGENTS.md, rule 8 of `SECURITY.md`). +- **The image check extends the existing check of the `k8s` row.**: `tests/gate/k8s_checks/devops_runtime.py` today asserts only that the image of Deployment `searxng` starts with `docker.io/searxng/searxng:`. It now compares the full image with `SEARXNG_IMAGE`, as it does for Crawl4AI. +- **The reply fixture comes from the Publish stage.**: A saved reply needs the cluster. The Publish stage runs the command once, saves one reply of the pinned image with the masking of private data, and writes the image into a file next to it. The test of the fixture is part of the same group, so no stage between leaves a red suite. +- **The command is not an MCP tool.**: It starts pods, so it is not read-only. `--dry-run` has the class `request-plan`. +- **A changed behavior is not fixed here.**: If a row fails on the pinned image, the person files the evidence as an issue. The change adds no compensating code (AGENTS.md rule 2). + +## Rejected alternatives + +- **A live pytest test with a marker.**: `docs/TESTING.md` forbids a skip on the state of the host, and the owner rejected it on 2026-11-07. +- **Follow the redirect of `!!g`.**: The command would send a request to an external site, and a followed redirect hides whether the service redirects. +- **Assert an error for the unused category.**: The source falls back to `general`, so the assertion would fail on the correct image. +- **A third pod with the label of `roadmap-service` as a positive control.**: It would run under the label of the one allowed client and add a pod and a policy path that nobody asked for. The forward already proves that the service answers, and an exec that fails for another reason raises `ServiceProbeError` and does not read as `blocked`. +- **Read the kept engines from the ConfigMap file.**: The command would depend on the path of the repository. The reply shows the engines that run, and the list of those that stay off is short and fixed. +- **A second copy of the forward and pod code.**: `service_probe.py` was written for this reuse. +- **A new scanner.**: No open, maintained tool checks a running SearXNG for these points. diff --git a/openspec/changes/archive/2026-11-08-3929-searxng-check-command/proposal.md b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/proposal.md new file mode 100644 index 000000000..040b86dfb --- /dev/null +++ b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/proposal.md @@ -0,0 +1,68 @@ +# Proposal + +## Why + +Nothing in the repository fails when a later SearXNG digest bump changes one of three points (#3929). First, the categories in which the kept engines answer. Second, what the service does with a query that starts with `!`: a redirect, or the choice of an engine that the ConfigMap leaves off. Third, the NetworkPolicy peers of the service: which pods reach port 8080, and where the SearXNG pod can connect. + +A live pytest test cannot cover this. The test floor fakes the network and forbids a skip on the state of the host. The owner chose on 2026-11-07 a `devops k8s` command, in the shape of `devops k8s crawl4ai-check` (#3894, merged). A person runs the command at each digest bump. The same item replaces the hand-made search reply fixture with a reply of the pinned image. + +## What Changes + +- Add `devops k8s searxng-check` in `src/devops_cli/commands/k8s/searxng_check.py`. It forwards a local port to Service `searxng`, sends six search requests, tries three connects from throwaway pods, and prints one report row for each. It exits 1 when a request has a problem or a pod reaches a destination that the policy must block. +- Add `src/devops_cli/k8s/searxng_check.py` with the request rows, the judgment of a reply (`reply_problems`), the list of engines that must stay off, and `SEARXNG_IMAGE`, the image that the command was written for. +- Reuse `src/devops_cli/k8s/service_probe.py` (`forward_service`, `probe_from_pod`, `plan_probe_from_pod`, `service_cluster_ip`). The change adds no second copy of the forward or of the pod code. +- The six requests are `GET /search` with `format=json`, and the client follows no redirect: + - `json-general`, `json-it` and `json-science`: a plain query in each category of `DEFAULT_SEARXNG_CATEGORIES`. The reply must hold the seven keys `answers`, `corrections`, `infoboxes`, `query`, `results`, `suggestions` and `unresponsive_engines`, `unresponsive_engines` must be a list, `results` must hold a hit, and no engine of the list that must stay off may appear in `results` or in `unresponsive_engines`. + - `bang-off-engine`: the query `!google ...`. The status must be 200 and the reply must hold no hit of `google`. + - `bang-external`: the query `!!g ...`. SearXNG answers an external bang with a redirect. The row records the status and the `Location` header. The command sends no request to that address. A fault or a missing answer fails the row. + - `unused-category`: a category that no kept engine serves. The row records the status and the number of hits. The status must be 200 or 400 and not a fault. +- The three connects, each from a throwaway pod of `probe_from_pod`: + - A pod with a neutral label, not the label of a client, tries the ClusterIP of Service `searxng` on port 8080. It must be blocked. + - A pod with the label `app.kubernetes.io/name=searxng` tries the metadata address and the ClusterIP of Service `roadmap-service`. Both must be blocked. +- Add `DEFAULT_SEARXNG_CHECK_LOCAL_PORT` to `src/devops_cli/config/defaults.py`, and the messages and help texts of the command, and a `--dry-run` request plan. +- Change the `k8s` row of `devops ci`: the check of `tests/gate/k8s_checks/devops_runtime.py` compares the full image of Deployment `searxng` with `SEARXNG_IMAGE`, so a bump of the digest fails the row until a person runs the command against the new image. +- `tests/invariants/test_k8s_searxng.py` takes the list of engines that must stay off from `devops_cli.k8s.searxng_check`, so the list exists once. +- Replace `tests/fixtures/searxng/search_reply.json` with a reply that the pinned image gave, saved by the Publish stage, and name the image next to it. A test holds the seven keys and the image. +- The command is a new command with a request plan, so the change carries a spec delta (`k8s-searxng-check`) and `skip_specs` is false. The command is not an MCP tool: it starts pods, so it is not read-only. +- Rollout of settings: the change removes, renames or tightens no setting. It adds no setting. `DEFAULT_SEARXNG_CATEGORIES` keeps `general,it,science`. The example ConfigMap and the live ConfigMap need no change, and `tests/config/test_k8s_devops_runtime.py` already loads the example ConfigMap into Settings. +- The change fits one pull request. + +## Capabilities + +### New Capabilities + +- `k8s-searxng-check`: The six requests, the judgment of a reply, the engines that stay off, the redirect rule, the NetworkPolicy peers, the throwaway pods, the pinned image and the report. + +### Modified Capabilities + +None. + +## Impact + +- Code: `src/devops_cli/commands/k8s/searxng_check.py` (new), `src/devops_cli/commands/k8s/__init__.py`, `src/devops_cli/k8s/searxng_check.py` (new), `src/devops_cli/lang/en/messages.py`, `src/devops_cli/lang/en/help.py`, `src/devops_cli/config/defaults.py` +- Tests: `tests/k8s/test_searxng_check.py` (new), `tests/commands/k8s/test_k8s_searxng_check.py` (new), `tests/support/kubectl_calls.py`, `tests/commands/k8s/test_k8s_crawl4ai_check.py`, `tests/gate/k8s_checks/devops_runtime.py`, `tests/invariants/test_k8s_searxng.py`, `tests/ai/test_web_search_searxng.py`, `tests/fixtures/searxng/search_reply.json`, `tests/fixtures/searxng/search_reply_image.txt` (new) +- Documents: `k8s/README.md`, `docs/commands/k8s.md`, `docs/CLI_REFERENCE.md`, `README.md` + +## Feasibility + +The premise is that a command can reach the service, send the requests, and test the NetworkPolicy peers from pods. I checked each part on 2026-11-08 against the repository and the upstream source of the pinned image. Nothing ran against the cluster in the Tests stage. + +- **The shared code exists.** #4108 is merged. `src/devops_cli/k8s/service_probe.py` holds `forward_service`, `probe_from_pod`, `plan_probe_from_pod`, `service_cluster_ip`, `PodTarget` and `PodProbeRow`, and none of them knows a service name. `src/devops_cli/k8s/crawl4ai_check.py` and its command are the shape to follow. +- **No command exists.** `git grep -n searxng-check -- src tests docs k8s README.md` finds no hit in the base commit. No open or merged pull request builds it. +- **The pinned image.** `k8s/devops/searxng/deployment.yaml:37` pins `docker.io/searxng/searxng:2026.10.23-629368e40@sha256:aecad882e86eeb022b76351c8dce4eb48d0cbb16a5f1691f77ea32c96d317502`. The probes of the Deployment use `/healthz`, so the forward waits on that route. Service `searxng` listens on 8080 in namespace `devops`. +- **The format rule.** `searx/webapp.py` at commit 629368e40 calls `flask.abort(403)` when the `format` is not in `settings['search']['formats']`. So a ConfigMap without `json` shows as a 403 on the JSON requests, and the report names `search.formats`. +- **The redirect.** `searx/webapp.py` line 687 returns `redirect(result_container.redirect_url)` for every output format. `searx/query.py` sets `redirect_url` for a known external bang (`!!`). A single `!x` selects an engine or category only when it is in the loaded engines and categories. So `!google` is plain text when the ConfigMap leaves `google` off. The client of `http_client` follows redirects by default (`options.setdefault("follow_redirects", True)`), so the command passes `follow_redirects=False`. +- **The unused category.** `parse_category_form` drops a category that is not loaded and falls back to `general`. So the request gives 200 and general hits, and the row records the status and the number of hits and asserts no error text. This was read from the source and not run. +- **The categories.** The engine files at commit 629368e40 give `duckduckgo` and `wikipedia` in `general`, `github`, `stackoverflow`, `askubuntu`, `superuser`, `mdn` and `docker hub` in `it`, and `arxiv` in `science`. So `DEFAULT_SEARXNG_CATEGORIES = "general,it,science"` is right, and the live rows hold it to the pinned image. +- **By hand on 2026-11-07** (comment on #3929): the roadmap-service pod reaches `searxng:8080` and gets 200, the crawl4ai pod does not, and `/search?format=json` returns the keys `answers`, `corrections`, `infoboxes`, `query`, `results`, `suggestions` and `unresponsive_engines`. The kept engines changed in #4121: `pypi` is removed, nine remain. +- **The NetworkPolicy.** `searxng-perimeter` allows ingress on 8080 from the pod label `roadmap-service` only, and egress to CoreDNS and to public port 443 with the private, shared, link-local and metadata ranges excepted. `kubectl port-forward` is not subject to policy, so the forward reaches the pod, and the pods of `probe_from_pod` test the policy. +- **Rate limit.** `k8s/README.md` records that the upstream sites suspend an engine after about 30 queries. The command sends six queries, so a person can run it several times a day. +- **Not run.** The Tests stage made no request to the cluster and started no pod. The Publish stage runs the command once through a port-forward, puts the result for each request and target role in a pull request comment, and saves the reply fixture. + +## Existing Tool Considered + +`kubectl` (Apache-2.0) covers the forward and the throwaway pod, and `devops_cli.http.entry` covers the requests; both are already wrapped in `service_probe.py`. No open, maintained tool checks a running SearXNG for its reply keys, its bang handling and its NetworkPolicy peers. An offline NetworkPolicy checker reads the policy document only (`tests/commands/k8s/test_k8s_network_policies.py` already does that). So the change builds one command and one module of rows, and reuses the shared probe module. Cost: about 250 lines, an estimated 40k tokens to write, an estimated 8k tokens for each later fix, an estimated 4k tokens a year to keep, and half a day of elapsed time. The change deletes one internal copy: the list of engines that must stay off in `tests/invariants/test_k8s_searxng.py`, which moves into the module. + +## Dependencies + +None. #4108 is merged. diff --git a/openspec/changes/archive/2026-11-08-3929-searxng-check-command/specs/k8s-searxng-check/spec.md b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/specs/k8s-searxng-check/spec.md new file mode 100644 index 000000000..5a257dc47 --- /dev/null +++ b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/specs/k8s-searxng-check/spec.md @@ -0,0 +1,140 @@ +## Purpose +`devops k8s searxng-check` tests a running SearXNG service for the points that a digest bump can change: the keys of the JSON reply, the categories of the kept engines, the handling of a query that starts with `!`, the engines that must stay off, and the NetworkPolicy peers of the pod. A person runs it again at each bump of the pinned image. + +## ADDED Requirements + +### Requirement: Dry run +The command SHALL print each call that it would make and MUST make no request and start no process with `--dry-run`. + +#### Scenario: Dry run lists the calls +- **WHEN** a person runs `devops k8s searxng-check --dry-run` with the process and HTTP fakes installed +- **THEN** the output lists the port-forward, a line for each of the six search rows, the `kubectl run` of the throwaway pods and the connects, and the process fake and the HTTP fake record no call. + +### Requirement: Search rows +The command SHALL send one `GET /search` request with `format=json` for each of the rows `json-general`, `json-it`, `json-science`, `bang-off-engine`, `bang-external` and `unused-category`, and SHALL send every request through the local forward and with no credential. The categories of the three `json-` rows MUST be those of `DEFAULT_SEARXNG_CATEGORIES`. + +#### Scenario: Six requests +- **WHEN** the command runs against a service that answers each row as the pinned image does +- **THEN** the HTTP fake records six requests to `/search`, all to the local forward, all with `format=json`, none with an `Authorization` header, and the command exits 0. + +### Requirement: Reply keys +The command MUST fail a `json-` row whose reply lacks one of the keys `answers`, `corrections`, `infoboxes`, `query`, `results`, `suggestions` and `unresponsive_engines`, or whose `unresponsive_engines` is not a list, and the output MUST name the key. + +#### Scenario: A key is missing +- **WHEN** the service leaves `corrections` out of a reply +- **THEN** the command exits 1 and the output names `corrections`. + +#### Scenario: Unresponsive engines is a string +- **WHEN** the service gives a string for `unresponsive_engines` +- **THEN** the command exits 1 and the output names `unresponsive_engines`. + +### Requirement: Kept categories +The command MUST fail the `json-` row of a category in `DEFAULT_SEARXNG_CATEGORIES` that returns no hit, and the output MUST name the category. + +#### Scenario: A category has no hit +- **WHEN** the service returns no hit for the category `science` +- **THEN** the command exits 1 and the output names `science`. + +### Requirement: Format json +The command MUST fail a `json-` row that gets HTTP 403, and the output MUST name `search.formats`. + +#### Scenario: Json is not a format +- **WHEN** the service answers the JSON requests with HTTP 403 +- **THEN** the command exits 1 and the output names `search.formats`. + +### Requirement: Engines that stay off +The command MUST fail a row whose reply names an engine that must stay off in `results` or in `unresponsive_engines`. The engines are `brave`, `braveapi`, `google`, `bing`, `startpage`, `mojeek` and `qwant`, matched as a prefix. The output MUST name the engine. + +#### Scenario: A keyed engine is on +- **WHEN** a reply holds a hit whose engine is `google` +- **THEN** the command exits 1 and the output names `google`. + +#### Scenario: A suspended engine is listed as unresponsive +- **WHEN** `unresponsive_engines` of a reply holds the engine `bing` +- **THEN** the command exits 1 and the output names `bing`. + +### Requirement: Bang that names an engine that is off +The command SHALL send a query that starts with `!` and the name of an engine that the ConfigMap leaves off, and MUST fail the row `bang-off-engine` when the status is not 200 or when the reply holds a hit of that engine. + +#### Scenario: The bang stays text +- **WHEN** the service answers the query with HTTP 200 and hits of kept engines only +- **THEN** the row `bang-off-engine` passes. + +#### Scenario: The bang selects the engine +- **WHEN** the service answers the query with a hit of the engine that is off +- **THEN** the command exits 1 and the output names that engine. + +### Requirement: External bang is not followed +The command SHALL send a query that starts with `!!`, MUST NOT follow a redirect, MUST send no request to the address of a `Location` header, and SHALL record the status of the answer. The row MUST fail on a status of 500 or more and on no answer. + +#### Scenario: Redirect is not followed +- **WHEN** the service answers the query with HTTP 302 and a `Location` header +- **THEN** the report holds the status 302 for the row `bang-external`, the HTTP fake records no request to the `Location` address, and the command exits 0. + +### Requirement: Category that no kept engine serves +The command SHALL send a request with a category that no kept engine serves, SHALL record its status and its number of hits, and MUST fail the row on a status of 500 or more or on no answer. + +#### Scenario: Unused category +- **WHEN** the service answers the unused category with HTTP 200 and one hit +- **THEN** the report holds one row `unused-category` with the status 200 and the count 1, and the command exits 0. + +### Requirement: Forward to the service +The command SHALL forward a local port to Service `searxng` in namespace `devops`, SHALL wait for `/healthz`, SHALL stop the forward when it ends, and MUST stop before it sends a search request when the forward ends at once or `/healthz` gives no answer. The code of the forward and of the pods SHALL be the code of `src/devops_cli/k8s/service_probe.py`, and the command module MUST define no second copy. + +#### Scenario: The forward ends at once +- **WHEN** the process fake answers the forward with a child that has ended +- **THEN** the command exits 1 with a stated reason and the HTTP fake records no request. + +#### Scenario: Health gives no answer +- **WHEN** `/healthz` gives no answer +- **THEN** the command exits 1, sends no search request and stops the forward. + +### Requirement: NetworkPolicy peers +The command SHALL start a throwaway pod with a neutral label that is not the label of a client of the service, and MUST fail when it connects to port 8080 of Service `searxng`. It SHALL start a throwaway pod with the label `app.kubernetes.io/name=searxng`, and MUST fail when it connects to the metadata address or to the ClusterIP of Service `roadmap-service`. It SHALL delete each pod after its probes. + +#### Scenario: Both pods are blocked +- **WHEN** each connect probe of each pod fails +- **THEN** the report holds three `blocked` rows, the command exits 0, and the last `kubectl` call deletes a pod. + +#### Scenario: The unlabeled pod connects +- **WHEN** the connect from the pod with the neutral label to port 8080 succeeds +- **THEN** the command exits 1, the report names the unlabeled pod and the service as `reached`, and the pods are deleted. + +#### Scenario: The labeled pod reaches the metadata address +- **WHEN** the connect from the pod with the label of `searxng` to the metadata address succeeds +- **THEN** the command exits 1 and the report names the SearXNG pod and the metadata role as `reached`. + +### Requirement: Pinned image +An offline check of the `k8s` row of `devops ci` MUST compare the image of Deployment `searxng` with the image that the command names. + +#### Scenario: Image bump +- **WHEN** the Deployment pins an image that differs from `SEARXNG_IMAGE` +- **THEN** the check fails, names both images and the command that a person runs again. + +### Requirement: Roles in the output +The command and its output MUST name each pod and each target by a role and MUST NOT hold a private host name or a private address. + +#### Scenario: Report rows +- **WHEN** the command runs for all rows +- **THEN** each pod and target appears as a role such as ``, ``, ``, `` or ``, and no row holds a metadata address, a cluster host name or a ClusterIP. + +### Requirement: Context +The command SHALL pass `--context` to every `kubectl` call. + +#### Scenario: Context option +- **WHEN** a person runs the command with `--context test-ctx` +- **THEN** every recorded `kubectl` call names the context `test-ctx`. + +### Requirement: Reply fixture from the pinned image +The reply fixture of the SearXNG provider tests SHALL be a reply that the pinned image gave, and a file next to it SHALL name that image. + +#### Scenario: Fixture names the image +- **WHEN** a test reads the fixture and `SEARXNG_IMAGE` +- **THEN** the fixture holds the seven reply keys and the file next to it holds `SEARXNG_IMAGE`. + +### Requirement: Hermetic tests +The tests of the command MUST run with the process and HTTP fakes and MUST NOT skip on the state of the host. + +#### Scenario: No cluster +- **WHEN** the suite runs on a host with no cluster +- **THEN** each test of the command runs to a pass or a fail, none skips, and no test opens a socket. diff --git a/openspec/changes/archive/2026-11-08-3929-searxng-check-command/tasks.md b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/tasks.md new file mode 100644 index 000000000..5b4e101b2 --- /dev/null +++ b/openspec/changes/archive/2026-11-08-3929-searxng-check-command/tasks.md @@ -0,0 +1,50 @@ +# Tasks + +## 1. The rows and the judgment of a reply + +Kind: behavior. Reads: `src/devops_cli/k8s/crawl4ai_check.py`, `src/devops_cli/k8s/service_probe.py`, `src/devops_cli/ai/web_search.py`, `tests/invariants/test_k8s_searxng.py`. Changes: `tests/k8s/test_searxng_check.py` (new), `src/devops_cli/k8s/searxng_check.py` (new). + +- [x] 1.1 Write `tests/k8s/test_searxng_check.py`. The tests assert `reply_problems` for a full reply, a missing key, an `unresponsive_engines` that is no list, a 403, an empty `results`, an engine that stays off in `results` and in `unresponsive_engines`, an engine that the caller names, and a body that is no JSON object, and they assert the shape of `SEARXNG_IMAGE` and the reuse of the shared probe code. Run it and see it fail + - Verification: `$ uv run devops ci test tests/k8s/test_searxng_check.py` fails on the base +- [x] 1.2 Add `src/devops_cli/k8s/searxng_check.py` with `SEARXNG_IMAGE`, `REPLY_KEYS`, `OFF_ENGINES`, the six row names, the request of each row, and `reply_problems` + - Verification: `$ uv run devops ci test tests/k8s/test_searxng_check.py` passes + +## 2. The command + +Kind: behavior. Reads: `src/devops_cli/commands/k8s/crawl4ai_check.py`, `src/devops_cli/commands/k8s/__init__.py`, `src/devops_cli/lang/en/messages.py`, `src/devops_cli/lang/en/help.py`, `tests/support/kubectl_calls.py`. Changes: `tests/commands/k8s/test_k8s_searxng_check.py` (new), `tests/support/kubectl_calls.py`, `tests/commands/k8s/test_k8s_crawl4ai_check.py`, `src/devops_cli/commands/k8s/searxng_check.py` (new), `src/devops_cli/commands/k8s/__init__.py`, `src/devops_cli/lang/en/messages.py`, `src/devops_cli/lang/en/help.py`, `src/devops_cli/config/defaults.py`. + +- [x] 2.1 Write `tests/commands/k8s/test_k8s_searxng_check.py`. The tests assert the six requests, the reply keys, the categories, the 403, the engines that stay off, the two bang rows, the unused category, the two pods and their probes, the roles in the output, the forward that ends at once, the health route that gives no answer, the context option and the dry run. Move the fake forward child into `tests/support/kubectl_calls.py` for both command tests. Run them and see them fail + - Verification: `$ uv run devops ci test tests/commands/k8s/test_k8s_searxng_check.py` fails on the base +- [x] 2.2 Add `src/devops_cli/commands/k8s/searxng_check.py` in the shape of `k8s crawl4ai-check`: the options `--local-port`, `--context` and `--dry-run`, the class `request-plan`, the tables, and the exit 1 on a problem row or a reached destination. Register it as `searxng-check`, and add its messages, help texts and `DEFAULT_SEARXNG_CHECK_LOCAL_PORT` + - Verification: `$ uv run devops ci test tests/commands/k8s/test_k8s_searxng_check.py tests/commands/k8s/test_k8s_crawl4ai_check.py tests/invariants/test_dry_run_class_ratchet.py` passes + +## 3. Fail the gate on an image bump + +Kind: other. Reads: `tests/gate/k8s_checks/devops_runtime.py`, `tests/invariants/test_k8s_searxng.py`, `k8s/devops/searxng/deployment.yaml`. Changes: `tests/gate/k8s_checks/devops_runtime.py`, `tests/invariants/test_k8s_searxng.py`. + +- [x] 3.1 Make the check of Deployment `searxng` in `tests/gate/k8s_checks/devops_runtime.py` compare the full image with `SEARXNG_IMAGE`, and name both images and the command in the failure text + - Verification: `$ uv run devops ci --only k8s` passes +- [x] 3.2 Make `tests/invariants/test_k8s_searxng.py` import `OFF_ENGINES` from `devops_cli.k8s.searxng_check` and delete its own `FORBIDDEN_ENGINES` + - Verification: `$ uv run devops ci test tests/invariants/test_k8s_searxng.py` passes + +## 4. Documents + +Kind: documents. Reads: `k8s/README.md`, `docs/commands/k8s.md`. Changes: `k8s/README.md`, `docs/commands/k8s.md`, `docs/CLI_REFERENCE.md`, `README.md`. + +- [x] 4.1 In the SearXNG section of `k8s/README.md`, add a bullet that names the command, what it checks, that a person runs it at each bump of the digest, and that the `k8s` row of `devops ci` fails until `SEARXNG_IMAGE` matches + - Verification: `$ uv run devops ci docs` passes +- [x] 4.2 Run `uv run devops docs generate --sync-readme` so the command documents list `devops k8s searxng-check` + - Verification: `$ uv run devops ci docs` passes + +## 5. The reply fixture of the pinned image + +Kind: other. Reads: `tests/ai/test_web_search_searxng.py`, `tests/fixtures/searxng/search_reply.json`. Changes: `tests/k8s/test_searxng_check.py`, `tests/ai/test_web_search_searxng.py`, `tests/fixtures/searxng/search_reply.json`, `tests/fixtures/searxng/search_reply_image.txt` (new). + +- [x] 5.1 Run `uv run devops k8s searxng-check` once against the cluster through a port-forward, and save one live reply of the pinned image (the `it` row, cut to its first three hits, because the `general` row gave one hit while duckduckgo answered CAPTCHA) as `tests/fixtures/searxng/search_reply.json`, with each private value masked. Write `SEARXNG_IMAGE` into `tests/fixtures/searxng/search_reply_image.txt` + - Verification: `$ uv run devops ci test tests/ai/test_web_search_searxng.py` passes +- [x] 5.2 Add a test to `tests/k8s/test_searxng_check.py` that the fixture holds the seven reply keys, no engine of `OFF_ENGINES`, and that the file next to it holds `SEARXNG_IMAGE` + - Verification: `$ uv run devops ci test tests/k8s/test_searxng_check.py` passes + +## Workflow follow-up + +- Pending a person: review the diff in tuicr (tuicr -r ..HEAD). diff --git a/openspec/specs/k8s-searxng-check/spec.md b/openspec/specs/k8s-searxng-check/spec.md new file mode 100644 index 000000000..38211eb19 --- /dev/null +++ b/openspec/specs/k8s-searxng-check/spec.md @@ -0,0 +1,142 @@ +# k8s-searxng-check Specification + +## Purpose +`devops k8s searxng-check` tests a running SearXNG service for the points that a digest bump can change: the keys of the JSON reply, the categories of the kept engines, the handling of a query that starts with `!`, the engines that must stay off, and the NetworkPolicy peers of the pod. A person runs it again at each bump of the pinned image. + +## Requirements + +### Requirement: Dry run +The command SHALL print each call that it would make and MUST make no request and start no process with `--dry-run`. + +#### Scenario: Dry run lists the calls +- **WHEN** a person runs `devops k8s searxng-check --dry-run` with the process and HTTP fakes installed +- **THEN** the output lists the port-forward, a line for each of the six search rows, the `kubectl run` of the throwaway pods and the connects, and the process fake and the HTTP fake record no call. + +### Requirement: Search rows +The command SHALL send one `GET /search` request with `format=json` for each of the rows `json-general`, `json-it`, `json-science`, `bang-off-engine`, `bang-external` and `unused-category`, and SHALL send every request through the local forward and with no credential. The categories of the three `json-` rows MUST be those of `DEFAULT_SEARXNG_CATEGORIES`. + +#### Scenario: Six requests +- **WHEN** the command runs against a service that answers each row as the pinned image does +- **THEN** the HTTP fake records six requests to `/search`, all to the local forward, all with `format=json`, none with an `Authorization` header, and the command exits 0. + +### Requirement: Reply keys +The command MUST fail a `json-` row whose reply lacks one of the keys `answers`, `corrections`, `infoboxes`, `query`, `results`, `suggestions` and `unresponsive_engines`, or whose `unresponsive_engines` is not a list, and the output MUST name the key. + +#### Scenario: A key is missing +- **WHEN** the service leaves `corrections` out of a reply +- **THEN** the command exits 1 and the output names `corrections`. + +#### Scenario: Unresponsive engines is a string +- **WHEN** the service gives a string for `unresponsive_engines` +- **THEN** the command exits 1 and the output names `unresponsive_engines`. + +### Requirement: Kept categories +The command MUST fail the `json-` row of a category in `DEFAULT_SEARXNG_CATEGORIES` that returns no hit, and the output MUST name the category. + +#### Scenario: A category has no hit +- **WHEN** the service returns no hit for the category `science` +- **THEN** the command exits 1 and the output names `science`. + +### Requirement: Format json +The command MUST fail a `json-` row that gets HTTP 403, and the output MUST name `search.formats`. + +#### Scenario: Json is not a format +- **WHEN** the service answers the JSON requests with HTTP 403 +- **THEN** the command exits 1 and the output names `search.formats`. + +### Requirement: Engines that stay off +The command MUST fail a row whose reply names an engine that must stay off in `results` or in `unresponsive_engines`. The engines are `brave`, `braveapi`, `google`, `bing`, `startpage`, `mojeek` and `qwant`, matched as a prefix. The output MUST name the engine. + +#### Scenario: A keyed engine is on +- **WHEN** a reply holds a hit whose engine is `google` +- **THEN** the command exits 1 and the output names `google`. + +#### Scenario: A suspended engine is listed as unresponsive +- **WHEN** `unresponsive_engines` of a reply holds the engine `bing` +- **THEN** the command exits 1 and the output names `bing`. + +### Requirement: Bang that names an engine that is off +The command SHALL send a query that starts with `!` and the name of an engine that the ConfigMap leaves off, and MUST fail the row `bang-off-engine` when the status is not 200 or when the reply holds a hit of that engine. + +#### Scenario: The bang stays text +- **WHEN** the service answers the query with HTTP 200 and hits of kept engines only +- **THEN** the row `bang-off-engine` passes. + +#### Scenario: The bang selects the engine +- **WHEN** the service answers the query with a hit of the engine that is off +- **THEN** the command exits 1 and the output names that engine. + +### Requirement: External bang is not followed +The command SHALL send a query that starts with `!!`, MUST NOT follow a redirect, MUST send no request to the address of a `Location` header, and SHALL record the status of the answer. The row MUST fail on a status of 500 or more and on no answer. + +#### Scenario: Redirect is not followed +- **WHEN** the service answers the query with HTTP 302 and a `Location` header +- **THEN** the report holds the status 302 for the row `bang-external`, the HTTP fake records no request to the `Location` address, and the command exits 0. + +### Requirement: Category that no kept engine serves +The command SHALL send a request with a category that no kept engine serves, SHALL record its status and its number of hits, and MUST fail the row on a status of 500 or more or on no answer. + +#### Scenario: Unused category +- **WHEN** the service answers the unused category with HTTP 200 and one hit +- **THEN** the report holds one row `unused-category` with the status 200 and the count 1, and the command exits 0. + +### Requirement: Forward to the service +The command SHALL forward a local port to Service `searxng` in namespace `devops`, SHALL wait for `/healthz`, SHALL stop the forward when it ends, and MUST stop before it sends a search request when the forward ends at once or `/healthz` gives no answer. The code of the forward and of the pods SHALL be the code of `src/devops_cli/k8s/service_probe.py`, and the command module MUST define no second copy. + +#### Scenario: The forward ends at once +- **WHEN** the process fake answers the forward with a child that has ended +- **THEN** the command exits 1 with a stated reason and the HTTP fake records no request. + +#### Scenario: Health gives no answer +- **WHEN** `/healthz` gives no answer +- **THEN** the command exits 1, sends no search request and stops the forward. + +### Requirement: NetworkPolicy peers +The command SHALL start a throwaway pod with a neutral label that is not the label of a client of the service, and MUST fail when it connects to port 8080 of Service `searxng`. It SHALL start a throwaway pod with the label `app.kubernetes.io/name=searxng`, and MUST fail when it connects to the metadata address or to the ClusterIP of Service `roadmap-service`. It SHALL delete each pod after its probes. + +#### Scenario: Both pods are blocked +- **WHEN** each connect probe of each pod fails +- **THEN** the report holds three `blocked` rows, the command exits 0, and the last `kubectl` call deletes a pod. + +#### Scenario: The unlabeled pod connects +- **WHEN** the connect from the pod with the neutral label to port 8080 succeeds +- **THEN** the command exits 1, the report names the unlabeled pod and the service as `reached`, and the pods are deleted. + +#### Scenario: The labeled pod reaches the metadata address +- **WHEN** the connect from the pod with the label of `searxng` to the metadata address succeeds +- **THEN** the command exits 1 and the report names the SearXNG pod and the metadata role as `reached`. + +### Requirement: Pinned image +An offline check of the `k8s` row of `devops ci` MUST compare the image of Deployment `searxng` with the image that the command names. + +#### Scenario: Image bump +- **WHEN** the Deployment pins an image that differs from `SEARXNG_IMAGE` +- **THEN** the check fails, names both images and the command that a person runs again. + +### Requirement: Roles in the output +The command and its output MUST name each pod and each target by a role and MUST NOT hold a private host name or a private address. + +#### Scenario: Report rows +- **WHEN** the command runs for all rows +- **THEN** each pod and target appears as a role such as ``, ``, ``, `` or ``, and no row holds a metadata address, a cluster host name or a ClusterIP. + +### Requirement: Context +The command SHALL pass `--context` to every `kubectl` call. + +#### Scenario: Context option +- **WHEN** a person runs the command with `--context test-ctx` +- **THEN** every recorded `kubectl` call names the context `test-ctx`. + +### Requirement: Reply fixture from the pinned image +The reply fixture of the SearXNG provider tests SHALL be a reply that the pinned image gave, and a file next to it SHALL name that image. + +#### Scenario: Fixture names the image +- **WHEN** a test reads the fixture and `SEARXNG_IMAGE` +- **THEN** the fixture holds the seven reply keys and the file next to it holds `SEARXNG_IMAGE`. + +### Requirement: Hermetic tests +The tests of the command MUST run with the process and HTTP fakes and MUST NOT skip on the state of the host. + +#### Scenario: No cluster +- **WHEN** the suite runs on a host with no cluster +- **THEN** each test of the command runs to a pass or a fail, none skips, and no test opens a socket. diff --git a/src/devops_cli/commands/k8s/__init__.py b/src/devops_cli/commands/k8s/__init__.py index c9ce1535d..0b13098c8 100644 --- a/src/devops_cli/commands/k8s/__init__.py +++ b/src/devops_cli/commands/k8s/__init__.py @@ -33,6 +33,7 @@ service_url, ) from devops_cli.commands.k8s.ollama import app as ollama_app +from devops_cli.commands.k8s.searxng_check import k8s_searxng_check from devops_cli.commands.k8s.security_audit import ( k8s_audit, k8s_check_deprecated, @@ -84,6 +85,7 @@ app.command("check-deprecated")(k8s_check_deprecated) app.command("image-pins")(k8s_image_pins) app.command("crawl4ai-check")(k8s_crawl4ai_check) +app.command("searxng-check")(k8s_searxng_check) app.command("create-tls-secret")(create_tls_secret) app.command("enable-tls")(enable_tls_stack) app.command("validate")(k8s_validate) diff --git a/src/devops_cli/commands/k8s/searxng_check.py b/src/devops_cli/commands/k8s/searxng_check.py new file mode 100644 index 000000000..74b7b7cb5 --- /dev/null +++ b/src/devops_cli/commands/k8s/searxng_check.py @@ -0,0 +1,146 @@ +"""`devops k8s searxng-check`: what a digest bump of SearXNG can change (#3929).""" + +from __future__ import annotations + +from typing import Annotated + +import httpx2 +import typer + +from devops_cli.config.defaults import ( + DEFAULT_SEARXNG_CHECK_LOCAL_PORT, + DEFAULT_SEARXNG_TIMEOUT_SECONDS, +) +from devops_cli.dry_run import dry_run_class +from devops_cli.dry_run.requests import render_request_plan +from devops_cli.dry_run.state import is_dry_run, set_dry_run +from devops_cli.exceptions.k8s import ServiceProbeError +from devops_cli.http.egress import EgressLevel +from devops_cli.http.entry import http_client +from devops_cli.k8s.searxng_check import ( + HEALTH_PATH, + NAMESPACE, + SEARXNG_IMAGE, + SERVICE, + SERVICE_PORT, + PodRow, + SearchResult, + check_network_policy, + forward_service, + plan_check, + send_searches, +) +from devops_cli.lang import HELP, MESSAGES +from devops_cli.output import TableColumn, TablePayload, print, print_success, print_warning + +_STATUS_NONE = "no answer" +_PASSED = "ok" +_FAILED = "FAILED" + + +def _print_searches(rows: list[SearchResult]) -> None: + print( + TablePayload( + title=MESSAGES.k8s.table_title_searxng_search, + columns=[ + TableColumn(header="Row"), + TableColumn(header="Status"), + TableColumn(header="Hits"), + TableColumn(header="Result"), + ], + rows=[ + [ + r.name, + _STATUS_NONE if r.status is None else str(r.status), + "-" if r.hits is None else str(r.hits), + _FAILED if r.problems else _PASSED, + ] + for r in rows + ], + ) + ) + for row in rows: + for problem in row.problems: + print_warning(f"{row.name}: {problem}") + + +def _print_pods(rows: list[PodRow]) -> None: + print( + TablePayload( + title=MESSAGES.k8s.table_title_searxng_pod, + columns=[ + TableColumn(header="Pod"), + TableColumn(header="Destination"), + TableColumn(header="Result"), + ], + rows=[[r.pod, r.destination, "reached" if r.reached else "blocked"] for r in rows], + ) + ) + + +def _send_requests(local_port: int, context: str | None) -> list[SearchResult]: + """Forward the port, send the six rows and stop the forward.""" + with ( + forward_service( + service=SERVICE, + namespace=NAMESPACE, + remote_port=SERVICE_PORT, + local_port=local_port, + health_path=HEALTH_PATH, + context=context, + ) as base_url, + http_client( + EgressLevel.LOOPBACK, + timeout=httpx2.Timeout(DEFAULT_SEARXNG_TIMEOUT_SECONDS), + follow_redirects=False, + ) as client, + ): + return list(send_searches(client, base_url)) + + +@dry_run_class("request-plan") +def k8s_searxng_check( + local_port: Annotated[ + int, typer.Option("--local-port", help=HELP.k8s.searxng_check_local_port) + ] = DEFAULT_SEARXNG_CHECK_LOCAL_PORT, + context: Annotated[str | None, typer.Option("--context", help=HELP.options.context)] = None, + dry_run: Annotated[bool, typer.Option("--dry-run", help=HELP.options.dry_run)] = False, +) -> None: + """Check the points of the SearXNG service that a bump of the image digest can change. It sends + six search requests with format=json through a local port-forward: one for each category of + the provider, a query that starts with ! and names an engine that is off, a query that starts + with !! (the redirect is recorded and never followed), and a category that no kept engine + serves. It judges the keys of each reply, the hits, and the engines that must stay off. It + also tries three connects from two throwaway pods, so the NetworkPolicy applies to them: a + pod that is not a client of the service to port 8080, and a pod with the label of the service + to the metadata address and to a private ClusterIP. Run it at each bump of the image digest. + Exits 1 when a row fails or a pod opens a connection.""" + set_dry_run(dry_run) + if dry_run or is_dry_run(): + render_request_plan( + "devops k8s searxng-check would make these calls:", + plan_check(local_port, context), + [MESSAGES.k8s.searxng_check_dry_run], + ) + return + searches = _send_requests(local_port, context) + _print_searches(searches) + pods = check_network_policy(context) + _print_pods(pods) + failed: list[str] = [] + if bad := [r.name for r in searches if r.problems]: + failed.append(MESSAGES.k8s.searxng_check_failed_rows.format(rows=", ".join(bad))) + if opened := [f"{r.pod} -> {r.destination}" for r in pods if r.reached]: + failed.append(MESSAGES.k8s.searxng_check_failed_pod.format(connects=", ".join(opened))) + if failed: + raise ServiceProbeError( + MESSAGES.k8s.searxng_check_failed.format(count=len(failed), detail="; ".join(failed)) + ) + print_success( + MESSAGES.k8s.searxng_check_passed.format( + rows=len(searches), destinations=len(pods), image=SEARXNG_IMAGE + ) + ) + + +__all__ = ["forward_service", "k8s_searxng_check"] diff --git a/src/devops_cli/config/constants.py b/src/devops_cli/config/constants.py index 3aeaa71bd..d65e9634a 100644 --- a/src/devops_cli/config/constants.py +++ b/src/devops_cli/config/constants.py @@ -2945,7 +2945,7 @@ class WebhookExposure(StrEnum): "169.254.42.42", # Scaleway ) ) -# The metadata address that every major cloud answers on. `devops k8s crawl4ai-check` aims at it. +# The metadata address that every major cloud answers on. `devops k8s crawl4ai-check` and `devops k8s searxng-check` aim at it. CONST_CLOUD_METADATA_PRIMARY_IPV4: Final[ipaddress.IPv4Address] = ipaddress.IPv4Address( "169.254.169.254" ) diff --git a/src/devops_cli/config/defaults.py b/src/devops_cli/config/defaults.py index af0ea0f36..e3cc00ce4 100644 --- a/src/devops_cli/config/defaults.py +++ b/src/devops_cli/config/defaults.py @@ -609,6 +609,11 @@ # Seconds that one request of the check may take. A refusal is fast; a slow reply is a failure. DEFAULT_CRAWL4AI_CHECK_TIMEOUT_SECONDS: float = 30.0 +# ── SearXNG Check Defaults (`devops k8s searxng-check`, #3929) ────────────────── +# Local port of the `kubectl port-forward` to Service `searxng`. It differs from the port 8080 that a +# person may forward for `search.searxng_url`, so the check never meets a forward of a person. +DEFAULT_SEARXNG_CHECK_LOCAL_PORT: int = 8081 + # ── Service Probe Defaults (throwaway pod and port-forward of `devops_cli.k8s.service_probe`) ── # Seconds that a forward has to answer its health route before the check stops. DEFAULT_SERVICE_FORWARD_WAIT_SECONDS: float = 30.0 diff --git a/src/devops_cli/exceptions/k8s.py b/src/devops_cli/exceptions/k8s.py index 88e4115d8..7678b6c92 100644 --- a/src/devops_cli/exceptions/k8s.py +++ b/src/devops_cli/exceptions/k8s.py @@ -48,8 +48,8 @@ class ImagePinError(KubernetesError): class ServiceProbeError(KubernetesError): - """Raised when `devops k8s crawl4ai-check` cannot probe a service: the forward, the throwaway pod - or the probe fails, or a check fails (#3894).""" + """Raised when `devops k8s crawl4ai-check` or `devops k8s searxng-check` cannot probe a service: + the forward, the throwaway pod or the probe fails, or a check fails (#3894, #3929).""" error_code = "K8S_SERVICE_PROBE_ERROR" diff --git a/src/devops_cli/k8s/crawl4ai_check.py b/src/devops_cli/k8s/crawl4ai_check.py index 3ab9a5770..d2561969e 100644 --- a/src/devops_cli/k8s/crawl4ai_check.py +++ b/src/devops_cli/k8s/crawl4ai_check.py @@ -35,6 +35,7 @@ forward_argv, plan_probe_from_pod, probe_from_pod, + read_capped, service_cluster_ip, ) @@ -129,15 +130,6 @@ class RequestRow: refused: bool -def _read_capped(response: httpx2.Response) -> bytes: - body = bytearray() - for chunk in response.iter_bytes(): - body += chunk - if len(body) > DEFAULT_CRAWL4AI_MAX_REPLY_BYTES: - break - return bytes(body) - - def _send( client: httpx2.Client, base_url: str, route: Route, url: str, headers: dict[str, str] ) -> tuple[int, bytes]: @@ -159,7 +151,7 @@ def _send( ) response = client.send(request, stream=True) try: - return response.status_code, _read_capped(response) + return response.status_code, read_capped(response, DEFAULT_CRAWL4AI_MAX_REPLY_BYTES) finally: response.close() diff --git a/src/devops_cli/k8s/searxng_check.py b/src/devops_cli/k8s/searxng_check.py new file mode 100644 index 000000000..d02567387 --- /dev/null +++ b/src/devops_cli/k8s/searxng_check.py @@ -0,0 +1,343 @@ +"""The checks of `devops k8s searxng-check`: what a digest bump of SearXNG can change (#3929). + +The Service `searxng` is the search provider of the roadmap Service (#3927). A bump of the pinned +image can change six points, and this module tests each one on a running cluster: + +- The reply: `GET /search?format=json` answers HTTP 200 with a JSON object of seven keys. The + format `json` must be in `search.formats`, or the service answers HTTP 403. +- The categories: each category of the provider (`DEFAULT_SEARXNG_CATEGORIES`) returns a hit. +- The engines: the scraping and keyed engines of `OFF_ENGINES` stay off. A hit of one of them, or + one of them in `unresponsive_engines`, means that the settings keep an engine that must stay off. +- The bang `!name`: a name of an engine that is off stays text and selects no engine. +- The bang `!!name`: the service answers with a redirect to an external site. The check records the + answer and never follows it. +- A category that no kept engine serves: the check records the status and the number of hits. +- The NetworkPolicy of the pod (`k8s/devops/searxng/networkpolicy*.yaml`): a pod that is not a + client of the service gets no connection to port 8080, and a pod with the label of the service + cannot open a connection to the metadata address or to a private ClusterIP. + +The forward and the throwaway pods are the code of `devops_cli.k8s.service_probe`. The report names +each pod and target by a role, never by an address or a host name. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final, Literal + +import httpx2 + +from devops_cli.config.constants import CONST_CLOUD_METADATA_PRIMARY_IPV4 +from devops_cli.config.defaults import DEFAULT_SEARXNG_CATEGORIES, DEFAULT_SEARXNG_MAX_REPLY_BYTES +from devops_cli.dry_run.requests import PlannedRequest +from devops_cli.http.entry import request_headers +from devops_cli.k8s.service_probe import ( + PodProbeRow, + PodTarget, + forward_argv, + forward_service, + plan_probe_from_pod, + probe_from_pod, + read_capped, + service_cluster_ip, +) + +# The image the check was written for. The gate row `k8s` compares it with the image of +# Deployment `searxng`, so a bump of the digest fails until a person runs `devops k8s +# searxng-check` against the new image and sets this value. +SEARXNG_IMAGE = "docker.io/searxng/searxng:2026.10.23-629368e40@sha256:aecad882e86eeb022b76351c8dce4eb48d0cbb16a5f1691f77ea32c96d317502" + +NAMESPACE = "devops" +SERVICE = "searxng" +SERVICE_PORT = 8080 +HEALTH_PATH = "/healthz" +POD_LABELS = {"app.kubernetes.io/name": SERVICE} +# A label that is neither the label of the service nor the label of its one client. +UNLABELED_POD_LABELS = {"app.kubernetes.io/name": "searxng-check-unlabeled"} +POD_NAME_PREFIX = "searxng-check" +UNLABELED_POD_NAME_PREFIX = "searxng-check-unlabeled" + +# The keys of a reply of `/search?format=json`, read from the pinned image on 2026-11-07. +REPLY_KEYS: Final = ( + "answers", + "corrections", + "infoboxes", + "query", + "results", + "suggestions", + "unresponsive_engines", +) +# The scraping engines that the upstream sites suspended in October 2026 (SearXNG issues 6866 and +# 6819), and the keyed Brave engine, which the owner dropped on 2026-11-07. A name here is a name +# of the default settings.yml, and a prefix: `brave` also covers `brave.news`. +OFF_ENGINES: Final = ("brave", "braveapi", "google", "bing", "startpage", "mojeek", "qwant") + +ROLE_UNLABELED_POD = "" +ROLE_SEARXNG_POD = "" +ROLE_SEARXNG_SERVICE = "" +ROLE_METADATA = "" +ROLE_PRIVATE_CLUSTER_IP = "" + +_ROADMAP_SERVICE = "roadmap-service" +_ROADMAP_PORT = 8000 +_METADATA_PORT = 80 +_QUERY = "python" +_QUERY_OFF_ENGINE = "!google python" +_QUERY_EXTERNAL = "!!g python" +# A category that the provider does not send and no kept engine serves. +_UNUSED_CATEGORY = "images" +_GENERAL_CATEGORY = "general" +_FAULT_STATUS = 500 +_FORMAT_REFUSED_STATUS = 403 + +Judge = Literal["reply", "text", "recorded"] + + +@dataclass(frozen=True) +class SearchSpec: + """One search row: the query and the categories of the request, and how to judge the reply. + + `reply`: a full reply with a hit. `text`: a full reply, with no hit needed, for a query that + must stay text. `recorded`: the status is recorded and only a fault or no answer fails. + """ + + name: str + query: str + categories: str + judge: Judge + + +@dataclass(frozen=True) +class SearchResult: + """What the service answered to one row. `status` is None when the request got no answer, and + `hits` is None when the body holds no list of results.""" + + name: str + status: int | None + hits: int | None + problems: tuple[str, ...] + + +@dataclass(frozen=True) +class PodRow: + """The result of one connect: the role of the pod, the role of the destination, and whether + the pod opened a connection.""" + + pod: str + destination: str + reached: bool + + +def search_specs() -> tuple[SearchSpec, ...]: + """The six rows, in order. The categories of the first three are those of the provider.""" + return ( + *( + SearchSpec(f"json-{category}", _QUERY, category, "reply") + for category in DEFAULT_SEARXNG_CATEGORIES.split(",") + ), + SearchSpec("bang-off-engine", _QUERY_OFF_ENGINE, _GENERAL_CATEGORY, "text"), + SearchSpec("bang-external", _QUERY_EXTERNAL, _GENERAL_CATEGORY, "recorded"), + SearchSpec("unused-category", _QUERY, _UNUSED_CATEGORY, "recorded"), + ) + + +def _off_engines(engines: list[object], names: tuple[str, ...]) -> list[str]: + """The names in `engines` that start with one of `names`.""" + return [e for e in engines if isinstance(e, str) and e.startswith(names)] + + +def reply_problems( + status: int, body: bytes, *, expect_hits: bool = True, also_off: tuple[str, ...] = () +) -> list[str]: + """One text for each point of a `/search?format=json` reply that fails. An empty list is a pass. + + The points are the status, the seven keys, the type of `unresponsive_engines`, a hit (only when + `expect_hits`), and an engine of `OFF_ENGINES` or `also_off` in `results` or in + `unresponsive_engines`. An engine that is on and suspended shows only in the second list. + """ + if status == _FORMAT_REFUSED_STATUS: + return ["HTTP 403: the format json is not in search.formats of the settings"] + if status != httpx2.codes.OK: + return [f"HTTP {status}, not 200"] + try: + reply = json.loads(body) + except ValueError: + reply = None + if not isinstance(reply, dict): + return ["the body is not a JSON object"] + problems = [f"the reply lacks the key {key}" for key in REPLY_KEYS if key not in reply] + names = (*OFF_ENGINES, *also_off) + off: list[str] = [] + results = reply.get("results") + if "results" in reply: + if not isinstance(results, list): + problems.append("results is not a list") + else: + if expect_hits and not results: + problems.append("the reply holds no hit") + off += _off_engines([h.get("engine") for h in results if isinstance(h, dict)], names) + unresponsive = reply.get("unresponsive_engines") + if "unresponsive_engines" in reply: + if not isinstance(unresponsive, list): + problems.append("unresponsive_engines is not a list") + else: + off += _off_engines( + [i[0] for i in unresponsive if isinstance(i, list | tuple) and i], names + ) + problems += [f"the engine {engine} must stay off" for engine in dict.fromkeys(off)] + return problems + + +def _hit_count(body: bytes) -> int | None: + try: + reply = json.loads(body) + except ValueError: + return None + results = reply.get("results") if isinstance(reply, dict) else None + return len(results) if isinstance(results, list) else None + + +def judge_search(spec: SearchSpec, status: int | None, body: bytes) -> SearchResult: + """The verdict for one row. A row with no problem passes.""" + if status is None: + return SearchResult(spec.name, None, None, ("the request got no answer",)) + hits = _hit_count(body) + if spec.judge == "recorded": + problems = [f"HTTP {status}, a fault of the service"] if status >= _FAULT_STATUS else [] + else: + problems = reply_problems(status, body, expect_hits=spec.judge == "reply") + return SearchResult(spec.name, status, hits, tuple(problems)) + + +def _search(client: httpx2.Client, base_url: str, spec: SearchSpec) -> SearchResult: + request = client.build_request( + "GET", + f"{base_url}/search", + params={"q": spec.query, "format": "json", "categories": spec.categories}, + headers=request_headers({"Accept": "application/json"}), + ) + try: + response = client.send(request, stream=True, follow_redirects=False) + try: + status, body = ( + response.status_code, + read_capped(response, DEFAULT_SEARXNG_MAX_REPLY_BYTES), + ) + finally: + response.close() + except httpx2.HTTPError: + return judge_search(spec, None, b"") + return judge_search(spec, status, body) + + +def send_searches(client: httpx2.Client, base_url: str) -> Iterator[SearchResult]: + """Send the six rows through the forward, with no credential. The client must not follow a + redirect: the answer of `!!g` points to an external site.""" + for spec in search_specs(): + yield _search(client, base_url, spec) + + +def check_network_policy(context: str | None = None) -> list[PodRow]: + """Try the three connects of the NetworkPolicy from two throwaway pods. A row with `reached` + true means that the policy lets the connect through. + + A pod with a neutral label tries port 8080 of the Service. A pod with the label of the service + tries the metadata address and the ClusterIP of the roadmap Service. + """ + searxng_ip = service_cluster_ip(NAMESPACE, SERVICE, context) + roadmap_ip = service_cluster_ip(NAMESPACE, _ROADMAP_SERVICE, context) + unlabeled = probe_from_pod( + namespace=NAMESPACE, + labels=UNLABELED_POD_LABELS, + targets=[PodTarget(ROLE_SEARXNG_SERVICE, searxng_ip, SERVICE_PORT)], + name_prefix=UNLABELED_POD_NAME_PREFIX, + context=context, + ) + labeled = probe_from_pod( + namespace=NAMESPACE, + labels=POD_LABELS, + targets=[ + PodTarget(ROLE_METADATA, str(CONST_CLOUD_METADATA_PRIMARY_IPV4), _METADATA_PORT), + PodTarget(ROLE_PRIVATE_CLUSTER_IP, roadmap_ip, _ROADMAP_PORT), + ], + name_prefix=POD_NAME_PREFIX, + context=context, + ) + return [ + *(PodRow(ROLE_UNLABELED_POD, row.role, row.reached) for row in unlabeled), + *(PodRow(ROLE_SEARXNG_POD, row.role, row.reached) for row in labeled), + ] + + +def plan_check(local_port: int, context: str | None = None) -> list[PlannedRequest]: + """Every call of the check, in order, with a role in place of each address.""" + plan = [ + PlannedRequest( + method="kubectl port-forward", + target=f"svc/{SERVICE}", + argv=tuple(forward_argv(SERVICE, NAMESPACE, SERVICE_PORT, local_port, context)), + ), + PlannedRequest(method="GET", target=HEALTH_PATH), + ] + plan.extend( + PlannedRequest( + method="GET", + target=f"/search {spec.name}", + condition=f"q={spec.query} categories={spec.categories} format=json", + ) + for spec in search_specs() + ) + plan.append(PlannedRequest(method="kubectl get service", target=SERVICE)) + plan.append(PlannedRequest(method="kubectl get service", target=_ROADMAP_SERVICE)) + for labels, prefix, targets in ( + ( + UNLABELED_POD_LABELS, + UNLABELED_POD_NAME_PREFIX, + [PodTarget(ROLE_SEARXNG_SERVICE, "", SERVICE_PORT)], + ), + ( + POD_LABELS, + POD_NAME_PREFIX, + [ + PodTarget(ROLE_METADATA, "", _METADATA_PORT), + PodTarget(ROLE_PRIVATE_CLUSTER_IP, "", _ROADMAP_PORT), + ], + ), + ): + plan.extend( + plan_probe_from_pod( + namespace=NAMESPACE, + labels=labels, + targets=targets, + name_prefix=prefix, + context=context, + ) + ) + return plan + + +__all__ = [ + "HEALTH_PATH", + "NAMESPACE", + "OFF_ENGINES", + "REPLY_KEYS", + "SEARXNG_IMAGE", + "SERVICE", + "SERVICE_PORT", + "PodProbeRow", + "PodRow", + "SearchResult", + "SearchSpec", + "check_network_policy", + "forward_service", + "judge_search", + "plan_check", + "plan_probe_from_pod", + "probe_from_pod", + "reply_problems", + "search_specs", + "send_searches", + "service_cluster_ip", +] diff --git a/src/devops_cli/k8s/service_probe.py b/src/devops_cli/k8s/service_probe.py index 8eb79ac0f..d95231fdf 100644 --- a/src/devops_cli/k8s/service_probe.py +++ b/src/devops_cli/k8s/service_probe.py @@ -9,8 +9,7 @@ of the workload applies to it, tries a TCP connect to each target from inside the pod, and deletes the pod. A connect that fails means that the policy blocks the destination. -The module knows no service name. `devops k8s crawl4ai-check` uses it, and the SearXNG check of -#3929 reuses it. +The module knows no service name. `devops k8s crawl4ai-check` and `devops k8s searxng-check` (#3929) use it. The pod must pass two checks of the cluster: @@ -108,6 +107,16 @@ def _kubectl(argv: list[str]) -> tuple[int, str, str]: return result.returncode, result.stdout or "", result.stderr or "" +def read_capped(response: httpx2.Response, limit: int) -> bytes: + """The body of a streamed reply, cut once it passes `limit` bytes.""" + body = bytearray() + for chunk in response.iter_bytes(): + body += chunk + if len(body) > limit: + break + return bytes(body) + + def _label_selector(labels: Mapping[str, str]) -> str: return ",".join(f"{key}={value}" for key, value in labels.items()) diff --git a/src/devops_cli/lang/en/help.py b/src/devops_cli/lang/en/help.py index 6c7c784fa..516a63b9a 100644 --- a/src/devops_cli/lang/en/help.py +++ b/src/devops_cli/lang/en/help.py @@ -463,6 +463,7 @@ class K8sCommandHelp: lint_target: str = "Target K8s manifest file or directory to lint." pluto_target: str = "Target manifest file or directory to scan for deprecated APIs." crawl4ai_check_local_port: str = "Local port of the kubectl port-forward to Service crawl4ai. Use a port that no program holds." + searxng_check_local_port: str = "Local port of the kubectl port-forward to Service searxng. Use a port that no program holds." image_pins_root: str = "Repository root whose Argo CD Applications in k8s/argocd/ are rendered." rbac_namespace: str = ( "Audit only this namespace's RoleBindings and Roles; ClusterRoleBindings are always read." diff --git a/src/devops_cli/lang/en/messages.py b/src/devops_cli/lang/en/messages.py index e4f077ee1..905cd9307 100644 --- a/src/devops_cli/lang/en/messages.py +++ b/src/devops_cli/lang/en/messages.py @@ -729,6 +729,19 @@ class K8sMessages: crawl4ai_check_dry_run: str = ( "A dry run sends no request and starts no process. The calls above run in this order." ) + searxng_check_passed: str = ( + "All {rows} search rows of SearXNG passed, and the pods open no connection to the " + "{destinations} destinations. The check was written for the image {image}." + ) + searxng_check_failed: str = ( + "{count} check(s) of the SearXNG service failed: {detail}. Do not fix this in code: " + "file the evidence as an issue, and set the settings or the image to match." + ) + searxng_check_failed_rows: str = "the search rows {rows} failed" + searxng_check_failed_pod: str = "the pods opened these connections: {connects}" + searxng_check_dry_run: str = ( + "A dry run sends no request and starts no process. The calls above run in this order." + ) rbac_audit_passed: str = ( "RBAC audit passed: no broad or wildcard role granted to a non-system subject " "in {bindings} binding(s)." @@ -839,6 +852,8 @@ class K8sMessages: table_title_crawl4ai_internal: str = "Crawl4AI: requests with the token for internal targets" table_title_crawl4ai_open: str = "Crawl4AI: requests without the token" table_title_crawl4ai_pod: str = "Pod with the label of the service: connects" + table_title_searxng_search: str = "SearXNG: search requests with format=json" + table_title_searxng_pod: str = "SearXNG: connects from throwaway pods" table_title_image_pins: str = "Image Pins of the Argo CD Applications" table_title_kubeconform: str = "Kubeconform Strict Schema Validation" table_title_nodes: str = "Nodes" diff --git a/tests/ai/test_web_search_searxng.py b/tests/ai/test_web_search_searxng.py index eb43371bc..afe73f5ff 100644 --- a/tests/ai/test_web_search_searxng.py +++ b/tests/ai/test_web_search_searxng.py @@ -38,9 +38,32 @@ _SERVICE_ADDRESS = "10.0.0.5" _QUERY = "private roadmap query" _FROM_FIXTURE = [ - SearchHit("https://example.com/pydantic-ai", "Pydantic AI", "Agents for Python.", "searxng"), - SearchHit("https://example.com/ruff", "Ruff", "A linter.", "searxng"), - SearchHit("https://example.com/mypy", "Mypy", "A type checker.", "searxng"), + SearchHit( + "https://developer.mozilla.org/en-US/docs/Glossary/Python", + "Python", + "Python is a high-level general-purpose programming language. It uses a multi-paradigm " + "approach, meaning it supports procedural, object-oriented, and some functional " + "programming constructs.", + "searxng", + ), + SearchHit( + "https://developer.mozilla.org/en-US/docs/Learn_web_development/Extensions/Server-side/Django", + "Django Web Framework (Python)", + "Django is an extremely popular and fully featured server-side web framework, written in " + "Python. This module shows you why Django is one of the most popular web server " + "frameworks, how to set up a development environment, and how to start using it to " + "create your own web applications.", + "searxng", + ), + SearchHit( + "https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model", + "Document Object Model (DOM)", + "The Document Object Model (DOM) connects web pages to scripts or programming languages " + "by representing the structure of a document\u2014such as the HTML representing a web " + "page\u2014in memory. Usually it refers to JavaScript, even though modeling HTML, SVG, or " + "XML documents as objects are not part of the core JavaScript language.", + "searxng", + ), ] _DUCKDUCKGO = "https://html.duckduckgo.com/html/" diff --git a/tests/commands/k8s/test_k8s_crawl4ai_check.py b/tests/commands/k8s/test_k8s_crawl4ai_check.py index 43901ac6e..90621e1db 100644 --- a/tests/commands/k8s/test_k8s_crawl4ai_check.py +++ b/tests/commands/k8s/test_k8s_crawl4ai_check.py @@ -25,6 +25,7 @@ from tests.support.fakes.keyring import FakeKeyring from tests.support.fakes.process import ProcessAnswer, ProcessCall, ProcessFake from tests.support.kubectl_calls import ( + RunningForward, context_of, flag, kubectl_calls, @@ -57,15 +58,6 @@ _METADATA_ADDRESSES = ("169.254.169.254",) -class _RunningForward: - """The `kubectl port-forward` child while it runs. Its pid is one that no process holds.""" - - pid = -1 - - def poll(self) -> int | None: - return None - - def _route(request: RecordedRequest) -> str: path = urlsplit(request.url).path return "/llm/{url}" if path.startswith("/llm/") else path @@ -132,7 +124,7 @@ def cluster( """A cluster that blocks the pod, a service that refuses each internal target, a running forward, the token in the keyring and a ClusterIP for the Service of the roadmap.""" keyring_store.set_password("devops-cli", "crawl4ai_api_token", _TOKEN) - process_fake.answer(lambda call: call.detached, ProcessAnswer(process=_RunningForward())) + process_fake.answer(lambda call: call.detached, ProcessAnswer(process=RunningForward())) process_fake.answer( lambda call: call.argv[:1] == ("kubectl",) and "get" in call.argv, ProcessAnswer(stdout=_CLUSTER_IP), diff --git a/tests/commands/k8s/test_k8s_searxng_check.py b/tests/commands/k8s/test_k8s_searxng_check.py new file mode 100644 index 000000000..0f507505a --- /dev/null +++ b/tests/commands/k8s/test_k8s_searxng_check.py @@ -0,0 +1,558 @@ +"""`devops k8s searxng-check`: the points that a digest bump of SearXNG can change (#3929). + +The command forwards a local port to Service `searxng`, sends six search requests with +`format=json`, and tries three connects from throwaway pods, so the NetworkPolicy applies to them. +The tests fake the process edge and the HTTP edge, and none skips on the state of the host. +""" + +from __future__ import annotations + +import json +import signal +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlsplit + +import httpx2 +import pytest + +from devops_cli.commands.k8s import searxng_check as command +from devops_cli.config.defaults import DEFAULT_SEARXNG_CATEGORIES +from devops_cli.k8s import service_probe +from tests.support.fakes.clock import ClockFake +from tests.support.fakes.http import RecordedRequest, WebFake +from tests.support.fakes.process import ProcessAnswer, ProcessCall, ProcessFake +from tests.support.kubectl_calls import ( + RunningForward, + context_of, + flag, + kubectl_calls, + namespace_of, + verb, + without_context, +) + +_PORT = 18236 +_BASE = f"http://127.0.0.1:{_PORT}" +_SEARXNG_IP = "192.0.2.10" +_ROADMAP_IP = "192.0.2.20" +_METADATA = "169.254.169.254" +_EXTERNAL = "https://example.com/redirected" +_ROWS = ( + "json-general", + "json-it", + "json-science", + "bang-off-engine", + "bang-external", + "unused-category", +) +_CATEGORIES = DEFAULT_SEARXNG_CATEGORIES.split(",") +# The engine that answers in each category, as the pinned image does. +_ENGINE = {"general": "duckduckgo", "it": "github", "science": "arxiv"} +_KEYS = ("answers", "corrections", "infoboxes", "query", "results", "suggestions") + + +@dataclass(frozen=True) +class _Search: + q: str + categories: str + format: str + + @property + def kind(self) -> str: + if self.q.startswith("!!"): + return "bang-external" + if self.q.startswith("!"): + return "bang-off-engine" + return "plain" + + +def _search(request: httpx2.Request | RecordedRequest) -> _Search: + params = dict(httpx2.URL(str(request.url)).params) + return _Search(params.get("q", ""), params.get("categories", ""), params.get("format", "")) + + +def _reply(search: _Search, **changes: Any) -> bytes: + engine = _ENGINE.get(search.categories, _ENGINE["general"]) + reply: dict[str, Any] = { + "query": search.q, + "number_of_results": 0, + "results": [ + {"url": "https://example.com/a", "title": "A", "content": "B", "engine": engine} + ], + "answers": [], + "corrections": [], + "infoboxes": [], + "suggestions": [], + "unresponsive_engines": [], + } + reply.update(changes) + return json.dumps(reply).encode() + + +class _Service: + """SearXNG behind the forward, as the pinned image answers. `/healthz` is open, a query that + starts with `!!` gets a redirect, and every other query gets a reply with one hit of the + engine of its category. A test sets `answer` to give another reply for one search.""" + + def __init__(self) -> None: + self.answer: Callable[[_Search], tuple[int, dict[str, str], bytes] | None] | None = None + + def __call__(self, request: httpx2.Request) -> httpx2.Response | None: + if request.url.host != "127.0.0.1" or request.url.port != _PORT: + return None + if request.url.path == "/healthz": + return httpx2.Response(200, content=b"OK") + search = _search(request) + custom = self.answer(search) if self.answer else None + if custom: + status, headers, body = custom + return httpx2.Response(status, headers=headers, content=body) + if search.kind == "bang-external": + return httpx2.Response(302, headers={"location": _EXTERNAL}) + return httpx2.Response( + 200, headers={"content-type": "application/json"}, content=_reply(search) + ) + + +@pytest.fixture +def service(web: WebFake) -> _Service: + """The service, installed on the HTTP fake.""" + fake = _Service() + web.serve(fake) + return fake + + +def _is_exec(call: ProcessCall) -> bool: + return call.argv[:1] == ("kubectl",) and "exec" in call.argv + + +def _is_get_of(name: str) -> Callable[[ProcessCall], bool]: + return lambda call: call.argv[:1] == ("kubectl",) and "get" in call.argv and name in call.argv + + +@pytest.fixture +def cluster(service: _Service, process_fake: ProcessFake, clock: ClockFake) -> ProcessFake: + """A cluster that blocks every pod, a running forward, and the ClusterIP of each Service.""" + process_fake.answer(lambda call: call.detached, ProcessAnswer(process=RunningForward())) + process_fake.answer(_is_get_of("searxng"), ProcessAnswer(stdout=_SEARXNG_IP)) + process_fake.answer(_is_get_of("roadmap-service"), ProcessAnswer(stdout=_ROADMAP_IP)) + process_fake.answer(_is_exec, ProcessAnswer(returncode=1)) + return process_fake + + +def _check(cli: Callable[..., Any], *args: str) -> Any: + return cli("k8s", "searxng-check", "--local-port", str(_PORT), *args) + + +def _searches(web: WebFake) -> list[_Search]: + return [_search(r) for r in web.recorded if urlsplit(r.url).path == "/search"] + + +def _pod_of(argv: tuple[str, ...]) -> str: + return without_context(argv)[2] + + +def _exec_targets(cluster: ProcessFake, pod: str) -> list[tuple[str, str]]: + """The host and the port of each connect probe that ran in `pod`.""" + return [ + (argv[-2], argv[-1]) + for argv in kubectl_calls(cluster) + if verb(argv) == "exec" and _pod_of(argv) == pod + ] + + +def _lines(result: Any, *words: str) -> list[str]: + return [line for line in result.output.splitlines() if all(w in line for w in words)] + + +def test_six_requests_go_through_the_forward_with_format_json_and_no_credential( + cli: Callable[..., Any], cluster: ProcessFake, web: WebFake +) -> None: + result = _check(cli) + + searches = _searches(web) + plain = [s for s in searches if s.kind == "plain"] + unused = {s.categories for s in plain} - set(_CATEGORIES) + assert ( + result.exit_code, + len(searches), + sorted(s.kind for s in searches), + {s.categories for s in plain} - unused, + len(unused), + {s.format for s in searches}, + {urlsplit(r.url).netloc for r in web.recorded}, + [r for r in web.recorded if "authorization" in r.headers], + ) == ( + 0, + 6, + ["bang-external", "bang-off-engine", "plain", "plain", "plain", "plain"], + set(_CATEGORIES), + 1, + {"json"}, + {f"127.0.0.1:{_PORT}"}, + [], + ) + + +def test_the_report_has_one_row_for_each_search( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + result = _check(cli) + + assert {row: len(_lines(result, row)) >= 1 for row in _ROWS} == {row: True for row in _ROWS} + + +def test_the_forward_goes_to_service_searxng_and_stops_when_the_command_ends( + cli: Callable[..., Any], cluster: ProcessFake, web: WebFake +) -> None: + _check(cli) + + forward = next(call for call in cluster.calls if call.detached) + assert ( + forward.argv[:2], + "svc/searxng" in forward.argv, + f"{_PORT}:8080" in forward.argv, + namespace_of(forward.argv), + cluster.terminated_pids, + ) == (("kubectl", "port-forward"), True, True, "devops", [(-1, signal.SIGTERM)]) + assert next(r.url for r in web.recorded) == f"{_BASE}/healthz" + + +@pytest.mark.parametrize("key", _KEYS) +def test_a_reply_without_a_key_fails_the_check_and_names_the_key( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service, key: str +) -> None: + def leave_out(search: _Search) -> tuple[int, dict[str, str], bytes] | None: + if search.categories != "general" or search.kind != "plain": + return None + reply = json.loads(_reply(search)) + del reply[key] + return 200, {}, json.dumps(reply).encode() + + service.answer = leave_out + + result = _check(cli) + + assert (result.exit_code, key in result.output) == (1, True) + + +def test_unresponsive_engines_that_is_a_string_fails_the_check( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + service.answer = lambda search: ( + (200, {}, _reply(search, unresponsive_engines="duckduckgo: timeout")) + if search.kind == "plain" and search.categories == "it" + else None + ) + + result = _check(cli) + + assert (result.exit_code, "unresponsive_engines" in result.output) == (1, True) + + +def test_a_category_of_the_provider_with_no_hit_fails_the_check_and_names_it( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + service.answer = lambda search: ( + (200, {}, _reply(search, results=[])) + if search.kind == "plain" and search.categories == "science" + else None + ) + + result = _check(cli) + + assert (result.exit_code, _lines(result, "json-science", "FAILED") != []) == (1, True) + assert "science" in result.output + + +def test_a_service_that_refuses_json_fails_the_check_and_names_the_formats_setting( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + service.answer = lambda search: (403, {}, b"Forbidden") + + result = _check(cli) + + assert (result.exit_code, "search.formats" in result.output) == (1, True) + + +def test_a_hit_of_an_engine_that_must_stay_off_fails_the_check_and_names_the_engine( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + def keyed(search: _Search) -> tuple[int, dict[str, str], bytes] | None: + if search.kind != "plain" or search.categories != "general": + return None + hit = {"url": "https://example.com/g", "title": "G", "content": "G", "engine": "google"} + return 200, {}, _reply(search, results=[hit]) + + service.answer = keyed + + result = _check(cli) + + assert (result.exit_code, "google" in result.output) == (1, True) + + +def test_an_engine_that_must_stay_off_and_is_unresponsive_fails_the_check( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + service.answer = lambda search: ( + (200, {}, _reply(search, unresponsive_engines=[["bing", "CAPTCHA"]])) + if search.kind == "plain" and search.categories == "general" + else None + ) + + result = _check(cli) + + assert (result.exit_code, "bing" in result.output) == (1, True) + + +def test_the_external_bang_redirect_is_recorded_and_never_followed( + cli: Callable[..., Any], cluster: ProcessFake, web: WebFake +) -> None: + result = _check(cli) + + assert ( + result.exit_code, + len(_lines(result, "bang-external", "302")), + [r.url for r in web.recorded if "example.com" in urlsplit(r.url).netloc], + ) == (0, 1, []) + + +def test_an_external_bang_that_gets_a_fault_fails_the_check( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + service.answer = lambda search: (500, {}, b"error") if search.kind == "bang-external" else None + + result = _check(cli) + + assert (result.exit_code, _lines(result, "bang-external", "500") != []) == (1, True) + + +def test_a_bang_for_an_engine_that_is_off_stays_text( + cli: Callable[..., Any], cluster: ProcessFake, web: WebFake +) -> None: + result = _check(cli) + + (bang,) = (s for s in _searches(web) if s.kind == "bang-off-engine") + assert (result.exit_code, bang.q.startswith("!google ")) == (0, True) + + +def test_a_bang_that_selects_the_engine_that_is_off_fails_the_check_and_names_it( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + def selects(search: _Search) -> tuple[int, dict[str, str], bytes] | None: + if search.kind != "bang-off-engine": + return None + hit = {"url": "https://example.com/g", "title": "G", "content": "G", "engine": "google"} + return 200, {}, _reply(search, results=[hit]) + + service.answer = selects + + result = _check(cli) + + assert (result.exit_code, _lines(result, "bang-off-engine", "FAILED") != []) == (1, True) + assert "google" in result.output + + +def test_a_bang_that_redirects_fails_the_check( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + service.answer = lambda search: ( + (302, {"location": _EXTERNAL}, b"") if search.kind == "bang-off-engine" else None + ) + + result = _check(cli) + + assert (result.exit_code, _lines(result, "bang-off-engine", "FAILED") != []) == (1, True) + + +def test_the_unused_category_row_records_its_status_and_its_number_of_hits( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + result = _check(cli) + + rows = _lines(result, "unused-category") + assert (result.exit_code, len(rows), "200" in rows[0], "1" in rows[0]) == (0, 1, True, True) + + +def test_an_unused_category_that_gets_a_fault_fails_the_check( + cli: Callable[..., Any], cluster: ProcessFake, service: _Service +) -> None: + service.answer = lambda search: ( + (500, {}, b"error") + if search.kind == "plain" and search.categories not in _CATEGORIES + else None + ) + + result = _check(cli) + + assert result.exit_code == 1 + assert _lines(result, "unused-category", "500") != [] + + +def test_two_pods_run_the_three_connects_of_the_network_policy( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + result = _check(cli) + + runs = [argv for argv in kubectl_calls(cluster) if verb(argv) == "run"] + labels = {_pod_of(argv): flag(argv, "--labels") for argv in runs} + labeled = next( + pod for pod, label in labels.items() if label == "app.kubernetes.io/name=searxng" + ) + unlabeled = next(pod for pod in labels if pod != labeled) + client_labels = ("app.kubernetes.io/name=searxng", "app.kubernetes.io/name=roadmap-service") + assert ( + result.exit_code, + len(runs), + {namespace_of(argv) for argv in runs}, + labels[unlabeled] in client_labels, + _exec_targets(cluster, unlabeled), + _exec_targets(cluster, labeled), + verb(kubectl_calls(cluster)[-1]), + sorted(verb(a) for a in kubectl_calls(cluster) if verb(a) == "delete"), + ) == ( + 0, + 2, + {"devops"}, + False, + [(_SEARXNG_IP, "8080")], + [(_METADATA, "80"), (_ROADMAP_IP, "8000")], + "delete", + ["delete", "delete"], + ) + + +def test_the_report_names_each_pod_and_target_by_a_role_and_shows_three_blocked_rows( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + result = _check(cli) + + rows = _lines(result, "blocked") + assert (result.exit_code, len(rows)) == (0, 3) + assert [ + ("" in rows[0]) and ("" in rows[0]), + all("" in row for row in rows[1:]), + "" in " ".join(rows), + "" in " ".join(rows), + ] == [True, True, True, True] + + +def test_the_output_holds_no_address_and_no_host_name( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + result = _check(cli) + + leaks = [ + text + for text in (_METADATA, _SEARXNG_IP, _ROADMAP_IP, ".svc", ".cluster.local") + if text in result.output + ] + assert (result.exit_code, leaks) == (0, []) + + +def test_an_unlabeled_pod_that_connects_to_the_service_fails_the_check_and_the_pods_are_deleted( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + cluster.answer( + lambda call: _is_exec(call) and _SEARXNG_IP in call.argv, ProcessAnswer(returncode=0) + ) + + result = _check(cli) + + reached = _lines(result, "reached") + assert ( + result.exit_code, + [("" in r and "" in r) for r in reached], + verb(kubectl_calls(cluster)[-1]), + ) == (1, [True], "delete") + + +def test_a_searxng_pod_that_reaches_the_metadata_address_fails_the_check( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + cluster.answer( + lambda call: _is_exec(call) and _METADATA in call.argv, ProcessAnswer(returncode=0) + ) + + result = _check(cli) + + reached = _lines(result, "reached") + assert ( + result.exit_code, + [("" in r and "" in r) for r in reached], + ) == (1, [True]) + + +def test_a_searxng_pod_that_reaches_a_private_cluster_ip_fails_the_check( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + cluster.answer( + lambda call: _is_exec(call) and _ROADMAP_IP in call.argv, ProcessAnswer(returncode=0) + ) + + result = _check(cli) + + reached = _lines(result, "reached") + assert ( + result.exit_code, + [("" in r and "" in r) for r in reached], + ) == (1, [True]) + + +def test_a_forward_that_ends_at_once_stops_the_command_before_it_sends_a_request( + cli: Callable[..., Any], process_fake: ProcessFake, web: WebFake, service: _Service +) -> None: + result = _check(cli) + + assert (result.exit_code, web.recorded) == (1, []) + assert "run" not in [verb(argv) for argv in kubectl_calls(process_fake)] + + +def test_a_service_that_does_not_answer_health_stops_the_command_and_stops_the_forward( + cli: Callable[..., Any], cluster: ProcessFake, web: WebFake +) -> None: + web.refuse(f"{_BASE}/healthz") + + result = _check(cli) + + assert (result.exit_code, _searches(web), cluster.terminated_pids) == ( + 1, + [], + [(-1, signal.SIGTERM)], + ) + + +def test_the_context_option_reaches_every_kubectl_call( + cli: Callable[..., Any], cluster: ProcessFake +) -> None: + result = _check(cli, "--context", "test-ctx") + + assert (result.exit_code, {context_of(argv) for argv in kubectl_calls(cluster)}) == ( + 0, + {"test-ctx"}, + ) + + +def test_a_dry_run_lists_every_call_and_makes_none( + cli: Callable[..., Any], process_fake: ProcessFake, web: WebFake +) -> None: + result = _check(cli, "--dry-run") + + lines = result.output.splitlines() + missing = [row for row in _ROWS if not any(row in line for line in lines)] + leaks = [text for text in (_METADATA, ".svc") if text in result.output] + assert ( + result.exit_code, + missing, + leaks, + "kubectl port-forward" in result.output, + "kubectl run" in result.output, + process_fake.calls, + web.recorded, + ) == (0, [], [], True, True, [], []) + + +def test_the_command_uses_the_forward_of_the_shared_probe_module() -> None: + assert command.forward_service is service_probe.forward_service diff --git a/tests/fixtures/searxng/search_reply.json b/tests/fixtures/searxng/search_reply.json index 105ca9aeb..a66808b13 100644 --- a/tests/fixtures/searxng/search_reply.json +++ b/tests/fixtures/searxng/search_reply.json @@ -1,53 +1,107 @@ { - "query": "pydantic ai agents", - "number_of_results": 0, + "query": "python", "results": [ { - "url": "https://example.com/pydantic-ai", - "title": "Pydantic AI", - "content": "Agents for Python.", - "engine": "duckduckgo", - "parsed_url": ["https", "example.com", "/pydantic-ai", "", "", ""], + "url": "https://developer.mozilla.org/en-US/docs/Glossary/Python", + "title": "Python", + "content": "Python is a high-level general-purpose programming language. It uses a multi-paradigm approach, meaning it supports procedural, object-oriented, and some functional programming constructs.", + "engine": "mdn", "template": "default.html", - "engines": ["duckduckgo", "wikipedia"], - "positions": [1, 2], - "score": 9.0, - "category": "general", - "thumbnail": null, - "publishedDate": null + "parsed_url": [ + "https", + "developer.mozilla.org", + "/en-US/docs/Glossary/Python", + "", + "", + "" + ], + "img_src": "", + "thumbnail": "", + "priority": "", + "engines": [ + "mdn" + ], + "positions": [ + 1 + ], + "score": 1.0, + "category": "it", + "publishedDate": null, + "iframe_src": null }, { - "url": "https://example.com/ruff", - "title": "Ruff", - "content": "A linter.", - "engine": "github", - "parsed_url": ["https", "example.com", "/ruff", "", "", ""], + "url": "https://developer.mozilla.org/en-US/docs/Learn_web_development/Extensions/Server-side/Django", + "title": "Django Web Framework (Python)", + "content": "Django is an extremely popular and fully featured server-side web framework, written in Python. This module shows you why Django is one of the most popular web server frameworks, how to set up a development environment, and how to start using it to create your own web applications.", + "engine": "mdn", "template": "default.html", - "engines": ["github"], - "positions": [1], - "score": 3.0, + "parsed_url": [ + "https", + "developer.mozilla.org", + "/en-US/docs/Learn_web_development/Extensions/Server-side/Django", + "", + "", + "" + ], + "img_src": "", + "thumbnail": "", + "priority": "", + "engines": [ + "mdn" + ], + "positions": [ + 2 + ], + "score": 0.5, "category": "it", - "thumbnail": null, - "publishedDate": null + "publishedDate": null, + "iframe_src": null }, { - "url": "https://example.com/mypy", - "title": "Mypy", - "content": "A type checker.", - "engine": "pypi", - "parsed_url": ["https", "example.com", "/mypy", "", "", ""], + "url": "https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model", + "title": "Document Object Model (DOM)", + "content": "The Document Object Model (DOM) connects web pages to scripts or programming languages by representing the structure of a document—such as the HTML representing a web page—in memory. Usually it refers to JavaScript, even though modeling HTML, SVG, or XML documents as objects are not part of the core JavaScript language.", + "engine": "mdn", "template": "default.html", - "engines": ["pypi"], - "positions": [1], - "score": 1.0, + "parsed_url": [ + "https", + "developer.mozilla.org", + "/en-US/docs/Web/API/Document_Object_Model", + "", + "", + "" + ], + "img_src": "", + "thumbnail": "", + "priority": "", + "engines": [ + "mdn" + ], + "positions": [ + 3 + ], + "score": 0.3333333333333333, "category": "it", - "thumbnail": null, - "publishedDate": null + "publishedDate": null, + "iframe_src": null } ], "answers": [], "corrections": [], "infoboxes": [], "suggestions": [], - "unresponsive_engines": [] + "unresponsive_engines": [ + [ + "askubuntu", + "too many requests" + ], + [ + "stackoverflow", + "too many requests" + ], + [ + "superuser", + "too many requests" + ] + ] } diff --git a/tests/fixtures/searxng/search_reply_image.txt b/tests/fixtures/searxng/search_reply_image.txt new file mode 100644 index 000000000..45e3eb3fc --- /dev/null +++ b/tests/fixtures/searxng/search_reply_image.txt @@ -0,0 +1 @@ +docker.io/searxng/searxng:2026.10.23-629368e40@sha256:aecad882e86eeb022b76351c8dce4eb48d0cbb16a5f1691f77ea32c96d317502 diff --git a/tests/gate/k8s_checks/devops_runtime.py b/tests/gate/k8s_checks/devops_runtime.py index a35e29859..2d86cb484 100644 --- a/tests/gate/k8s_checks/devops_runtime.py +++ b/tests/gate/k8s_checks/devops_runtime.py @@ -9,6 +9,7 @@ import yaml from devops_cli.k8s.crawl4ai_check import CRAWL4AI_IMAGE +from devops_cli.k8s.searxng_check import SEARXNG_IMAGE from tests.support.paths import REPO_ROOT K8S_DIR = REPO_ROOT / "k8s" @@ -66,10 +67,16 @@ def check_both_workloads_run_the_digest_image_updater_sets_and_pull_it_only_when "image, then set CRAWL4AI_IMAGE in src/devops_cli/k8s/crawl4ai_check.py." ) containers.discard(crawl4ai) - # Deployment `searxng` pins a third-party image by tag and digest (#3924). + # Deployment `searxng` pins a third-party image by tag and digest (#3924). `devops k8s + # searxng-check` names the image it was written for (#3929), so a bump fails here until a + # person runs the command against the new image and sets the constant. searxng = next(c for c in containers if c[1] == "searxng") assert searxng[0] == "Deployment" - assert searxng[2].startswith("docker.io/searxng/searxng:") + assert searxng[2] == SEARXNG_IMAGE, ( + f"k8s/devops/searxng/deployment.yaml pins {searxng[2]}, but `devops k8s searxng-check` " + f"was written for {SEARXNG_IMAGE}. Run `uv run devops k8s searxng-check` against the new " + "image, then set SEARXNG_IMAGE in src/devops_cli/k8s/searxng_check.py." + ) containers.discard(searxng) assert containers == { ("CronJob", "devops-cli", f"{service}:latest@{digest}", "IfNotPresent"), diff --git a/tests/invariants/test_k8s_searxng.py b/tests/invariants/test_k8s_searxng.py index 959a6d932..425dd9962 100644 --- a/tests/invariants/test_k8s_searxng.py +++ b/tests/invariants/test_k8s_searxng.py @@ -13,6 +13,7 @@ from devops_cli.commands.k8s.stack_lifecycle import MANIFESTS_BY_STACK from devops_cli.k8s.cluster_secrets import CLUSTER_SECRETS from devops_cli.k8s.image_pins import names_latest +from devops_cli.k8s.searxng_check import OFF_ENGINES from tests.support.k8s_tree import K8S, manifests, pod_spec from tests.support.paths import REPO_ROOT @@ -29,10 +30,6 @@ IMAGE_PATTERN = re.compile( r"^docker\.io/searxng/searxng:(?P\d{4}\.\d+\.\d+-[0-9a-f]{7,9})@sha256:[0-9a-f]{64}$" ) -# The scraping engines that the upstream sites suspended in October 2026 (SearXNG issues 6866 and -# 6819), and the keyed Brave engine, which the owner dropped on 2026-11-07. A name here is a name -# of the default settings.yml, and a prefix: `brave` also covers `brave.news`. -FORBIDDEN_ENGINES = ("brave", "braveapi", "google", "bing", "startpage", "mojeek", "qwant") # The pod root, not a path of this host, and the IPv4 wildcard address that the pod binds. _POD_ROOT = PurePosixPath("/") BIND_ALL_IPV4 = str(ipaddress.IPv4Address(0)) @@ -320,7 +317,7 @@ def test_the_settings_keep_none_of_the_suspended_or_keyed_engines() -> None: named = {name.split(".")[0].split(" ")[0] for name in kept} declared = [engine["name"] for engine in settings.get("engines", [])] - assert (sorted(named & set(FORBIDDEN_ENGINES)), declared) == ([], []) + assert (sorted(named & set(OFF_ENGINES)), declared) == ([], []) def test_every_kept_engine_exists_in_the_default_settings_of_the_pinned_image() -> None: @@ -334,7 +331,7 @@ def test_the_names_file_holds_the_forbidden_engines_so_the_check_above_means_som engines would pass for any list if the names file were empty.""" names = _default_engine_names() - assert sorted(name for name in FORBIDDEN_ENGINES if name not in names) == [] + assert sorted(name for name in OFF_ENGINES if name not in names) == [] def test_every_file_of_the_service_is_in_both_kustomizations_and_the_deploy_stack() -> None: diff --git a/tests/k8s/test_searxng_check.py b/tests/k8s/test_searxng_check.py new file mode 100644 index 000000000..e4617c616 --- /dev/null +++ b/tests/k8s/test_searxng_check.py @@ -0,0 +1,172 @@ +"""What a reply of the SearXNG service must hold for `devops k8s searxng-check` to pass it (#3929). + +The pinned image answers `GET /search?format=json` with a JSON object of seven keys (`answers`, +`corrections`, `infoboxes`, `query`, `results`, `suggestions` and `unresponsive_engines`). The +format `json` must be in `search.formats`, or the service answers HTTP 403 (`searx/webapp.py`). +Six scraping and keyed engines must stay off. `reply_problems` returns one text for each point +that fails, and an empty list is a pass. +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +import pytest + +from devops_cli.k8s import searxng_check, service_probe +from devops_cli.k8s.searxng_check import OFF_ENGINES, REPLY_KEYS, SEARXNG_IMAGE, reply_problems +from tests.support.paths import FIXTURES + +_KEYS = ( + "answers", + "corrections", + "infoboxes", + "query", + "results", + "suggestions", + "unresponsive_engines", +) +_HIT = {"url": "https://example.com/a", "title": "A", "content": "B", "engine": "duckduckgo"} + + +def _reply(**changes: Any) -> dict[str, Any]: + reply: dict[str, Any] = { + "query": "test", + "results": [_HIT], + "answers": [], + "corrections": [], + "infoboxes": [], + "suggestions": [], + "unresponsive_engines": [], + } + reply.update(changes) + return reply + + +def _body(reply: object) -> bytes: + return json.dumps(reply).encode() + + +def test_the_reply_keys_are_the_seven_of_the_search_api() -> None: + assert sorted(REPLY_KEYS) == sorted(_KEYS) + + +def test_the_engines_that_stay_off_are_the_keyed_and_suspended_ones() -> None: + assert sorted(OFF_ENGINES) == sorted( + ["brave", "braveapi", "google", "bing", "startpage", "mojeek", "qwant"] + ) + + +def test_a_full_reply_with_a_hit_has_no_problem() -> None: + assert reply_problems(200, _body(_reply())) == [] + + +@pytest.mark.parametrize("key", _KEYS) +def test_a_reply_without_a_key_has_one_problem_that_names_the_key(key: str) -> None: + reply = _reply() + del reply[key] + + problems = reply_problems(200, _body(reply)) + + assert [key in problem for problem in problems] == [True] + + +def test_unresponsive_engines_that_is_a_string_is_a_problem_that_names_it() -> None: + problems = reply_problems(200, _body(_reply(unresponsive_engines="duckduckgo: timeout"))) + + assert [("unresponsive_engines" in problem) for problem in problems] == [True] + + +def test_http_403_is_a_problem_that_names_the_search_formats_setting() -> None: + problems = reply_problems(403, b"Forbidden") + + assert [("search.formats" in problem) for problem in problems] == [True] + + +@pytest.mark.parametrize("status", [400, 500, 502]) +def test_another_status_is_a_problem_that_names_the_status(status: int) -> None: + problems = reply_problems(status, _body(_reply())) + + assert [str(status) in problem for problem in problems] == [True] + + +@pytest.mark.parametrize("body", [b"not json", b"[]", b'"text"', b""]) +def test_a_body_that_is_no_json_object_is_one_problem(body: bytes) -> None: + assert len(reply_problems(200, body)) == 1 + + +def test_a_reply_without_a_hit_is_a_problem_unless_the_caller_expects_none() -> None: + body = _body(_reply(results=[])) + + assert ( + [("hit" in problem) for problem in reply_problems(200, body)], + reply_problems(200, body, expect_hits=False), + ) == ([True], []) + + +@pytest.mark.parametrize( + "engine", ["google", "bing", "brave", "brave.news", "braveapi", "startpage", "mojeek", "qwant"] +) +def test_a_hit_of_an_engine_that_stays_off_is_a_problem_that_names_the_engine(engine: str) -> None: + body = _body(_reply(results=[_HIT, {**_HIT, "engine": engine}])) + + problems = reply_problems(200, body) + + assert [engine in problem for problem in problems] == [True] + + +def test_an_engine_that_stays_off_and_is_listed_as_unresponsive_is_a_problem() -> None: + body = _body(_reply(unresponsive_engines=[["bing", "CAPTCHA"], ["duckduckgo", "timeout"]])) + + problems = reply_problems(200, body) + + assert [("bing" in problem) for problem in problems] == [True] + + +def test_a_kept_engine_that_is_unresponsive_is_no_problem() -> None: + body = _body(_reply(unresponsive_engines=[["duckduckgo", "timeout"]])) + + assert reply_problems(200, body) == [] + + +def test_an_engine_that_the_caller_names_is_a_problem_in_the_hits() -> None: + body = _body(_reply(results=[{**_HIT, "engine": "pypi"}])) + + assert ( + reply_problems(200, body), + [("pypi" in problem) for problem in reply_problems(200, body, also_off=("pypi",))], + ) == ([], [True]) + + +def test_the_image_names_a_release_tag_and_its_digest() -> None: + """The gate compares this value with the image of the Deployment, so a bump of the digest + fails until a person runs the command against the new image.""" + assert re.fullmatch( + r"docker\.io/searxng/searxng:\d{4}\.\d+\.\d+-[0-9a-f]{7,9}@sha256:[0-9a-f]{64}", + SEARXNG_IMAGE, + ) + + +def test_the_check_uses_the_probe_code_that_crawl4ai_check_uses() -> None: + assert ( + searxng_check.probe_from_pod is service_probe.probe_from_pod, + searxng_check.plan_probe_from_pod is service_probe.plan_probe_from_pod, + searxng_check.service_cluster_ip is service_probe.service_cluster_ip, + ) == (True, True, True) + + +def test_the_saved_reply_is_a_live_reply_of_the_pinned_image_that_the_check_passes() -> None: + """`search_reply.json` is a reply that the pinned image gave to `devops k8s searxng-check` + (the first hits of the `it` row), and the file next to it names that image. A bump of the + digest fails here until a person saves a new reply.""" + folder = FIXTURES / "searxng" + saved = (folder / "search_reply.json").read_bytes() + image = (folder / "search_reply_image.txt").read_text(encoding="utf-8").strip() + + assert (image, sorted(json.loads(saved)), reply_problems(200, saved)) == ( + SEARXNG_IMAGE, + sorted(REPLY_KEYS), + [], + ) diff --git a/tests/support/kubectl_calls.py b/tests/support/kubectl_calls.py index 4f1a48e5f..571705f49 100644 --- a/tests/support/kubectl_calls.py +++ b/tests/support/kubectl_calls.py @@ -8,6 +8,16 @@ from tests.support.fakes.process import ProcessFake +class RunningForward: + """The `kubectl port-forward` child while it runs. Its pid is one that no process holds, so a + stop of the forward signals nothing real.""" + + pid = -1 + + def poll(self) -> int | None: + return None + + def without_context(argv: Sequence[str]) -> list[str]: """`argv` without its `--context` flag and the value of the flag.""" kept: list[str] = []