From 37351cedcfdb7ad70d58434f831ecf3234a88528 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 7 Aug 2026 09:06:29 +0000 Subject: [PATCH 1/2] test: expand detection corpus to per-family floors; add gitleaks parity harness Five new shape families under tests/corpus/tree: xml (probing the markup allowlist), k8s (including base64 data: values), urls (all-vocabulary passwords), noise-code (amalgamations, generated bindings) and noise-artifacts (lockfiles, minified, certificates). 551 manifest rows total, 258 new. The harness now enforces a recall floor per family, set to the value measured at introduction, with precision a global 1.0 on classified rows. Known gaps start low by design: urls 0.4783 (#44), xml 0.5926 (#51), k8s 0.8333 (#53). No rule file changes. scripts/parity_gitleaks.py diffs siloscan against a checksum-pinned gitleaks 8.30.1 in directory mode and buckets every delta. --- crates/siloscan-core/tests/corpus/README.md | 55 +++- .../siloscan-core/tests/corpus/manifest.tsv | 258 ++++++++++++++++ .../tests/corpus/tree/k8s/configmap.yaml | 13 + .../tests/corpus/tree/k8s/helm/values.yaml | 33 ++ .../tests/corpus/tree/k8s/kustomization.yaml | 26 ++ .../corpus/tree/k8s/postgres-secret.yaml | 12 + .../tests/corpus/tree/k8s/redis-secret.yaml | 9 + .../corpus/tree/k8s/registry-credentials.yaml | 8 + .../tests/corpus/tree/k8s/session-secret.yaml | 12 + .../corpus/tree/k8s/storefront-stack.yaml | 54 ++++ .../corpus/tree/noise-artifacts/Cargo.lock | 30 ++ .../corpus/tree/noise-artifacts/ca-bundle.pem | 21 ++ .../corpus/tree/noise-artifacts/fonts.css | 14 + .../tests/corpus/tree/noise-artifacts/go.sum | 10 + .../tree/noise-artifacts/package-lock.json | 41 +++ .../tree/noise-artifacts/release.sha256sums | 6 + .../corpus/tree/noise-artifacts/vendor.min.js | 2 + .../tree/noise-artifacts/vendor.min.js.map | 8 + .../corpus/tree/noise-artifacts/yarn.lock | 27 ++ .../tree/noise-code/proto/authpb/auth.pb.go | 124 ++++++++ .../tree/noise-code/proto/telemetry_pb2.py | 24 ++ .../tree/noise-code/src/generated/expanded.rs | 105 +++++++ .../tree/noise-code/src/sys/linux_ppp.rs | 42 +++ .../corpus/tree/noise-code/src/sys/sodium.rs | 38 +++ .../noise-code/src/sys/windows_security.rs | 46 +++ .../tools/codegen/src/bake_tables.rs | 51 ++++ .../tree/noise-code/vendor/litevfs/configure | 56 ++++ .../tree/noise-code/vendor/litevfs/litevfs.c | 193 ++++++++++++ .../siloscan-core/tests/corpus/tree/urls/.env | 21 ++ .../tests/corpus/tree/urls/SETUP.md | 28 ++ .../tests/corpus/tree/urls/backup.sh | 27 ++ .../tests/corpus/tree/urls/celeryconfig.py | 21 ++ .../tests/corpus/tree/urls/database.yml | 27 ++ .../tests/corpus/tree/urls/settings.ini | 22 ++ .../tests/corpus/tree/xml/App.config | 25 ++ .../corpus/tree/xml/Directory.Build.props | 12 + .../tests/corpus/tree/xml/NuGet.Config | 21 ++ .../tests/corpus/tree/xml/Payments.Api.csproj | 18 ++ .../tests/corpus/tree/xml/ivysettings.xml | 13 + .../tests/corpus/tree/xml/settings.xml | 38 +++ .../tests/corpus/tree/xml/web.config | 28 ++ .../siloscan-core/tests/detection_corpus.rs | 232 +++++++++++++- scripts/parity_gitleaks.py | 289 ++++++++++++++++++ 43 files changed, 2121 insertions(+), 19 deletions(-) create mode 100644 crates/siloscan-core/tests/corpus/tree/k8s/configmap.yaml create mode 100644 crates/siloscan-core/tests/corpus/tree/k8s/helm/values.yaml create mode 100644 crates/siloscan-core/tests/corpus/tree/k8s/kustomization.yaml create mode 100644 crates/siloscan-core/tests/corpus/tree/k8s/postgres-secret.yaml create mode 100644 crates/siloscan-core/tests/corpus/tree/k8s/redis-secret.yaml create mode 100644 crates/siloscan-core/tests/corpus/tree/k8s/registry-credentials.yaml create mode 100644 crates/siloscan-core/tests/corpus/tree/k8s/session-secret.yaml create mode 100644 crates/siloscan-core/tests/corpus/tree/k8s/storefront-stack.yaml create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/Cargo.lock create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/ca-bundle.pem create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/fonts.css create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/go.sum create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/package-lock.json create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/release.sha256sums create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/vendor.min.js create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/vendor.min.js.map create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-artifacts/yarn.lock create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/proto/authpb/auth.pb.go create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/proto/telemetry_pb2.py create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/src/generated/expanded.rs create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/linux_ppp.rs create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/sodium.rs create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/windows_security.rs create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/tools/codegen/src/bake_tables.rs create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/vendor/litevfs/configure create mode 100644 crates/siloscan-core/tests/corpus/tree/noise-code/vendor/litevfs/litevfs.c create mode 100644 crates/siloscan-core/tests/corpus/tree/urls/.env create mode 100644 crates/siloscan-core/tests/corpus/tree/urls/SETUP.md create mode 100644 crates/siloscan-core/tests/corpus/tree/urls/backup.sh create mode 100644 crates/siloscan-core/tests/corpus/tree/urls/celeryconfig.py create mode 100644 crates/siloscan-core/tests/corpus/tree/urls/database.yml create mode 100644 crates/siloscan-core/tests/corpus/tree/urls/settings.ini create mode 100644 crates/siloscan-core/tests/corpus/tree/xml/App.config create mode 100644 crates/siloscan-core/tests/corpus/tree/xml/Directory.Build.props create mode 100644 crates/siloscan-core/tests/corpus/tree/xml/NuGet.Config create mode 100644 crates/siloscan-core/tests/corpus/tree/xml/Payments.Api.csproj create mode 100644 crates/siloscan-core/tests/corpus/tree/xml/ivysettings.xml create mode 100644 crates/siloscan-core/tests/corpus/tree/xml/settings.xml create mode 100644 crates/siloscan-core/tests/corpus/tree/xml/web.config create mode 100755 scripts/parity_gitleaks.py diff --git a/crates/siloscan-core/tests/corpus/README.md b/crates/siloscan-core/tests/corpus/README.md index fd3c1e7..1a827a4 100644 --- a/crates/siloscan-core/tests/corpus/README.md +++ b/crates/siloscan-core/tests/corpus/README.md @@ -15,10 +15,32 @@ survive a measurement. That is the whole argument for this directory. ## Layout - tree/ the corpus itself, one directory of realistic files + tree/ the corpus itself, realistic files grouped into families manifest.tsv the expectation table, one row per measured line README.md this file +The corpus is organized into **families**: a family is the first directory +under `tree/`, and files sitting directly in `tree/` form the family `core`. +Each family measures one territory, and each holds its own recall floor +(see "The floors"): + + core the original mixed corpus: .env, compose, properties, + JSON, Dockerfile, source files in six languages + keys private-key material in four formats, plus public + material that must stay quiet + k8s Kubernetes and Helm: Secret data:/stringData:, chart + values, kustomize generators, multi-document manifests + urls credentialed URLs across .env, shell, Python, YAML, INI + and Markdown, centered on the ticket #44 placeholder- + vocabulary gap + xml .NET and JVM XML configuration: App.config, web.config, + MSBuild, NuGet.Config, Maven settings.xml, Ivy - centered + on the ticket #51 =[^=] markup gap + noise-artifacts pure noise, no positives: lockfiles, checksums, PEM + certificates, inlined fonts, minified bundles, sourcemaps + noise-code pure noise, no positives: generated and vendored source + with credential-shaped identifiers beside numeric values + `crates/siloscan-core/Cargo.toml` carries `exclude = ["tests/corpus/**"]`, so none of this is packaged into the published crate. It is test material and belongs in the repository, not in every consumer's vendor directory. Verify @@ -104,6 +126,11 @@ underscores, and anything else in braces is left verbatim. expectation, justification. Rows are sorted by path then line. 3. Run the harness. `the_corpus_and_its_manifest_agree` fails if the row points at a line that does not exist or repeats one already claimed. +4. A new family - a new first-level directory under `tree/` - with any + positive rows needs an entry in `RECALL_FLOORS`, set to the value the + harness measures, not the value anyone hopes for. The recall test panics on + a family with positives and no floor, and fails on a floor entry naming a + family without positives. Expectations are one of: @@ -127,14 +154,28 @@ argue with later, which is how the current rules were arrived at. cargo test -p siloscan-core --test detection_corpus -`detection_recall_meets_its_floor` and `detection_precision_meets_its_floor` -print the full list of misses, spurious hits and findings on lines the manifest -does not classify, then assert: +`detection_recall_meets_its_floor_per_family` and +`detection_precision_meets_its_floor` print the per-family recall table and +the full list of misses, spurious hits and findings on lines the manifest does +not classify, then assert: -- **recall** - positives reported / positives - at or above `RECALL_FLOOR`. -- **precision proxy** - negatives left alone / negatives - at or above +- **recall, per family** - positives reported / positives, held separately for + each family against its entry in `RECALL_FLOORS`. One global number would + let a regression in a strong family hide behind a fix in a weak one. +- **precision proxy** - negatives left alone / negatives, global, at or above `PRECISION_FLOOR`, which is 1.0. Every negative is justified individually, so - one spurious hit is one defect. + one spurious hit is one defect wherever it lands. + +Every floor is the MEASURED value at the time its family landed - descriptive, +not aspirational. Families built to hold a known gap start low on purpose: +`urls` contracts the ticket #44 placeholder-vocabulary URLs, `xml` the ticket +#51 `=[^=]` markup allowlist plus names (`AccountKey`, `machineKey`, NuGet +`apikeys`, Ivy `name=`/`value=`) no rule lists, and `k8s` the `data:` decoding +that arrives with tickets #52/#53. A floor forbids regression below what was +measured; the ticket that closes a gap raises the floor in the same commit +that moves the number. The pure-noise families carry no positives and so no +recall floor - they exist to hold the precision line, and `RECALL_FLOORS` must +not name them. ### What the floors do not measure diff --git a/crates/siloscan-core/tests/corpus/manifest.tsv b/crates/siloscan-core/tests/corpus/manifest.tsv index 2ad1ddb..28313a5 100644 --- a/crates/siloscan-core/tests/corpus/manifest.tsv +++ b/crates/siloscan-core/tests/corpus/manifest.tsv @@ -220,6 +220,48 @@ k8s-deployment.yaml 40 NONE a Helm template interpolation k8s-deployment.yaml 42 NONE a placeholder value k8s-deployment.yaml 44 NONE a reference to an image pull secret k8s-deployment.yaml 45 NONE a config checksum annotation +k8s/configmap.yaml 7 NONE JVM flag soup in a ConfigMap data value, punctuation-dense but not a credential +k8s/configmap.yaml 8 NONE 40 hex characters under a build key is a git commit sha, a known false-positive shape +k8s/configmap.yaml 9 NONE a UUID deployment identifier in ConfigMap data +k8s/configmap.yaml 10 NONE base64 of a public certificate header under a ConfigMap key with no secret identifier - the classic base64-in-data false positive +k8s/configmap.yaml 11 NONE a session cookie name, not a session secret +k8s/configmap.yaml 12 NONE a numeric TTL +k8s/configmap.yaml 13 NONE a sha256 checksum under a checksum key, 64 hex that must stay quiet +k8s/helm/values.yaml 12 ANY 20-character plaintext chart default password in a Helm values file, a generic assignment shape +k8s/helm/values.yaml 13 ANY 24-character plaintext replication password in a Helm values file +k8s/helm/values.yaml 14 NONE an empty existingSecret override slot beside a secret-like identifier, the correct chart pattern +k8s/helm/values.yaml 18 ANY 43-character base64url API key in a Helm values file, the shape a 256-bit value encodes to +k8s/helm/values.yaml 19 secrets.github-pat a GitHub personal access token committed in a Helm values file +k8s/helm/values.yaml 24 NONE a Helm template interpolation in the password position, resolved at render time +k8s/helm/values.yaml 28 NONE a Helm template interpolation with a default filter in the password position, still no value +k8s/helm/values.yaml 32 NONE a cpu resource limit +k8s/helm/values.yaml 33 NONE a memory resource limit +k8s/kustomization.yaml 11 NONE an image pinned by sha256 digest, 64 hex that must stay quiet +k8s/kustomization.yaml 17 ANY 24-character password in a kustomize secretGenerator literal, a generic assignment shape people commit instead of using files +k8s/kustomization.yaml 18 ANY exactly 40 base64 characters in a secretGenerator literal, the length the 1.4.1 allowlist dropped +k8s/kustomization.yaml 20 NONE a secretGenerator file reference is a path, not a value - the correct kustomize pattern +k8s/kustomization.yaml 25 NONE a configMapGenerator log-level literal +k8s/kustomization.yaml 26 NONE a configMapGenerator feature-flag list, hyphenated identifier soup +k8s/postgres-secret.yaml 10 secrets.kubernetes-secret-yaml|secrets.generic-secret-assignment base64 of a 24-character password under data: beside a password key; the encoded text is itself a high-entropy value the generic assignment may own today, and the decoding reporter is secrets.kubernetes-secret-yaml, arriving via ticket 53 on the ticket 52 decode gate - either is an acceptable owner +k8s/postgres-secret.yaml 11 secrets.kubernetes-secret-yaml|secrets.generic-secret-assignment base64 of a 20-character replication password under data:; same encoded-value shape - the expected end-state reporter is secrets.kubernetes-secret-yaml (ticket 53), the generic assignment is the acceptable interim owner of the encoded text +k8s/postgres-secret.yaml 12 secrets.kubernetes-secret-yaml base64 of a credentialed postgres URL under a database-url key; no secret-like identifier, so no shipped rule can reach it without decoding - known miss until secrets.kubernetes-secret-yaml lands via tickets 52 and 53, which is the expected reporter +k8s/redis-secret.yaml 8 secrets.kubernetes-secret-yaml|secrets.generic-secret-assignment base64 of a 32-character redis password under data:; expected end-state reporter is secrets.kubernetes-secret-yaml (ticket 53), the generic assignment may own the encoded text until then +k8s/redis-secret.yaml 9 secrets.kubernetes-secret-yaml|secrets.generic-secret-assignment base64 of a 16-character requirepass value under data:; expected end-state reporter is secrets.kubernetes-secret-yaml (ticket 53), the generic assignment is the acceptable interim owner +k8s/registry-credentials.yaml 8 secrets.kubernetes-secret-yaml a .dockerconfigjson blob: base64 of JSON whose auth field is base64 of user:password - two encoding layers, the exact shape gating ticket 52; the key carries no secret identifier so nothing ships that can report it - known miss, expected reporter is secrets.kubernetes-secret-yaml via ticket 53 +k8s/session-secret.yaml 10 ANY 64 hex characters of session signing secret in stringData plaintext, reachable by the generic rules today; ownership may move to secrets.kubernetes-secret-yaml when ticket 53 lands +k8s/session-secret.yaml 11 ANY 43-character base64url CSRF token in stringData plaintext +k8s/session-secret.yaml 12 ANY a 40-hex OAuth client secret in stringData plaintext, the GitHub OAuth client secret length +k8s/storefront-stack.yaml 18 NONE a container image pinned by sha256 digest in a Deployment, 64 hex that must stay quiet +k8s/storefront-stack.yaml 23 NONE a cpu resource request +k8s/storefront-stack.yaml 24 NONE a memory resource request +k8s/storefront-stack.yaml 26 NONE a quoted cpu resource limit +k8s/storefront-stack.yaml 27 NONE a memory resource limit +k8s/storefront-stack.yaml 29 NONE an envFrom secretRef is the correct pattern and carries no value +k8s/storefront-stack.yaml 30 NONE the name of a referenced Secret, a low-entropy identifier +k8s/storefront-stack.yaml 35 NONE the Secret's own metadata name in the middle document, not a value +k8s/storefront-stack.yaml 39 secrets.kubernetes-secret-yaml|secrets.generic-secret-assignment base64 of a 20-character SMTP password under data: in a Secret sandwiched between a Deployment and a Service; expected end-state reporter is secrets.kubernetes-secret-yaml (ticket 53), the generic assignment may own the encoded text until then +k8s/storefront-stack.yaml 40 secrets.kubernetes-secret-yaml|secrets.generic-secret-assignment base64 of a 32-character JWT signing key under data: in a multi-document manifest; expected end-state reporter is secrets.kubernetes-secret-yaml (ticket 53), the generic assignment is the acceptable interim owner +k8s/storefront-stack.yaml 42 ANY 32-character base64url webhook secret in stringData plaintext in the same multi-document Secret, reachable by the generic rules today keys/deploy_key 1 secrets.private-key an OpenSSH private key keys/ec-private.pem 1 secrets.private-key an SEC1 elliptic curve private key keys/pgp-private.asc 1 secrets.private-key a PGP private key block @@ -259,6 +301,119 @@ main.tf 28 NONE a placeholder password main.tf 29 NONE a service endpoint main.tf 30 NONE a module source main.tf 31 NONE a resource name +noise-artifacts/Cargo.lock 8 NONE a crates.io registry source URL with no userinfo +noise-artifacts/Cargo.lock 9 NONE a Cargo.lock 64-hex package checksum, a hash of public bytes +noise-artifacts/Cargo.lock 14 NONE a crates.io registry source URL with no userinfo +noise-artifacts/Cargo.lock 15 NONE a Cargo.lock 64-hex package checksum, a hash of public bytes +noise-artifacts/Cargo.lock 20 NONE a crates.io registry source URL with no userinfo +noise-artifacts/Cargo.lock 21 NONE a Cargo.lock 64-hex package checksum, a hash of public bytes +noise-artifacts/Cargo.lock 29 NONE a crates.io registry source URL with no userinfo +noise-artifacts/Cargo.lock 30 NONE a Cargo.lock 64-hex package checksum, a hash of public bytes +noise-artifacts/ca-bundle.pem 2 NONE a certificate block is public material, not a private key +noise-artifacts/ca-bundle.pem 3 NONE the first 64-character base64 body line of a public certificate +noise-artifacts/ca-bundle.pem 20 NONE the padded closing base64 body line of a public certificate +noise-artifacts/fonts.css 7 NONE a data:font/woff2;base64 URI inlining a font, high-entropy public bytes +noise-artifacts/fonts.css 11 NONE a data:image/png;base64 URI inlining an icon +noise-artifacts/go.sum 1 NONE a go.sum h1: base64 module hash, a checksum of public source +noise-artifacts/go.sum 2 NONE a go.sum /go.mod h1: base64 hash, a checksum of public source +noise-artifacts/go.sum 3 NONE a go.sum h1: base64 module hash, a checksum of public source +noise-artifacts/go.sum 4 NONE a go.sum /go.mod h1: base64 hash, a checksum of public source +noise-artifacts/go.sum 5 NONE a go.sum h1: base64 module hash, a checksum of public source +noise-artifacts/go.sum 6 NONE a go.sum /go.mod h1: base64 hash, a checksum of public source +noise-artifacts/go.sum 7 NONE a go.sum h1: base64 module hash, a checksum of public source +noise-artifacts/go.sum 8 NONE a go.sum /go.mod h1: base64 hash, a checksum of public source +noise-artifacts/go.sum 9 NONE a go.sum h1: base64 module hash, a checksum of public source +noise-artifacts/go.sum 10 NONE a go.sum /go.mod h1: base64 hash, a checksum of public source +noise-artifacts/package-lock.json 9 NONE an npm registry tarball URL with no userinfo +noise-artifacts/package-lock.json 10 NONE a lockfileVersion 3 sha512 integrity hash, 88 base64 characters of checksum +noise-artifacts/package-lock.json 15 NONE an npm registry tarball URL with no userinfo +noise-artifacts/package-lock.json 16 NONE a lockfileVersion 3 sha512 integrity hash, 88 base64 characters of checksum +noise-artifacts/package-lock.json 24 NONE an npm registry tarball URL with no userinfo +noise-artifacts/package-lock.json 25 NONE a lockfileVersion 3 sha512 integrity hash, 88 base64 characters of checksum +noise-artifacts/package-lock.json 30 NONE an npm registry tarball URL with no userinfo +noise-artifacts/package-lock.json 31 NONE a legacy sha1 integrity hash, 28 base64 characters of checksum +noise-artifacts/package-lock.json 36 NONE an npm registry tarball URL with no userinfo +noise-artifacts/package-lock.json 37 NONE a lockfileVersion 3 sha512 integrity hash, 88 base64 characters of checksum +noise-artifacts/release.sha256sums 1 NONE a sha256sums line, 64 hex characters naming a release tarball +noise-artifacts/release.sha256sums 2 NONE a sha256sums line, 64 hex characters naming a release tarball +noise-artifacts/release.sha256sums 3 NONE a sha256sums line, 64 hex characters naming a release tarball +noise-artifacts/release.sha256sums 4 NONE a sha256sums line, 64 hex characters naming a release tarball +noise-artifacts/release.sha256sums 5 NONE a sha256sums line, 64 hex characters naming a release archive +noise-artifacts/release.sha256sums 6 NONE a sha256sums line, 64 hex characters naming a source tarball +noise-artifacts/vendor.min.js 1 NONE minified identifier soup carrying an embedded base64 image string in a bundle +noise-artifacts/vendor.min.js 2 NONE a sourceMappingURL comment, a file reference +noise-artifacts/vendor.min.js.map 6 NONE a source map mappings field, base64 VLQ soup of arbitrary length +noise-artifacts/yarn.lock 7 NONE a yarn registry URL whose fragment is a 40-hex tarball sha1 +noise-artifacts/yarn.lock 8 NONE a yarn.lock sha512 integrity hash, a checksum of public bytes +noise-artifacts/yarn.lock 14 NONE a yarn registry URL whose fragment is a 40-hex tarball sha1 +noise-artifacts/yarn.lock 15 NONE a yarn.lock sha512 integrity hash, a checksum of public bytes +noise-artifacts/yarn.lock 21 NONE a yarn registry URL whose fragment is a 40-hex tarball sha1 +noise-artifacts/yarn.lock 22 NONE a yarn.lock sha512 integrity hash, a checksum of public bytes +noise-artifacts/yarn.lock 26 NONE a yarn registry URL whose fragment is a 40-hex tarball sha1 +noise-artifacts/yarn.lock 27 NONE a yarn.lock sha512 integrity hash, a checksum of public bytes +noise-code/proto/authpb/auth.pb.go 22 NONE generated protobuf field named Password whose only value is its own wire tag, the canonical pb.go false positive +noise-code/proto/authpb/auth.pb.go 23 NONE client_secret field tag; the name=client_secret assignment inside the tag string carries the field name, not a value +noise-code/proto/authpb/auth.pb.go 34 NONE generated getter named after the password field, no value on the line +noise-code/proto/authpb/auth.pb.go 53 NONE access_token field tag; a token identifier beside a struct-tag string that spells access_token twice more +noise-code/proto/authpb/auth.pb.go 54 NONE refresh_token field tag in the same tag-string shape +noise-code/proto/authpb/auth.pb.go 75 NONE first row of a serialized file descriptor as comma-separated hex bytes, dense numeric soup +noise-code/proto/authpb/auth.pb.go 82 NONE descriptor bytes that encode the word password one 0xNN at a time +noise-code/proto/telemetry_pb2.py 14 NONE serialized descriptor byte string naming password, access_token and secret fields among escapes, the densest pb2 temptation +noise-code/proto/telemetry_pb2.py 21 NONE a short options byte string assigned to a serialized_options identifier +noise-code/src/generated/expanded.rs 25 NONE the field-name literal password in serde Serialize expansion, with no value anywhere near it +noise-code/src/generated/expanded.rs 30 NONE the field-name literal api_key in the same expansion +noise-code/src/generated/expanded.rs 71 NONE match arm mapping the string password to a field discriminant +noise-code/src/generated/expanded.rs 72 NONE match arm mapping the string api_key to a field discriminant +noise-code/src/generated/expanded.rs 86 NONE the same password arm as a byte-string literal +noise-code/src/generated/expanded.rs 87 NONE the same api_key arm as a byte-string literal +noise-code/src/generated/expanded.rs 93 NONE the generated FIELDS list spelling password and api_key side by side +noise-code/src/sys/linux_ppp.rs 10 NONE bindgen ioctl constant ending in PASS assigned a numeric request code, the exact shape behind the original mass findings +noise-code/src/sys/linux_ppp.rs 16 NONE keyring id constant; KEY-prefixed name with a small negative value +noise-code/src/sys/linux_ppp.rs 26 NONE keyctl opcode with both KEY and SECURITY in the name and the value 17 +noise-code/src/sys/linux_ppp.rs 27 NONE key permission mask; KEY name assigned a large integer that is a bitmask, not a secret +noise-code/src/sys/sodium.rs 5 NONE libsodium size constant with secret and key in one name and the value 32 +noise-code/src/sys/sodium.rs 8 NONE a very long generated identifier containing KEYBYTES, still just the number 32 +noise-code/src/sys/sodium.rs 12 NONE SECRETKEYBYTES size constant, value 64 +noise-code/src/sys/sodium.rs 22 NONE PASSWD-named limit constant assigned u32::MAX, a password identifier with a large plain integer +noise-code/src/sys/windows_security.rs 6 NONE Windows access-right bit named TOKEN with a one-digit value +noise-code/src/sys/windows_security.rs 12 NONE TOKEN_ALL_ACCESS mask; token identifier assigned a six-digit integer +noise-code/src/sys/windows_security.rs 15 NONE NCRYPT interface id with SECRET in the name and a numeric value +noise-code/src/sys/windows_security.rs 21 NONE credential-type enum value; PASSWORD-suffixed name assigned 2 +noise-code/src/sys/windows_security.rs 26 NONE user-account flag named PASSWD assigned a small bit value +noise-code/src/sys/windows_security.rs 29 NONE PASSWORD_EXPIRED flag assigned a seven-digit bit value +noise-code/src/sys/windows_security.rs 30 NONE type alias with KEY_HANDLE in the name; the right-hand side is a type +noise-code/tools/codegen/src/bake_tables.rs 7 NONE generated function named after a key schedule, carrying no value at all +noise-code/tools/codegen/src/bake_tables.rs 9 NONE Literal::u8_unsuffixed token-stream byte, the numeric soup proc-macro codegen emits by the thousand +noise-code/tools/codegen/src/bake_tables.rs 43 NONE byte-string spelling of an identifier containing secret, paired with a numeric kind id +noise-code/tools/codegen/src/bake_tables.rs 47 NONE api_token spelled as a lexer token name, followed only by its numeric kind +noise-code/vendor/litevfs/configure 18 NONE autoconf cache probe for getpass; the value is the word yes +noise-code/vendor/litevfs/configure 21 NONE cache probe with passphrase in the variable name, value no +noise-code/vendor/litevfs/configure 26 NONE library probe spelling crypt twice in one variable name +noise-code/vendor/litevfs/configure 29 NONE PASSWD-named cache variable assigned a program path +noise-code/vendor/litevfs/configure 31 NONE PASSWD_PROG cache variable whose value is the word passwd itself +noise-code/vendor/litevfs/configure 33 NONE key-size cache variable assigned the number 32 +noise-code/vendor/litevfs/configure 39 NONE output variable with KEY in the name and a quoted two-digit value +noise-code/vendor/litevfs/configure 40 NONE PASSWD-named variable holding the kpasswd port number 464 +noise-code/vendor/litevfs/configure 42 NONE MKPASSWD output variable assigned a quoted program path +noise-code/vendor/litevfs/configure 46 NONE the word passwd quoted as the value of a PASSWD variable, a password-identifier-with-short-value shape +noise-code/vendor/litevfs/configure 47 NONE SMBPASSWD variable assigned a quoted path +noise-code/vendor/litevfs/configure 53 NONE config.status sed substitution carrying PASSWD on both sides of the comma +noise-code/vendor/litevfs/litevfs.c 29 NONE amalgamation source stamp: a 64-hex checksum inside a version define, not a secret identifier +noise-code/vendor/litevfs/litevfs.c 50 NONE vendored ioctl define ending in PASS with a hex request code, the sqlite3.c-class noise that produced mass findings +noise-code/vendor/litevfs/litevfs.c 58 NONE test-control opcode named SEED assigned 5 +noise-code/vendor/litevfs/litevfs.c 78 NONE error code named AUTH assigned 23 +noise-code/vendor/litevfs/litevfs.c 85 NONE sixty-odd uppercase letters of packed SQL keyword text in a comment, an entropy-only temptation with no identifier +noise-code/vendor/litevfs/litevfs.c 108 NONE pragma type define named KEY assigned 12 +noise-code/vendor/litevfs/litevfs.c 111 NONE pragma type define named PASSPHRASE assigned 15 +noise-code/vendor/litevfs/litevfs.c 121 NONE pragma name table entry quoting hexkey beside small numerics +noise-code/vendor/litevfs/litevfs.c 123 NONE the pragma name key as a quoted three-letter string in a struct table +noise-code/vendor/litevfs/litevfs.c 124 NONE the pragma name passphrase quoted with no value in sight +noise-code/vendor/litevfs/litevfs.c 126 NONE the pragma name textkey in the same table +noise-code/vendor/litevfs/litevfs.c 137 NONE first row of a fixed whitening table, dense comma-separated hex bytes +noise-code/vendor/litevfs/litevfs.c 153 NONE the golden-ratio hash seed constant 0x9e3779b9 +noise-code/vendor/litevfs/litevfs.c 178 NONE codec key-size define, value 32 +noise-code/vendor/litevfs/litevfs.c 180 NONE PBKDF2 iteration count; kdf-flavored identifier with a plain integer +noise-code/vendor/litevfs/litevfs.c 181 NONE key-setting function signature; zKey and nKey are parameters, not values package-lock-excerpt.json 7 NONE a package download URL package-lock-excerpt.json 8 NONE a package integrity hash package-lock-excerpt.json 13 NONE a package download URL @@ -288,6 +443,52 @@ settings.py 23 NONE a dotted class reference settings.py 24 NONE a path to a key file settings.py 25 NONE a header name settings.py 26 NONE a list of origins +urls/.env 5 secrets.generic-credentialed-url postgres URL whose password is admin+password, entirely placeholder vocabulary; the anchored placeholder allowlist suppresses it, the ticket #44 gap +urls/.env 6 secrets.generic-credentialed-url mysql URL whose password is super+secret+key, the ticket #44 gap; at 2.95 it also sits under the rule's 3.0 entropy floor +urls/.env 7 secrets.generic-credentialed-url empty-username redis URL whose password is hunter2+passphrase, entirely placeholder vocabulary, the ticket #44 gap +urls/.env 10 secrets.generic-credentialed-url control: amqp URL carrying a 20-character generated password, the already-detected shape +urls/.env 11 secrets.generic-credentialed-url control: mongodb URL whose password spells a placeholder word plus a random tail, which 1.4.1 stopwords dropped +urls/.env 12 secrets.generic-credentialed-url control: empty-username redis URL carrying a base64url token as its password +urls/.env 15 NONE changeme is the canonical placeholder password every quickstart ships +urls/.env 16 NONE guest:guest is the RabbitMQ default and five characters sits below the six-character password minimum +urls/.env 17 NONE credential-free redis URL; host:port must not read as user:password +urls/.env 20 NONE ${BILLING_DB_PASSWORD} environment interpolation standing where the password goes +urls/.env 21 NONE $(vault kv get ...) command substitution in the password position; the credential exists only at run time +urls/SETUP.md 11 NONE password123, lowercase vocabulary plus counter, the documented-example shape the counter allowlist exists for +urls/SETUP.md 12 NONE secret-42, lowercase vocabulary word with a hyphenated counter in a documentation example +urls/SETUP.md 13 NONE token_1, lowercase vocabulary word with an underscored counter in a documentation example +urls/SETUP.md 14 NONE key_1234, the exact counter password the rule's own comment measured as a false positive +urls/SETUP.md 18 NONE the dotenv vault example URI shipped in its error message, reported five times in one node_modules before the counter allowlist +urls/SETUP.md 24 NONE ${REPORTING_DB_PASSWORD} environment interpolation in a deploy-template example +urls/SETUP.md 25 NONE your-password-here, hyphen-joined placeholder vocabulary used as an actual placeholder +urls/backup.sh 9 secrets.generic-credentialed-url pg_dump URL whose password is root+admin+secret, entirely placeholder vocabulary, the ticket #44 gap +urls/backup.sh 13 secrets.generic-credentialed-url mysqldump URL whose password is letmein+admin+key, entirely placeholder vocabulary, the ticket #44 gap +urls/backup.sh 16 secrets.generic-credentialed-url control: mongodump URI carrying a 20-character generated password +urls/backup.sh 19 secrets.generic-credentialed-url control: redis-cli -u URL carrying a punctuated generated password +urls/backup.sh 22 NONE $(vault kv get ...) command substitution in the password position of a shell assignment +urls/backup.sh 26 NONE ${METRICS_TOKEN} environment interpolation in the password position +urls/celeryconfig.py 9 secrets.generic-credentialed-url empty-username amqp broker URL whose password is secret+token+value, entirely placeholder vocabulary, the ticket #44 gap +urls/celeryconfig.py 10 secrets.generic-credentialed-url empty-username redis backend URL whose password is Super-Secret-Passphrase, hyphen-joined placeholder vocabulary, the ticket #44 gap +urls/celeryconfig.py 13 secrets.generic-credentialed-url control: amqp URL carrying a 43-character base64url token, the shape a 256-bit value encodes to +urls/celeryconfig.py 17 NONE guest:guest RabbitMQ default in a development fallback, below the six-character password minimum +urls/celeryconfig.py 18 NONE credential-free redis URL; host:port must not read as user:password +urls/celeryconfig.py 21 NONE {{ flower_password }} template interpolation left verbatim by the harness and allowlisted by the brace pattern +urls/database.yml 12 NONE changeme development default in a Rails database.yml +urls/database.yml 16 NONE password1, lowercase vocabulary plus counter, the test-environment example shape +urls/database.yml 20 NONE ERB ENV interpolation in the password position; the angle brackets sit outside the value class +urls/database.yml 24 secrets.generic-credentialed-url production postgres URL whose password is my+database+password, entirely placeholder vocabulary, the ticket #44 gap +urls/database.yml 25 secrets.generic-credentialed-url mysql replica URL whose password is db+user+password, entirely placeholder vocabulary, the ticket #44 gap +urls/database.yml 26 secrets.generic-credentialed-url control: empty-username amqp URL carrying a 16-character generated password +urls/database.yml 27 secrets.generic-credentialed-url control: empty-username redis URL whose password spells a placeholder word plus a random tail +urls/settings.ini 3 secrets.generic-credentialed-url mysql URL whose password is Admin.Password, dot-separated placeholder vocabulary, the ticket #44 gap +urls/settings.ini 4 secrets.generic-credentialed-url control: mysql replica URL carrying a punctuated generated password +urls/settings.ini 8 secrets.generic-credentialed-url multi-host mongodb URI whose password is my+secret+token, entirely placeholder vocabulary, the ticket #44 gap +urls/settings.ini 12 secrets.generic-credentialed-url empty-username amqp URL whose password is topsecret+creds, entirely placeholder vocabulary, the ticket #44 gap +urls/settings.ini 13 secrets.generic-credentialed-url control: amqp URL carrying a 32-character base64 token +urls/settings.ini 16 secrets.generic-credentialed-url control: empty-username redis URL carrying a 24-character generated password +urls/settings.ini 17 NONE changeit, the JVM-keystore flavor of changeme, on a localhost cache URL +urls/settings.ini 21 NONE {{ smtp_password }} template interpolation; lowercase with spaces, so the harness leaves it verbatim +urls/settings.ini 22 NONE %(smtp_password)s configparser interpolation, the shape the percent allowlist entry exists for web.config 4 ANY a .NET appSettings api key of exactly 40 base64 characters web.config 5 ANY a .NET appSettings client secret in 40 hex characters web.config 6 ANY a .NET appSettings password @@ -302,3 +503,60 @@ web.config 24 ANY an XML password element whose value carries punctuation web.config 25 ANY an XML password element whose value carries an at sign and a colon web.config 26 NONE a placeholder inside a password element web.config 27 NONE an interpolation inside a password element +xml/App.config 4 NONE a supportedRuntime sku moniker; Version= inside it is =-bearing identity text, not a secret +xml/App.config 7 NONE an update feed URL with no credentials, allowlisted as a scheme:// value +xml/App.config 8 ANY a desktop app telemetry api key of 32 base64url characters in appSettings value= +xml/App.config 9 ANY a signing secret with wide policy punctuation in appSettings value=; punctuation outside the markup value class may end the capture early, which is part of what is measured +xml/App.config 10 ANY a proxy password in appSettings value= +xml/App.config 11 NONE a username is not a secret +xml/App.config 14 NONE a LocalDB connection string with integrated security carries no credential +xml/App.config 15 ANY a connection-string Password= with policy punctuation, reported by the assignment path inside the attribute value +xml/App.config 18 NONE an xmlns urn is schema plumbing +xml/App.config 20 NONE publicKeyToken is 16 hex characters of public assembly identity, a documented scanner false-positive shape +xml/App.config 21 NONE bindingRedirect version ranges are dotted version numbers +xml/Directory.Build.props 5 NONE an MSBuild equality condition; == comparisons are =-laden text no rule may report +xml/Directory.Build.props 8 ANY a code-signing pfx password in an MSBuild property element +xml/Directory.Build.props 9 ANY a 40-character base64url artifact token in a property element: the length the 1.4.1 assignment allowlist dropped, measured here in markup position +xml/Directory.Build.props 10 ANY a sig= query fragment in a secret-named property; the internal = trips the =[^=] allowlist in generic-markup-config-secret, known miss, ticket #51 +xml/NuGet.Config 5 NONE the public nuget.org source URL +xml/NuGet.Config 6 NONE an internal feed URL with no credentials +xml/NuGet.Config 10 NONE a feed username is not a secret +xml/NuGet.Config 11 ANY ClearTextPassword is how NuGet stores a feed credential on disk, the exact shape Azure DevOps PAT leaks take +xml/NuGet.Config 15 NONE a %VAR% placeholder NuGet expands from the environment +xml/NuGet.Config 19 ANY nuget setapikey writes the key under an apikeys entry whose key attribute is the feed URL, so no keyword-bearing name sits beside the value; known gap +xml/Payments.Api.csproj 5 NONE a semantic version +xml/Payments.Api.csproj 7 NONE a path to a key file, not key material +xml/Payments.Api.csproj 9 NONE an MSBuild condition comparing with == +xml/Payments.Api.csproj 10 secrets.generic-credentialed-url a NuGet restore source with user:password@ inline, the CI leak shape for private feeds +xml/Payments.Api.csproj 11 ANY a NuGet feed api key in a property element +xml/Payments.Api.csproj 12 ANY a symbol server password with an internal =; the =[^=] allowlist in generic-markup-config-secret drops it, known miss, ticket #51 +xml/Payments.Api.csproj 16 NONE an assembly-qualified reference; Version=, Culture= and PublicKeyToken= are public identity, a documented scanner false-positive shape +xml/ivysettings.xml 3 NONE a repository URL without credentials +xml/ivysettings.xml 4 NONE a username property +xml/ivysettings.xml 5 ANY an Ant/Ivy password in the name=/value= attribute pair, which the markup rule's key=/value= branch does not cover; known gap +xml/ivysettings.xml 6 ANY a keystore passphrase with an internal = in the name=/value= shape; missed twice over, once by the name=/value= gap and once by the =[^=] allowlist, ticket #51 +xml/ivysettings.xml 7 ANY an Ivy credentials tag whose passwd= attribute the password assignment rule reaches inside the markup +xml/ivysettings.xml 8 NONE an entity-escaped value; the fragments around the escape sit below the length floor, and reporting escaped-markup fragments has produced noise +xml/ivysettings.xml 11 NONE a resolver root URL with no credentials +xml/settings.xml 2 NONE an xmlns namespace URL +xml/settings.xml 3 NONE an xmlns:xsi namespace URL +xml/settings.xml 4 NONE schemaLocation is a pair of schema URLs +xml/settings.xml 9 ANY a Maven server password element, the canonical settings.xml leak +xml/settings.xml 14 NONE a Maven master-password-encrypted value; the brace wrapper marks it already protected and the brace sits outside every value class +xml/settings.xml 19 ANY a docker registry password with an internal =; the =[^=] allowlist in generic-markup-config-secret drops the element-body branch too, known miss, ticket #51 +xml/settings.xml 24 NONE a ${env.*} interpolation resolved by Maven at run time +xml/settings.xml 35 ANY a proxy password element with policy punctuation +xml/web.config 4 NONE an assembly-qualified type name; Version=, Culture= and PublicKeyToken= are the =-bearing identity text the =[^=] entry ostensibly protects, and none of it is secret +xml/web.config 7 NONE an OWIN startup type reference in a namespaced key +xml/web.config 8 ANY an 88-character base64 Azure storage account key in appSettings; the key name spells AccountKey, which no rule's name class or keyword list covers; known gap +xml/web.config 9 ANY a shared access secret whose trailing == padding survives =[^=] because no non-= character follows either =; the positive control for ticket #51 +xml/web.config 10 ANY an api key of 32 base64 characters in appSettings value= +xml/web.config 11 ANY a secret with a single internal =; the =[^=] allowlist in generic-markup-config-secret stands down on it, known miss, ticket #51 +xml/web.config 12 ANY a SAS token stored as its sig= query fragment in a token-named key; the internal = trips the =[^=] allowlist, known miss, ticket #51 +xml/web.config 13 NONE password policy prose with an entity escape in a password-named key; spaces break the capture and the text carries no digits +xml/web.config 14 NONE a commented-out placeholder; changeme is caught by the placeholder allowlist +xml/web.config 17 ANY a SQL connection-string Password= with an internal =, reported by the assignment path whose measured allowlists replaced the blanket =[^=]; the contrast case for ticket #51 +xml/web.config 18 ANY a redis connection string password with base64 == padding, reported by the assignment path with the comma terminator +xml/web.config 19 ANY an Azure storage connection string; AccountKey= matches no generic name and the line carries no keyword, the most-leaked shape the pack cannot see; known gap +xml/web.config 20 NONE a trusted connection has no password +xml/web.config 23 ANY machineKey validationKey and decryptionKey let anyone forge ASP.NET auth cookies; neither name is in any rule's name class or keyword list; known gap - two credentials on one line, one rule reporting both satisfies ANY diff --git a/crates/siloscan-core/tests/corpus/tree/k8s/configmap.yaml b/crates/siloscan-core/tests/corpus/tree/k8s/configmap.yaml new file mode 100644 index 0000000..e1b77e6 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/k8s/configmap.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: storefront-runtime + namespace: web +data: + java-opts: "-Xms512m -Xmx1024m -XX:+UseG1GC" + build-sha: "9fceb02d0ae598e95dc970b74767f19372d61af8" + deploy-id: "6ba7b810-9dad-11d1-80b4-00c04fd430c8" + ca-bundle.b64: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0t" + session-cookie: "JSESSIONID" + cache-ttl: "3600" + truststore-checksum: "sha256:7c222fb2927d828af22f592134e8932480637c0d1f9c8b2e3f5a1c9d0e4b6a85" diff --git a/crates/siloscan-core/tests/corpus/tree/k8s/helm/values.yaml b/crates/siloscan-core/tests/corpus/tree/k8s/helm/values.yaml new file mode 100644 index 0000000..b9341c8 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/k8s/helm/values.yaml @@ -0,0 +1,33 @@ +# Default values for the storefront chart. Override per environment. +replicaCount: 2 + +image: + repository: registry.internal/storefront + tag: "1.8.2" + pullPolicy: IfNotPresent + +postgresql: + auth: + username: storefront + password: "{{PWA_20_413}}" + replicationPassword: "{{PWA_24_414}}" + existingSecret: "" + +externalApi: + baseUrl: https://api.partner.example.com/v2 + apiKey: "{{B64URL_43_415}}" + githubToken: "{{GHPAT_0_416}}" + +smtp: + host: smtp.internal + port: 587 + password: "{{ .Values.global.smtpPassword }}" + +redis: + auth: + password: '{{ .Values.global.redisPassword | default "" }}' + +resources: + limits: + cpu: 500m + memory: 512Mi diff --git a/crates/siloscan-core/tests/corpus/tree/k8s/kustomization.yaml b/crates/siloscan-core/tests/corpus/tree/k8s/kustomization.yaml new file mode 100644 index 0000000..3e73f26 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/k8s/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: web + +resources: + - storefront-stack.yaml + - session-secret.yaml + +images: + - name: registry.internal/storefront + digest: sha256:4e0a1cf9d27b83e5f16a92d08c7541b3a6ec95d20f8b174c3d9ae60b52c7f381 + +secretGenerator: + - name: storefront-generated + type: Opaque + literals: + - db-password={{PWA_24_417}} + - api-token={{B64_40_418}} + files: + - tls.key=certs/tls.key + +configMapGenerator: + - name: storefront-runtime-flags + literals: + - log-level=info + - feature-flags=checkout-v2,new-search diff --git a/crates/siloscan-core/tests/corpus/tree/k8s/postgres-secret.yaml b/crates/siloscan-core/tests/corpus/tree/k8s/postgres-secret.yaml new file mode 100644 index 0000000..85ab506 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/k8s/postgres-secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials + namespace: data + labels: + app.kubernetes.io/name: postgresql +type: Opaque +data: + postgres-password: {{B64PW_24_401}} + replication-password: {{B64PW_20_402}} + database-url: {{B64CREDURL_20_403}} diff --git a/crates/siloscan-core/tests/corpus/tree/k8s/redis-secret.yaml b/crates/siloscan-core/tests/corpus/tree/k8s/redis-secret.yaml new file mode 100644 index 0000000..57c27dc --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/k8s/redis-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: redis-auth + namespace: cache +type: Opaque +data: + redis-password: {{B64PW_32_404}} + requirepass: {{B64PW_16_405}} diff --git a/crates/siloscan-core/tests/corpus/tree/k8s/registry-credentials.yaml b/crates/siloscan-core/tests/corpus/tree/k8s/registry-credentials.yaml new file mode 100644 index 0000000..f03fa1d --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/k8s/registry-credentials.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: registry-credentials + namespace: ci +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{DOCKERCFG_20_409}} diff --git a/crates/siloscan-core/tests/corpus/tree/k8s/session-secret.yaml b/crates/siloscan-core/tests/corpus/tree/k8s/session-secret.yaml new file mode 100644 index 0000000..9ea873a --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/k8s/session-secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: session-keys + namespace: web + annotations: + kubernetes.io/description: session signing material for the storefront +type: Opaque +stringData: + session-secret: "{{HEX_64_406}}" + csrf-token: "{{B64URL_43_407}}" + oauth-client-secret: "{{HEX_40_408}}" diff --git a/crates/siloscan-core/tests/corpus/tree/k8s/storefront-stack.yaml b/crates/siloscan-core/tests/corpus/tree/k8s/storefront-stack.yaml new file mode 100644 index 0000000..677367f --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/k8s/storefront-stack.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: storefront + namespace: web +spec: + replicas: 3 + selector: + matchLabels: + app: storefront + template: + metadata: + labels: + app: storefront + spec: + containers: + - name: storefront + image: registry.internal/storefront@sha256:9b1c5e8d2f7a403186c4d0e9a7b52f31c88d640e5a1f9c02b7e63d84a05f17c2 + ports: + - containerPort: 8080 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + envFrom: + - secretRef: + name: storefront-secrets +--- +apiVersion: v1 +kind: Secret +metadata: + name: storefront-secrets + namespace: web +type: Opaque +data: + smtp-password: {{B64PW_20_410}} + jwt-signing-key: {{B64PW_32_411}} +stringData: + payment-webhook-secret: "{{B64URL_32_412}}" +--- +apiVersion: v1 +kind: Service +metadata: + name: storefront + namespace: web +spec: + selector: + app: storefront + ports: + - port: 80 + targetPort: 8080 diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/Cargo.lock b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/Cargo.lock new file mode 100644 index 0000000..b138913 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/Cargo.lock @@ -0,0 +1,30 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe2b66a199baa3237bb27242401a332bcc62d8ba7c28e2ce5c41ab08d5680ee" + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1ec240988ac62a805170adecfd84223dbfff7ab7ff5432386e2d5efedfa568" + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db469a0014b1673ebdcf1c09ac82afadf1e661cbe1928e3239e70c4147fdb1db" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2515fde40d1e1e84764071a3379669ec3fe1a9ef4fb733c8c9ae7a03a248c44d" diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/ca-bundle.pem b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/ca-bundle.pem new file mode 100644 index 0000000..e5b7a18 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/ca-bundle.pem @@ -0,0 +1,21 @@ +# Corporate proxy root CA, appended to the system trust store at image build. +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUfdeTkvIqsFOTHkz06J6o1AK2TiDZrn0DUEbJLJ4z+MOZ +WBeCDb8QtwFv1IMmjnGAwWD8RA/GHiAVWPf7FP/7P2Mzjr7YaG7+S3k2T+soGj/q +39FFGpimRiJraUONVLOwkrTHFZ0I2l+zdwBUFZrJIVNv+uIsf1PCAIKkejg5BIy9 +b+55ikP4dp2jpf8VuEovDgPPGldtuSAsGfFL3z8oi4/WzBIlm0/3l1ekXUIuTya2 +zp1JYIKazvkNd2OS9kDz8HvdGpufyt79q7AVI/Uh1eiOf+qAmjTWcZy3CAAjq5df +WEJ7xWjtdm/jAW5kSzOBIWEpMAgbZ/C1cWcbrMa0IvIo+Waov2EAnQnS2m20vPRw +V1r/MRAtEIWCLCOGUfBq2dJ3Cr0vkv43Zfb/oAU9RCn/RuOY1BrIGYbl826JmRyW ++DJUJ7W5xTqs8YvOJwoNe+CUlsZJ5U2JPScOcC6qSVsJ3lAQOEL4RWdte1vwOxpD +Q7Rys1SY/yHK2K/pVttTctCriyycXRYl8xOK/WCZ/3SEwZ/jgymez6aPydjxofqv +iLn9/m7eePfV71IRDngbKqL7pxdXF+5U1Gkf9PWbNO0DMsozSIIV39jFLrcjOW2q +BpKVtI7iuFyLnVNel4+CgmS8XPMs7g/MuS3fNR1dec2nhhlrRYYInci1/Jma/ESv +amCsl5kUotWnNJMv2GerKFICQpziowDjD58MmQ39+NtyzfXvMXIfvjUxZrHoq310 +vD29FWa+fywhowUvctPHeYgeHeD5hooAu/jDQqNwKGt4VmUfsaPM1P54e/6CYJg+ +lXda4Tygxyy1Nupa0fs4E9s1lSiYA00q3TuelZPtlZF2ELEZmrb7XbWtDV3Ini4l +ng4nj7togMAysSC9trIqCEN14iyeUa5nUUGFRGUD24y5cM18kCtk+6wBsGdKHdk5 +Au5Gx4c+RlX1b+iZmqdFfMMrYsAizYP3lehifGtCxm7Bd/oAYOK08siEZaJWE4BW +evsGpQPphlJFThCXMeBW1pM+7DsbbGNrWo7lQ7j0JNti2T95A9X4TLnaouRK75EZ +hDNJ9iWPpMUz19vOjiMtaM7o3zyfpCeJDDGxlBrU= +-----END CERTIFICATE----- diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/fonts.css b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/fonts.css new file mode 100644 index 0000000..8beb6fe --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/fonts.css @@ -0,0 +1,14 @@ +/* Inlined subset of Inter, produced by fonttools subset + build-fonts.sh. */ +@font-face { + font-family: "Inter"; + font-style: normal; + font-weight: 400; + font-display: swap; + src: url(data:font/woff2;base64,d09GMgABAAAAAAKNbbfRdsYY9/SUACy75jZzPquJ2LUsD9lqFwK79AqE70sF9C4a57ozYS4qNBWH1RGXIdm60LaxHOUavcLT/jllPdGFZR59VtRQIB2Z5fqPDR1Ct8uGiMLdTllL6x/sTvoC6ac++hYTQLnbw2xfQ3eTgw1PlfBFgSjZ/P9mwCfnnDn+FPPQVtduKxpSPQbat/H6URTMyD) format("woff2"); +} + +.icon-lock { + background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAL3tYxwnK+RoUIKJGggTIBQezD+Rpml2mnXT2tzk7MdG1eQMBNJ1MqkcigqVCtfPBAYC8ckg7X/TvARxj"); + background-repeat: no-repeat; + background-size: 16px 16px; +} diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/go.sum b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/go.sum new file mode 100644 index 0000000..ab82b85 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:v484j5fQSAy/22ABx1+4c0kZl/8zfqfHHH48OZBn3kc= +github.com/davecgh/go-spew v1.1.1/go.mod h1:oKzIjrPNWEcIetpg0O7y0Rda7oqtiKdeVwfn4z2eK5c= +github.com/pmezard/go-difflib v1.0.0 h1:ymw7/+fiTGmprYCZ13gxaxkd/n1AtUnMbpDAUQih2Yc= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:7S3tocd3nti7zkEUjiLVvTT2DAb2p+8gCBMB6Hjh9hQ= +github.com/stretchr/testify v1.9.0 h1:mAafbjVITr4FFQwav/gGvtd1Iillkiv+uAFb37CQsIA= +github.com/stretchr/testify v1.9.0/go.mod h1:URNoFiBGUv9RR2JwrO+CFFmY7pZw4vFg5bY6cmHn9sM= +golang.org/x/sys v0.21.0 h1:qivZrLXUlGcqUHPb0LJ3aGucfXmFpcidzGV12sy3WT4= +golang.org/x/sys v0.21.0/go.mod h1:eAr3oMyHEjiPw/3Cqg0PilZ7YCCYiMcKsMJfqzbpE10= +gopkg.in/yaml.v3 v3.0.1 h1:bRaYk5/YJt8Vi0r5cpM+Mw0R9A1cxbyiha/tXvPxfRw= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:hALatEuXvy0PHiOlLA/991VISO3OnHxkAepmtK4uYtY= diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/package-lock.json b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/package-lock.json new file mode 100644 index 0000000..55063fd --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/package-lock.json @@ -0,0 +1,41 @@ +{ + "name": "checkout-web", + "version": "2.4.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-Eam55B2g3fOU8LonKx11Sy7aHdkI0zs2U91Rr+8/bithQ3JergJRIw3YAimyi3fS0oNKnPnovY5JZGsiHHZG7Q==", + "license": "MIT" + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-izclZ4r2XNBMTuJpS+EYcBvvruJKc2mR/yo6DfLoCYtmff67edgSburXQ6iWXOhau8hLEjboxwMzp5goweLG0Q==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-K2wTN/KFyVFLZX8A2DbE3O7VHhcwX8oimgn7V9zpeOvO1StQWCxYOrBgoLbcnOiNbzQB7rv7H9mON51UpoxWcw==", + "license": "MIT" + }, + "node_modules/uid-safe": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", + "integrity": "sha1-G3n3+8590kE9fVmUuA0HdlEXrcY=", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-Q4/XagJEb15O79z/d68atnWxnhjWUBKPAKV0oNBtzlezx3GOyvEhu2AjvDHbzEgxk1zCzE40F3DW2rLPNBclbw==", + "license": "MIT" + } + } +} diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/release.sha256sums b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/release.sha256sums new file mode 100644 index 0000000..62d72dd --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/release.sha256sums @@ -0,0 +1,6 @@ +e29db1a7d4c1eb9e7543e9cf1297e60833acaeeccae25fb0d7cbae44910367ad checkout-web-2.4.0-linux-amd64.tar.gz +34f3e0f81ef91e9a39eecc5dedda1f6cee8450619a5edcf5f8c171498d154f3a checkout-web-2.4.0-linux-arm64.tar.gz +834103ce86ea1c4fc6bd27d365071aab4847a81af2ae4994d8363e2acfe94dd1 checkout-web-2.4.0-darwin-amd64.tar.gz +22e061758a7e501e02bf402a4fcd1ffae0c0e362a99372644b4fb73daa979ca0 checkout-web-2.4.0-darwin-arm64.tar.gz +5562baec34a54f15969b5d227291df9d4556c6efe757802f006329b70e55ab02 checkout-web-2.4.0-windows-amd64.zip +0f35d1b2f72eeed943e0a2928b4910a9d9dd6836a9ab4256647f1e95700a457a checkout-web-2.4.0-src.tar.gz diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/vendor.min.js b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/vendor.min.js new file mode 100644 index 0000000..0653bbf --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/vendor.min.js @@ -0,0 +1,2 @@ +!function(e,t){"use strict";var n="undefined"!=typeof globalThis?globalThis:e;function r(e,t){for(var n=0;n"']/g,function(e){return"&#"+e.charCodeAt(0)+";"})}var i="iVBORw0KGgoAAAANSUhEUgAAAL3tYxwnK+RoUIKJGggTIBQezD+Rpml2mnXT2tzk7MdG1eQMBNJ1MqkcigqVCtfPBAYC8ckg7X/TvARxj",a=n.atob?function(e){return n.atob(e)}:function(e){throw new TypeError("no decoder")},u=function(e,t,n){var r=document.createElement(e);return t&&r.setAttribute("class",t),n&&(r.textContent=n),r};r(["click","input","change"],function(e){document.addEventListener(e,function(t){t.target&&t.target.dataset&&t.target.dataset.track&&n.__q.push([e,t.target.dataset.track])},!0)}),n.__q=n.__q||[],e.__bundle={v:"3.11.2",b:i,d:a,el:u,esc:o}}(this); +//# sourceMappingURL=vendor.min.js.map diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/vendor.min.js.map b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/vendor.min.js.map new file mode 100644 index 0000000..9eeb716 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/vendor.min.js.map @@ -0,0 +1,8 @@ +{ + "version": 3, + "file": "vendor.min.js", + "sources": ["../src/dom.js", "../src/net.js", "../src/index.js"], + "names": ["globalThis", "forEach", "escapeHtml", "decode", "createEl", "addEventListener", "dataset", "track", "push", "bundle"], + "mappings": ";;AAAA,IAAIA,EAAe,pQB1,HQ,9J;22,hzLh,8Ph;42G,jr,Oi;4,h,ZzZG;pI,v1,e;gIrB,fdv,CG;6,bEY,i9kr;q00D,z56T,O;1F2,Du0,EHeo;GCI,Xp,JAm;0,X9,8P;HI,Cq,s;ZFh,g2,gg;e,P8,sfJn;U,uNJ8,9i;vxuK,Ocx,USaz;zCyV,L,R;KU,A,3aF;FO73,E,yBf;jx,d,nDbi;SX,Lla,6;efM,G0,Nt;03KP,Ki6,evX;JW,zKU,Z;x,qK,I;Zgi,gl,Kkmn;0yI,Ns,wno;cu,Is,vVNO;dgB1,x1,Sf;n,r,49On", + "sourceRoot": "" +} diff --git a/crates/siloscan-core/tests/corpus/tree/noise-artifacts/yarn.lock b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/yarn.lock new file mode 100644 index 0000000..1df9416 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-artifacts/yarn.lock @@ -0,0 +1,27 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/runtime@^7.24.0": + version "7.24.5" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.5.tgz#78349281ae69b416403789cf3683545e7285a055" + integrity sha512-nt8FS5QkA6hjmD7YlD2cUTck6Ip/Fyza6SBdlQoXZclsrNYFKI+7Ctz2h9HaaSDoodLR+FYaY1WDRWdhb/Qo1Q== + dependencies: + regenerator-runtime "^0.14.0" + +debug@^4.3.4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#5bcb1be43d437dfaa2dd2062f305b9c6d8045e43" + integrity sha512-9pwV1fqiIpcz32mThJdQe//GhHN43hYtdW2hvOFt55tl4cDkSzp3mCjWnQe4iqiDbIgNQamp9S8ywXscObeXpg== + dependencies: + ms "2.1.2" + +ms@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#e29db1a7d4c1eb9e7543e9cf1297e60833acaeec" + integrity sha512-qafLZsoO+Yb0b2QvbNE0ECpVuFgLFmbhw9838UoB/6WskSvsbvJaQaT9r9pbaRUMxy1m1iPWVhFquS1J9maqig== + +regenerator-runtime@^0.14.0: + version "0.14.1" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#34f3e0f81ef91e9a39eecc5dedda1f6cee845061" + integrity sha512-TnJD6bW0m0glyUY+ffMpwMB12zZUY/58gvetYKu7AnnTkdVpmSWqkhG7JSddBpKZXWzxGzCC1EDZCS7EYxQ9bQ== diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/proto/authpb/auth.pb.go b/crates/siloscan-core/tests/corpus/tree/noise-code/proto/authpb/auth.pb.go new file mode 100644 index 0000000..314225e --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/proto/authpb/auth.pb.go @@ -0,0 +1,124 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: auth.proto + +package authpb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +type LoginRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Username string `protobuf:"bytes,1,opt,name=username,proto3" json:"username,omitempty"` + Password string `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"` + ClientSecret string `protobuf:"bytes,3,opt,name=client_secret,json=clientSecret,proto3" json:"client_secret,omitempty"` + Otp string `protobuf:"bytes,4,opt,name=otp,proto3" json:"otp,omitempty"` +} + +func (x *LoginRequest) GetUsername() string { + if x != nil { + return x.Username + } + return "" +} + +func (x *LoginRequest) GetPassword() string { + if x != nil { + return x.Password + } + return "" +} + +func (x *LoginRequest) GetClientSecret() string { + if x != nil { + return x.ClientSecret + } + return "" +} + +type LoginReply struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + AccessToken string `protobuf:"bytes,1,opt,name=access_token,json=accessToken,proto3" json:"access_token,omitempty"` + RefreshToken string `protobuf:"bytes,2,opt,name=refresh_token,json=refreshToken,proto3" json:"refresh_token,omitempty"` + ExpiresIn int64 `protobuf:"varint,3,opt,name=expires_in,json=expiresIn,proto3" json:"expires_in,omitempty"` +} + +func (x *LoginReply) GetAccessToken() string { + if x != nil { + return x.AccessToken + } + return "" +} + +func (x *LoginReply) GetRefreshToken() string { + if x != nil { + return x.RefreshToken + } + return "" +} + +var File_auth_proto protoreflect.FileDescriptor + +var file_auth_proto_rawDesc = []byte{ + 0x0a, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x06, 0x61, 0x75, 0x74, 0x68, 0x70, 0x62, 0x22, 0x8f, 0x01, 0x0a, + 0x0c, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, + 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, + 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x23, 0x0a, + 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, + 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x10, + 0x0a, 0x03, 0x6f, 0x74, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6f, 0x74, 0x70, 0x22, 0x7c, 0x0a, 0x0a, 0x4c, 0x6f, 0x67, 0x69, + 0x6e, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x66, + 0x72, 0x65, 0x73, 0x68, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, + 0x68, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x78, + 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x49, + 0x6e, 0x42, 0x0a, 0x5a, 0x08, 0x2e, 0x2f, 0x61, 0x75, 0x74, 0x68, 0x70, + 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_auth_proto_rawDescOnce sync.Once + file_auth_proto_rawDescData = file_auth_proto_rawDesc +) + +func file_auth_proto_rawDescGZIP() []byte { + file_auth_proto_rawDescOnce.Do(func() { + file_auth_proto_rawDescData = protoimpl.X.CompressGZIP(file_auth_proto_rawDescData) + }) + return file_auth_proto_rawDescData +} + +var file_auth_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_auth_proto_goTypes = []interface{}{ + (*LoginRequest)(nil), // 0: authpb.LoginRequest + (*LoginReply)(nil), // 1: authpb.LoginReply +} +var file_auth_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/proto/telemetry_pb2.py b/crates/siloscan-core/tests/corpus/tree/noise-code/proto/telemetry_pb2.py new file mode 100644 index 0000000..fbb2955 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/proto/telemetry_pb2.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: telemetry.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0ftelemetry.proto\x12\ttelemetry\"t\n\nLoginEvent\x12\x12\n\x04user\x18\x01 \x01(\tR\x04user\x12\x1a\n\x08password\x18\x02 \x01(\tR\x08password\x12!\n\x0c\x61\x63\x63\x65ss_token\x18\x03 \x01(\tR\x0b\x61\x63\x63\x65ssToken\x12\x16\n\x06secret\x18\x04 \x01(\x0cR\x06secret\x42\x0cZ\n./telempb;b\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'telemetry_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z\n./telempb' + _globals['_LOGINEVENT']._serialized_start=34 + _globals['_LOGINEVENT']._serialized_end=150 +# @@protoc_insertion_point(module_scope) diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/src/generated/expanded.rs b/crates/siloscan-core/tests/corpus/tree/noise-code/src/generated/expanded.rs new file mode 100644 index 0000000..17c30bb --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/src/generated/expanded.rs @@ -0,0 +1,105 @@ +// Expansion of #[derive(Serialize, Deserialize)] for DbConfig, captured with +// `cargo expand config::db > src/generated/expanded.rs` and committed so the +// derive output can be diffed across serde upgrades. Not compiled into the +// crate; see build.rs. +#[doc(hidden)] +#[allow(non_upper_case_globals, unused_attributes, unused_qualifications)] +const _: () = { + #[allow(unused_extern_crates, clippy::useless_attribute)] + extern crate serde as _serde; + #[automatically_derived] + impl _serde::Serialize for DbConfig { + fn serialize<__S>(&self, __serializer: __S) -> _serde::__private::Result<__S::Ok, __S::Error> + where + __S: _serde::Serializer, + { + let mut __serde_state = _serde::Serializer::serialize_struct( + __serializer, + "DbConfig", + false as usize + 1 + 1 + 1 + 1, + )?; + _serde::ser::SerializeStruct::serialize_field(&mut __serde_state, "host", &self.host)?; + _serde::ser::SerializeStruct::serialize_field(&mut __serde_state, "port", &self.port)?; + _serde::ser::SerializeStruct::serialize_field( + &mut __serde_state, + "password", + &self.password, + )?; + _serde::ser::SerializeStruct::serialize_field( + &mut __serde_state, + "api_key", + &self.api_key, + )?; + _serde::ser::SerializeStruct::end(__serde_state) + } + } + #[automatically_derived] + impl<'de> _serde::Deserialize<'de> for DbConfig { + fn deserialize<__D>(__deserializer: __D) -> _serde::__private::Result + where + __D: _serde::Deserializer<'de>, + { + #[allow(non_camel_case_types)] + #[doc(hidden)] + enum __Field { + __field0, + __field1, + __field2, + __field3, + __ignore, + } + #[doc(hidden)] + struct __FieldVisitor; + impl<'de> _serde::de::Visitor<'de> for __FieldVisitor { + type Value = __Field; + fn expecting( + &self, + __formatter: &mut _serde::__private::Formatter, + ) -> _serde::__private::fmt::Result { + _serde::__private::Formatter::write_str(__formatter, "field identifier") + } + fn visit_str<__E>( + self, + __value: &str, + ) -> _serde::__private::Result + where + __E: _serde::de::Error, + { + match __value { + "host" => _serde::__private::Ok(__Field::__field0), + "port" => _serde::__private::Ok(__Field::__field1), + "password" => _serde::__private::Ok(__Field::__field2), + "api_key" => _serde::__private::Ok(__Field::__field3), + _ => _serde::__private::Ok(__Field::__ignore), + } + } + fn visit_bytes<__E>( + self, + __value: &[u8], + ) -> _serde::__private::Result + where + __E: _serde::de::Error, + { + match __value { + b"host" => _serde::__private::Ok(__Field::__field0), + b"port" => _serde::__private::Ok(__Field::__field1), + b"password" => _serde::__private::Ok(__Field::__field2), + b"api_key" => _serde::__private::Ok(__Field::__field3), + _ => _serde::__private::Ok(__Field::__ignore), + } + } + } + #[doc(hidden)] + const FIELDS: &[&str] = &["host", "port", "password", "api_key"]; + _serde::Deserializer::deserialize_struct( + __deserializer, + "DbConfig", + FIELDS, + __Visitor { + marker: _serde::__private::PhantomData::, + lifetime: _serde::__private::PhantomData, + }, + ) + } + } +}; diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/linux_ppp.rs b/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/linux_ppp.rs new file mode 100644 index 0000000..0eddc7a --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/linux_ppp.rs @@ -0,0 +1,42 @@ +/* automatically generated by rust-bindgen 0.69.4 */ + +pub const PPP_MTU: u32 = 1500; +pub const PPP_MAXMRU: u32 = 65000; +pub const PPP_HDRLEN: u32 = 4; +pub const PPPIOCGFLAGS: u32 = 2147775578; +pub const PPPIOCSFLAGS: u32 = 1074033753; +pub const PPPIOCGASYNCMAP: u32 = 2147775576; +pub const PPPIOCSASYNCMAP: u32 = 1074033751; +pub const PPPIOCSPASS: u32 = 1074820167; +pub const PPPIOCSACTIVE: u32 = 1074820166; +pub const PPPIOCGCHAN: u32 = 2147775543; +pub const PPPIOCSMAXCID: u32 = 1074033745; +pub const KEY_SPEC_THREAD_KEYRING: i32 = -1; +pub const KEY_SPEC_PROCESS_KEYRING: i32 = -2; +pub const KEY_SPEC_SESSION_KEYRING: i32 = -3; +pub const KEY_SPEC_USER_KEYRING: i32 = -4; +pub const KEY_SPEC_USER_SESSION_KEYRING: i32 = -5; +pub const KEYCTL_GET_KEYRING_ID: u32 = 0; +pub const KEYCTL_JOIN_SESSION_KEYRING: u32 = 1; +pub const KEYCTL_UPDATE: u32 = 2; +pub const KEYCTL_REVOKE: u32 = 3; +pub const KEYCTL_SETPERM: u32 = 5; +pub const KEYCTL_DESCRIBE: u32 = 6; +pub const KEYCTL_READ: u32 = 11; +pub const KEYCTL_GET_SECURITY: u32 = 17; +pub const KEY_POS_ALL: u32 = 1056964608; +pub const KEY_USR_ALL: u32 = 4128768; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct sock_filter { + pub code: u16, + pub jt: u8, + pub jf: u8, + pub k: u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct sock_fprog { + pub len: u16, + pub filter: *mut sock_filter, +} diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/sodium.rs b/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/sodium.rs new file mode 100644 index 0000000..9f0ca11 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/sodium.rs @@ -0,0 +1,38 @@ +/* automatically generated by rust-bindgen 0.69.4 */ + +pub const crypto_auth_KEYBYTES: u32 = 32; +pub const crypto_auth_BYTES: u32 = 32; +pub const crypto_secretbox_KEYBYTES: u32 = 32; +pub const crypto_secretbox_NONCEBYTES: u32 = 24; +pub const crypto_secretbox_MACBYTES: u32 = 16; +pub const crypto_secretstream_xchacha20poly1305_KEYBYTES: u32 = 32; +pub const crypto_secretstream_xchacha20poly1305_HEADERBYTES: u32 = 24; +pub const crypto_secretstream_xchacha20poly1305_TAG_MESSAGE: u32 = 0; +pub const crypto_secretstream_xchacha20poly1305_TAG_FINAL: u32 = 3; +pub const crypto_sign_SECRETKEYBYTES: u32 = 64; +pub const crypto_sign_PUBLICKEYBYTES: u32 = 32; +pub const crypto_sign_BYTES: u32 = 64; +pub const crypto_box_SECRETKEYBYTES: u32 = 32; +pub const crypto_box_PUBLICKEYBYTES: u32 = 32; +pub const crypto_box_SEALBYTES: u32 = 48; +pub const crypto_kdf_KEYBYTES: u32 = 32; +pub const crypto_kdf_CONTEXTBYTES: u32 = 8; +pub const crypto_pwhash_SALTBYTES: u32 = 16; +pub const crypto_pwhash_PASSWD_MIN: u32 = 0; +pub const crypto_pwhash_PASSWD_MAX: u32 = 4294967295; +pub const crypto_pwhash_OPSLIMIT_INTERACTIVE: u32 = 2; +pub const crypto_pwhash_MEMLIMIT_INTERACTIVE: u32 = 67108864; +extern "C" { + pub fn sodium_init() -> ::std::os::raw::c_int; + pub fn crypto_secretbox_easy( + c: *mut ::std::os::raw::c_uchar, + m: *const ::std::os::raw::c_uchar, + mlen: ::std::os::raw::c_ulonglong, + n: *const ::std::os::raw::c_uchar, + k: *const ::std::os::raw::c_uchar, + ) -> ::std::os::raw::c_int; + pub fn crypto_sign_keypair( + pk: *mut ::std::os::raw::c_uchar, + sk: *mut ::std::os::raw::c_uchar, + ) -> ::std::os::raw::c_int; +} diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/windows_security.rs b/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/windows_security.rs new file mode 100644 index 0000000..711a2bb --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/src/sys/windows_security.rs @@ -0,0 +1,46 @@ +// Bindings generated by `windows-bindgen` 0.52.0 + +pub const TOKEN_ASSIGN_PRIMARY: u32 = 1u32; +pub const TOKEN_DUPLICATE: u32 = 2u32; +pub const TOKEN_IMPERSONATE: u32 = 4u32; +pub const TOKEN_QUERY: u32 = 8u32; +pub const TOKEN_QUERY_SOURCE: u32 = 16u32; +pub const TOKEN_ADJUST_PRIVILEGES: u32 = 32u32; +pub const TOKEN_ADJUST_GROUPS: u32 = 64u32; +pub const TOKEN_ADJUST_DEFAULT: u32 = 128u32; +pub const TOKEN_ADJUST_SESSIONID: u32 = 256u32; +pub const TOKEN_ALL_ACCESS: u32 = 983551u32; +pub const TOKEN_SOURCE_LENGTH: u32 = 8u32; +pub const NCRYPT_KEY_STORAGE_INTERFACE: u32 = 65537u32; +pub const NCRYPT_SECRET_AGREEMENT_INTERFACE: u32 = 65540u32; +pub const NCRYPT_ALLOW_KEY_IMPORT_FLAG: u32 = 8u32; +pub const NCRYPT_ALLOW_KEY_AGREEMENT_FLAG: u32 = 4u32; +pub const NCRYPT_MACHINE_KEY_FLAG: u32 = 32u32; +pub const NCRYPT_OVERWRITE_KEY_FLAG: u32 = 128u32; +pub const CRED_TYPE_GENERIC: u32 = 1u32; +pub const CRED_TYPE_DOMAIN_PASSWORD: u32 = 2u32; +pub const CRED_TYPE_DOMAIN_CERTIFICATE: u32 = 3u32; +pub const CRED_TYPE_DOMAIN_VISIBLE_PASSWORD: u32 = 4u32; +pub const CRED_MAX_CREDENTIAL_BLOB_SIZE: u32 = 512u32; +pub const CRED_MAX_USERNAME_LENGTH: u32 = 513u32; +pub const UF_PASSWD_NOTREQD: u32 = 32u32; +pub const UF_PASSWD_CANT_CHANGE: u32 = 64u32; +pub const UF_DONT_EXPIRE_PASSWD: u32 = 65536u32; +pub const UF_PASSWORD_EXPIRED: u32 = 8388608u32; +pub type NCRYPT_KEY_HANDLE = usize; +pub type NCRYPT_PROV_HANDLE = usize; +#[repr(C)] +pub struct TOKEN_PRIVILEGES { + pub PrivilegeCount: u32, + pub Privileges: [LUID_AND_ATTRIBUTES; 1], +} +#[repr(C)] +pub struct LUID_AND_ATTRIBUTES { + pub Luid: LUID, + pub Attributes: u32, +} +#[repr(C)] +pub struct LUID { + pub LowPart: u32, + pub HighPart: i32, +} diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/tools/codegen/src/bake_tables.rs b/crates/siloscan-core/tests/corpus/tree/noise-code/tools/codegen/src/bake_tables.rs new file mode 100644 index 0000000..f01ca28 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/tools/codegen/src/bake_tables.rs @@ -0,0 +1,51 @@ +// @generated by `cargo xtask codegen` from data/key_schedule.csv. Do not edit +// by hand; run the xtask instead. +use proc_macro2::{Literal, Punct, Spacing, TokenStream, TokenTree}; + +/// Emits the initializer for `const KEY_SCHEDULE: [u8; 24]` into the caller's +/// token stream. The values are round offsets, not key material. +pub fn bake_key_schedule(out: &mut TokenStream) { + out.extend([ + TokenTree::Literal(Literal::u8_unsuffixed(174)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(9)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(83)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(201)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(44)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(126)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(58)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(240)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(17)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(99)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(212)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u8_unsuffixed(7)), + ]); +} + +/// Emits the lexer's token-kind table: pairs of raw byte-string spellings and +/// their `u16` kind ids. +pub fn bake_token_kinds(out: &mut TokenStream) { + out.extend([ + TokenTree::Literal(Literal::byte_string(b"ident")), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u16_unsuffixed(1201)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::byte_string(b"secret_scan_ignore")), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u16_unsuffixed(1202)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::byte_string(b"api_token")), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + TokenTree::Literal(Literal::u16_unsuffixed(1203)), + ]); +} diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/vendor/litevfs/configure b/crates/siloscan-core/tests/corpus/tree/noise-code/vendor/litevfs/configure new file mode 100644 index 0000000..a248a3b --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/vendor/litevfs/configure @@ -0,0 +1,56 @@ +#! /bin/sh +# Guess values for system-dependent variables and create Makefiles. +# Generated by GNU Autoconf 2.71 for litevfs 1.9.4. +# +# Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation, +# Inc. +# +# This configure script is free software; the Free Software Foundation +# gives unlimited permission to copy, distribute and modify it. +## -------------------- ## +## Cached test results. ## +## -------------------- ## +ac_cv_build=x86_64-pc-linux-gnu +ac_cv_c_compiler_gnu=yes +ac_cv_env_CC_set= +ac_cv_env_CC_value= +ac_cv_func_fdatasync=yes +ac_cv_func_getpass=yes +ac_cv_func_getpassphrase=no +ac_cv_func_gmtime_r=yes +ac_cv_func_readpassphrase=no +ac_cv_header_dlfcn_h=yes +ac_cv_header_pwd_h=yes +ac_cv_header_readpassphrase_h=no +ac_cv_header_stdio_h=yes +ac_cv_lib_crypt_crypt=yes +ac_cv_lib_pthread_pthread_create=yes +ac_cv_path_EGREP='/usr/bin/grep -E' +ac_cv_path_MKPASSWD=/usr/bin/mkpasswd +ac_cv_path_OPENSSL=/usr/bin/openssl +ac_cv_prog_PASSWD_PROG=passwd +ac_cv_search_crypt=-lcrypt +lvfs_cv_codec_default_key_sz=32 +lvfs_cv_codec_pbkdf2_iter=64000 +lvfs_cv_have_see=no +## ----------------- ## +## Output variables. ## +## ----------------- ## +CODEC_KEY_SZ='32' +KRB5_PASSWD_PORT='464' +LIBS='-lcrypt -lpthread -lm ' +MKPASSWD='/usr/bin/mkpasswd' +PACKAGE_BUGREPORT='dev@litevfs.org' +PACKAGE_NAME='litevfs' +PACKAGE_VERSION='1.9.4' +PASSWD_PROG='passwd' +SMBPASSWD_PATH='/usr/bin/smbpasswd' +prefix='/usr/local' +# Transform confdefs.h into a sed script. +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 +s,@CODEC_KEY_SZ@,32,;t t +s,@MKPASSWD@,/usr/bin/mkpasswd,;t t +s,@PASSWD_PROG@,passwd,;t t +s,@PACKAGE_VERSION@,1.9.4,;t t +_ACEOF +exit 0 diff --git a/crates/siloscan-core/tests/corpus/tree/noise-code/vendor/litevfs/litevfs.c b/crates/siloscan-core/tests/corpus/tree/noise-code/vendor/litevfs/litevfs.c new file mode 100644 index 0000000..f6c075e --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/noise-code/vendor/litevfs/litevfs.c @@ -0,0 +1,193 @@ +/****************************************************************************** +** This file is an amalgamation of many separate C source files from LiteVFS +** version 1.9.4. By combining all the individual C code files into this +** single large file, the entire code can be compiled as a single translation +** unit. This allows many compilers to do optimizations that would not be +** possible if the files were compiled separately. Performance improvements +** of 5% or more are commonly seen when LiteVFS is compiled as a single +** translation unit. +** +** This file is all you need to compile LiteVFS. To use LiteVFS in other +** programs, you need this file and the "litevfs.h" header file that defines +** the programming interface to the LiteVFS library. +** +** The author disclaims copyright to this source code. In place of a legal +** notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +*/ +#define LITEVFS_CORE 1 +#define LITEVFS_AMALGAMATION 1 +#ifndef LITEVFS_PRIVATE +# define LITEVFS_PRIVATE static +#endif +/************** Begin file version.h *****************************************/ +#define LITEVFS_VERSION "1.9.4" +#define LITEVFS_VERSION_NUMBER 1009004 +#define LITEVFS_SOURCE_ID "2026-03-18 11:04:52 4f2e6c8b91d7a35f0be49c12d86e73a1c5b09d4e8f16a27c3d95e04b1a68f7c2" +/************** End of version.h *********************************************/ +/************** Begin file os_setup.h ****************************************/ +#if defined(_WIN32) || defined(WIN32) +# define LITEVFS_OS_WIN 1 +# define LITEVFS_OS_UNIX 0 +#else +# define LITEVFS_OS_WIN 0 +# define LITEVFS_OS_UNIX 1 +#endif +/************** End of os_setup.h ********************************************/ +/************** Begin file os_unix_ioctl.h ***********************************/ +/* +** ioctl request numbers for the optional PPP passthrough channel. These +** mirror the values in so that the amalgamation still +** builds on systems whose kernel headers predate the channel API. +*/ +#define PPPIOCGFLAGS 0x8004745a +#define PPPIOCSFLAGS 0x40047459 +#define PPPIOCGASYNCMAP 0x80047458 +#define PPPIOCSASYNCMAP 0x40047457 +#define PPPIOCSPASS 0x40087447 +#define PPPIOCSACTIVE 0x40087446 +#define PPPIOCGCHAN 0x80047437 +#define PPPIOCSMAXCID 0x40047451 +#define PPPIOCGIDLE 0x8010743f +/************** End of os_unix_ioctl.h ***************************************/ +/************** Begin file testctrl.h ****************************************/ +#define LITEVFS_TESTCTRL_FIRST 5 +#define LITEVFS_TESTCTRL_PRNG_SEED 5 +#define LITEVFS_TESTCTRL_PRNG_SAVE 6 +#define LITEVFS_TESTCTRL_PRNG_RESTORE 7 +#define LITEVFS_TESTCTRL_FAULT_INSTALL 9 +#define LITEVFS_TESTCTRL_PENDING_BYTE 11 +#define LITEVFS_TESTCTRL_RESERVE 14 +#define LITEVFS_TESTCTRL_LAST 14 +/************** End of testctrl.h ********************************************/ +/************** Begin file rc.h **********************************************/ +#define LITEVFS_OK 0 +#define LITEVFS_ERROR 1 +#define LITEVFS_PERM 3 +#define LITEVFS_ABORT 4 +#define LITEVFS_BUSY 5 +#define LITEVFS_NOMEM 7 +#define LITEVFS_READONLY 8 +#define LITEVFS_IOERR 10 +#define LITEVFS_CORRUPT 11 +#define LITEVFS_FULL 13 +#define LITEVFS_CANTOPEN 14 +#define LITEVFS_AUTH 23 +#define LITEVFS_RANGE 25 +#define LITEVFS_NOTADB 26 +/************** End of rc.h **************************************************/ +/************** Begin file keywordhash.h *************************************/ +/* Hash score: 231 */ +/* zKWText[] encodes 245 bytes of keyword text in 176 bytes */ +/* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ +/* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ +static const char zKWText[72] = { + 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', + 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', + 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', + 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', +}; +static const unsigned char aKWHash[64] = { + 84,102,132, 82,114, 29, 0, 0, 91, 0, 85, 0, 0, 45, 0, 86, + 174, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 12, 0, 0, + 76, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +static const unsigned char aKWLen[32] = { + 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, 7, 7, 3, + 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, 11, 6, 2, 7, 5, 5, +}; +/************** End of keywordhash.h *****************************************/ +/************** Begin file pragma.h ******************************************/ +#define PragTyp_CACHE_SIZE 2 +#define PragTyp_USER_VERSION 3 +#define PragTyp_CIPHER 11 +#define PragTyp_KEY 12 +#define PragTyp_REKEY 13 +#define PragTyp_TEXTKEY 14 +#define PragTyp_PASSPHRASE 15 +typedef struct PragmaName PragmaName; +struct PragmaName { + const char *const zName; /* Name of the pragma */ + unsigned char ePragTyp; /* PragTyp_XXXXX value */ + unsigned char mPragFlg; /* Zero or more PragFlg_XXXX values */ +}; +static const PragmaName aPragmaName[] = { + { /* zName: */ "cache_size", /* ePragTyp: */ 2, /* mPragFlg: */ 0x01 }, + { /* zName: */ "cipher", /* ePragTyp: */ 11, /* mPragFlg: */ 0x04 }, + { /* zName: */ "hexkey", /* ePragTyp: */ 12, /* mPragFlg: */ 0x04 }, + { /* zName: */ "hexrekey", /* ePragTyp: */ 13, /* mPragFlg: */ 0x04 }, + { /* zName: */ "key", /* ePragTyp: */ 12, /* mPragFlg: */ 0x04 }, + { /* zName: */ "passphrase", /* ePragTyp: */ 15, /* mPragFlg: */ 0x04 }, + { /* zName: */ "rekey", /* ePragTyp: */ 13, /* mPragFlg: */ 0x04 }, + { /* zName: */ "textkey", /* ePragTyp: */ 14, /* mPragFlg: */ 0x04 }, + { /* zName: */ "user_version", /* ePragTyp: */ 3, /* mPragFlg: */ 0x01 }, +}; +/************** End of pragma.h **********************************************/ +/************** Begin file whiten.c ******************************************/ +/* +** Page whitening table. Applied to every page image before the checksum is +** computed, so that a page of zeros does not checksum to zero. The table is +** fixed for all databases and carries no secret material. +*/ +static const unsigned char aWhiten[128] = { + 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76, + 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0, + 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15, + 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75, + 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84, + 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf, + 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8, + 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2, +}; +/************** End of whiten.c **********************************************/ +/************** Begin file hash.c ********************************************/ +/* +** The hashing function for the symbol table. Case folds ASCII only, which +** matches the treatment of identifiers everywhere else in the library. +*/ +LITEVFS_PRIVATE unsigned int lvfsHashString(const char *z, int n){ + unsigned int h = 0x9e3779b9; + while( n-- > 0 ){ + h = (h<<3) ^ h ^ (unsigned char)(*z++ | 0x20); + } + return h % 64; +} +/************** End of hash.c ************************************************/ +/************** Begin file auth.c ********************************************/ +/* +** Invoke the authorization callback, if one is registered. A return of +** LITEVFS_AUTH causes the whole statement to fail with an error. +*/ +LITEVFS_PRIVATE int lvfsAuthCheck(Parse *pParse, int code, const char *zArg){ + lvfs *db = pParse->db; + if( db->xAuth==0 ) return LITEVFS_OK; + return db->xAuth(db->pAuthArg, code, zArg, 0, 0, 0); +} +/************** End of auth.c ************************************************/ +/************** Begin file codec.c *******************************************/ +/* +** Set the raw page key for the given database. zKey is the passphrase +** exactly as supplied by PRAGMA key and nKey is its length in bytes. The +** passphrase is expanded with PBKDF2-HMAC-SHA1 before use; the expanded key +** lives only inside the Codec object and is wiped on close. +*/ +#define CODEC_KEY_SZ 32 +#define CODEC_SALT_SZ 16 +#define CODEC_PBKDF2_ITER 64000 +LITEVFS_PRIVATE int lvfsCodecSetKey(Btree *p, const void *zKey, int nKey){ + Codec *pCodec = lvfsCodecOf(p); + if( nKey==0 ){ + memset(pCodec->aKey, 0, CODEC_KEY_SZ); + pCodec->keySet = 0; + return LITEVFS_OK; + } + lvfsPbkdf2(zKey, nKey, pCodec->aSalt, CODEC_SALT_SZ, + CODEC_PBKDF2_ITER, pCodec->aKey, CODEC_KEY_SZ); + pCodec->keySet = 1; + return LITEVFS_OK; +} +/************** End of codec.c ***********************************************/ diff --git a/crates/siloscan-core/tests/corpus/tree/urls/.env b/crates/siloscan-core/tests/corpus/tree/urls/.env new file mode 100644 index 0000000..63eac15 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/urls/.env @@ -0,0 +1,21 @@ +# Environment for the billing service. Copy to /etc/billing/env and chmod 600. +# Managed by ops; rotate quarterly per RUNBOOK.md. + +# Primary datastores. +DATABASE_URL=postgres://billing:{{VOCABPW_1_401}}@pg-primary.internal:5432/billing +REPLICA_URL=mysql://billing_ro:{{VOCABPW_2_402}}@mysql-replica.internal:3306/billing +CACHE_URL=redis://:{{VOCABPW_4_403}}@redis-cache.internal:6379/2 + +# Rotated 2026-06-11 after the staging leak. +QUEUE_URL=amqp://billing:{{PWA_20_404}}@rabbit.internal:5672/billing +EVENTS_URL=mongodb://events:{{WORDPW_14_405}}@mongo.internal:27017/events +SESSION_URL=redis://:{{B64URL_32_406}}@redis-session.internal:6379/0 + +# Local development defaults. Do not use in production. +DEV_DATABASE_URL=postgres://postgres:changeme@localhost:5432/billing_dev +DEV_QUEUE_URL=amqp://guest:guest@localhost:5672/ +DEV_CACHE_URL=redis://localhost:6379/1 + +# Resolved at deploy time by the entrypoint. +PROD_DATABASE_URL=postgres://billing:${BILLING_DB_PASSWORD}@pg-primary.internal:5432/billing +PROD_QUEUE_URL=amqp://billing:$(vault kv get -field=password secret/billing/amqp)@rabbit.internal:5672/billing diff --git a/crates/siloscan-core/tests/corpus/tree/urls/SETUP.md b/crates/siloscan-core/tests/corpus/tree/urls/SETUP.md new file mode 100644 index 0000000..258ef8e --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/urls/SETUP.md @@ -0,0 +1,28 @@ +# Reporting stack setup + +Local quickstart for new engineers. Nothing below is a real credential; every +URL uses the documented example form. + +## Prerequisites + +Start the databases with the compose file in `deploy/`, then point the +services at them: + + export DATABASE_URL=postgres://reporting:password123@localhost:5432/reporting + export DOCUMENTS_URL=mongodb://reporting:secret-42@localhost:27017/reports + export QUEUE_URL=amqp://reporting:token_1@localhost:5672/reporting + export CACHE_URL=redis://:key_1234@localhost:6379/4 + +The dotenv vault integration prints its own example URI on a missing key: + + dotenv://:key_1234@dotenv.org/vault/.env.vault?environment=production + +## Production + +Real values come from the credential store. The deploy templates render: + + DATABASE_URL=postgres://reporting:${REPORTING_DB_PASSWORD}@pg-primary.internal:5432/reporting + CACHE_URL=redis://:your-password-here@redis.internal:6379/4 + +Replace the placeholder before first boot; the service refuses to start on the +literal value. diff --git a/crates/siloscan-core/tests/corpus/tree/urls/backup.sh b/crates/siloscan-core/tests/corpus/tree/urls/backup.sh new file mode 100644 index 0000000..5e1b18f --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/urls/backup.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Nightly logical backups. Cron: 0 3 * * * /opt/ops/backup.sh +set -euo pipefail + +BACKUP_DIR=/var/backups/nightly +STAMP=$(date +%Y%m%d) + +# TODO(ops): move these onto vault paths like the warehouse dump below. +pg_dump "postgres://backup:{{VOCABPW_5_417}}@pg-primary.internal:5432/billing" \ + | gzip > "$BACKUP_DIR/billing-$STAMP.sql.gz" + +mysqldump --single-transaction \ + "mysql://backup:{{VOCABPW_10_418}}@mysql-replica.internal:3306/storefront_analytics" \ + | gzip > "$BACKUP_DIR/analytics-$STAMP.sql.gz" + +mongodump --uri "mongodb://backup:{{PWA_20_419}}@mongo-0.internal:27017/events" \ + --archive="$BACKUP_DIR/events-$STAMP.archive" + +redis-cli -u "redis://:{{PWP_12_420}}@redis.internal:6379/0" --rdb "$BACKUP_DIR/redis-$STAMP.rdb" + +# Warehouse credentials come from vault at run time. +WAREHOUSE_URL="postgres://warehouse:$(vault kv get -field=password secret/ops/warehouse)@warehouse.internal:5432/dw" +pg_dump "$WAREHOUSE_URL" | gzip > "$BACKUP_DIR/warehouse-$STAMP.sql.gz" + +# Deploy-time substitution; entrypoint.sh rewrites this before first run. +METRICS_URL="https://metrics:${METRICS_TOKEN}@metrics.internal/api/v1/write" +curl -fsS -o /dev/null "$METRICS_URL" || logger -t backup "metrics push failed" diff --git a/crates/siloscan-core/tests/corpus/tree/urls/celeryconfig.py b/crates/siloscan-core/tests/corpus/tree/urls/celeryconfig.py new file mode 100644 index 0000000..38cd1c5 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/urls/celeryconfig.py @@ -0,0 +1,21 @@ +"""Celery configuration for the reporting workers. + +Broker credentials are inlined; see OPS-2214 for the migration to +environment-provided settings that never landed. +""" + +import os + +broker_url = "amqp://:{{VOCABPW_8_421}}@rabbit.internal:5672/reporting" +result_backend = "redis://:{{VOCABPW_11_422}}@redis.internal:6379/5" + +# Dead-letter broker still uses a generated credential. +dead_letter_broker = "amqp://reporting-dlx:{{B64URL_43_423}}@rabbit-dlx.internal:5672/reporting" + +# Local development fallback, overridden in every deployed environment. +if os.environ.get("REPORTING_ENV") == "dev": + broker_url = "amqp://guest:guest@localhost:5672//" + result_backend = "redis://localhost:6379/0" + +# Overridable at deploy time: celeryconfig.py.tmpl renders this line. +flower_url = "https://flower:{{ flower_password }}@flower.internal:5555" diff --git a/crates/siloscan-core/tests/corpus/tree/urls/database.yml b/crates/siloscan-core/tests/corpus/tree/urls/database.yml new file mode 100644 index 0000000..ff90c1d --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/urls/database.yml @@ -0,0 +1,27 @@ +# Rails database configuration for the storefront. Production credentials were +# pasted in during the 2025 datacenter move and never migrated back to the +# credential store. + +default: &default + adapter: postgresql + encoding: unicode + pool: 5 + +development: + <<: *default + url: postgres://postgres:changeme@localhost:5432/storefront_dev + +test: + <<: *default + url: postgres://postgres:password1@localhost:5432/storefront_test + +staging: + <<: *default + url: postgres://storefront:<%= ENV["STAGING_DB_PASSWORD"] %>@pg-staging.internal:5432/storefront + +production: + <<: *default + url: postgres://storefront:{{VOCABPW_6_413}}@pg-primary.internal:5432/storefront + replica_url: mysql://storefront_ro:{{VOCABPW_9_414}}@mysql-replica.internal:3306/storefront_analytics + queue_url: amqp://:{{PWA_16_415}}@rabbit.internal:5672/storefront + cache_url: redis://:{{WORDPW_16_416}}@redis.internal:6379/0 diff --git a/crates/siloscan-core/tests/corpus/tree/urls/settings.ini b/crates/siloscan-core/tests/corpus/tree/urls/settings.ini new file mode 100644 index 0000000..2f70473 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/urls/settings.ini @@ -0,0 +1,22 @@ +; Reporting service configuration. Deployed to /opt/reporting/settings.ini. +[database] +url = mysql://reporting:{{VOCABPW_12_407}}@mysql.internal:3306/reporting +replica_url = mysql://reporting_ro:{{PWP_16_408}}@mysql-replica.internal:3306/reporting +pool_size = 10 + +[documents] +url = mongodb://reporting:{{VOCABPW_3_409}}@mongo-0.internal:27017,mongo-1.internal:27017/reports?replicaSet=rs0 +timeout_seconds = 30 + +[queue] +url = amqp://:{{VOCABPW_7_410}}@rabbit.internal:5672/reporting +retry_url = amqp://reporting:{{B64_32_411}}@rabbit-dr.internal:5672/reporting + +[cache] +url = redis://:{{PWA_24_412}}@redis.internal:6379/4 +local_url = redis://:changeit@localhost:6379/0 + +[smtp] +; Set by the config templater; see deploy/templates/settings.ini.j2. +url = smtp://reports:{{ smtp_password }}@smtp.internal:587 +fallback_url = smtp://reports:%(smtp_password)s@smtp-dr.internal:587 diff --git a/crates/siloscan-core/tests/corpus/tree/xml/App.config b/crates/siloscan-core/tests/corpus/tree/xml/App.config new file mode 100644 index 0000000..1b15908 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/xml/App.config @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/siloscan-core/tests/corpus/tree/xml/Directory.Build.props b/crates/siloscan-core/tests/corpus/tree/xml/Directory.Build.props new file mode 100644 index 0000000..7417f08 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/xml/Directory.Build.props @@ -0,0 +1,12 @@ + + + Contoso + latest + true + + + {{PWA_18_921}} + {{B64URL_40_922}} + sig={{B64URL_43_923}} + + diff --git a/crates/siloscan-core/tests/corpus/tree/xml/NuGet.Config b/crates/siloscan-core/tests/corpus/tree/xml/NuGet.Config new file mode 100644 index 0000000..a143adf --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/xml/NuGet.Config @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/siloscan-core/tests/corpus/tree/xml/Payments.Api.csproj b/crates/siloscan-core/tests/corpus/tree/xml/Payments.Api.csproj new file mode 100644 index 0000000..9ec33b8 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/xml/Payments.Api.csproj @@ -0,0 +1,18 @@ + + + net8.0 + Contoso.Payments.Api + 2.4.1 + true + contoso.snk + + + https://build:{{PWA_20_913}}@pkgs.corp.example/nuget/v3/index.json + {{B64URL_46_914}} + {{PWEQ_16_915}} + + + + + + diff --git a/crates/siloscan-core/tests/corpus/tree/xml/ivysettings.xml b/crates/siloscan-core/tests/corpus/tree/xml/ivysettings.xml new file mode 100644 index 0000000..d7423db --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/xml/ivysettings.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/crates/siloscan-core/tests/corpus/tree/xml/settings.xml b/crates/siloscan-core/tests/corpus/tree/xml/settings.xml new file mode 100644 index 0000000..f5eb549 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/xml/settings.xml @@ -0,0 +1,38 @@ + + + + + corp-releases + deploy + {{PWA_20_918}} + + + corp-snapshots + deploy + {COQLCE6DU6GtcS5P=} + + + docker-registry + svc-docker + {{PWEQ_20_919}} + + + ossrh + contoso-bot + ${env.OSSRH_PASSWORD} + + + + + corp + true + http + proxy.corp.example + 3128 + proxyuser + {{PWP_12_920}} + + + diff --git a/crates/siloscan-core/tests/corpus/tree/xml/web.config b/crates/siloscan-core/tests/corpus/tree/xml/web.config new file mode 100644 index 0000000..d5d5395 --- /dev/null +++ b/crates/siloscan-core/tests/corpus/tree/xml/web.config @@ -0,0 +1,28 @@ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/siloscan-core/tests/detection_corpus.rs b/crates/siloscan-core/tests/detection_corpus.rs index 0d60f86..f2e6137 100644 --- a/crates/siloscan-core/tests/detection_corpus.rs +++ b/crates/siloscan-core/tests/detection_corpus.rs @@ -4,8 +4,8 @@ //! source files; `tests/corpus/manifest.tsv` says, for every line that is a //! test case, whether the pack must report it and under which rule. This file //! materializes the corpus, scans it with the default pack, and turns the -//! result into two numbers: recall over the positives and a precision proxy -//! over the negatives. +//! result into per-family recall over the positives and one global precision +//! proxy over the negatives. //! //! No file in the repository spells a complete credential. Every credential is //! assembled here at run time: vendor prefixes come from `concat!` halves, and @@ -29,10 +29,43 @@ use siloscan_core::default_pack::default_rules; use siloscan_core::engines::secret; use siloscan_core::rules::{CompiledRule, load_str}; -/// Fraction of positives the pack must report. Chosen above what the 1.4.1 -/// pack scores: the generic rules were tuned in one pass against no corpus, -/// and the gap between this floor and what they measure is the work. -const RECALL_FLOOR: f64 = 0.95; +/// Fraction of positives the pack must report, per corpus family. A family is +/// the first path segment under `tree/`; files sitting directly in `tree/` +/// form the family `core`. +/// +/// Every floor is the MEASURED value at the time its family landed - +/// descriptive, not aspirational. A family whose positives contract known +/// gaps (the ticket #44 placeholder-vocabulary URLs, the `=[^=]` markup +/// allowlist of #51, the Kubernetes `data:` decoding rule arriving with #53) +/// starts low on purpose: the floor records where the pack stands, and the +/// ticket that closes the gap raises the floor in the same commit that moves +/// the number. What a floor forbids is regression below what was measured. +const RECALL_FLOORS: &[(&str, f64)] = &[ + ("core", 0.9880), // measured 166/168: the netrc and word-password misses + ("k8s", 0.8333), // measured 15/18: `data:` decoding waits on #52/#53 + ("keys", 1.0), // measured 4/4: every private-key format reported + ("urls", 0.4782), // measured 11/23: the #44 placeholder-vocabulary gap + ("xml", 0.5925), // measured 16/27: the #51 `=[^=]` gap and unlisted names +]; + +fn recall_floor(family: &str) -> f64 { + RECALL_FLOORS + .iter() + .find(|(name, _)| *name == family) + .map(|(_, floor)| *floor) + .unwrap_or_else(|| { + panic!("family {family} carries positives but has no entry in RECALL_FLOORS") + }) +} + +/// The corpus family a manifest path belongs to: its first directory under +/// `tree/`, or `core` for files sitting directly in `tree/`. +fn family(path: &str) -> &str { + match path.split_once('/') { + Some((first, _)) => first, + None => "core", + } +} /// Fraction of negatives the pack must leave alone. Every negative in the /// manifest is justified one by one, so a single spurious hit is a defect and @@ -303,6 +336,27 @@ const NON_ASCII: &str = "\u{e1}\u{e9}\u{ed}\u{f3}\u{fa}\u{fc}\u{f1}\u{e7}\u{df}\ /// allowlists match on. A password is not a placeholder because it spells one. const WORDS: [&str; 5] = ["Passw0rd", "S3cretW", "T0kenB", "MyPassword", "Secr3tWord"]; +/// Passwords built entirely from the placeholder vocabulary of the +/// credentialed-url anchored allowlist - the ticket #44 shape. `VOCABPW`'s +/// PARAM is a 1-based index into this table, not a length; the entry is +/// returned verbatim, so the value is byte-identical on every platform. Every +/// entry is at least six characters, and every one except `SuperSecretKey` +/// (Shannon entropy 2.95, under the rule's 3.0 floor) sits above the floor. +const VOCAB_PASSWORDS: [&str; 12] = [ + "AdminPassword", + "SuperSecretKey", + "MySecretToken", + "hunter2Passphrase", + "RootAdminSecret", + "MyDatabasePassword", + "TopSecretCreds", + "SecretTokenValue", + "DbUserPassword", + "LetmeinAdminKey", + "Super-Secret-Passphrase", + "Admin.Password", +]; + // Vendor prefixes, split so no line in this file spells a credential format. const AWS_KEY_ID_PREFIX: &str = concat!("AK", "IA"); const GITHUB_PAT_PREFIX: &str = concat!("gh", "p_"); @@ -352,6 +406,18 @@ fn credential(name: &str) -> Option { "PWAT" => infix(&mut rng, param, '@'), "PWPCT" => percent_encoded(&mut rng, param), "WORDPW" => word_password(&mut rng, param, name), + "VOCABPW" => VOCAB_PASSWORDS[param - 1].to_string(), + "PWEQ" => eq_probe_password(&mut rng, param), + "B64PAD" => padded_base64(&mut rng, param), + "B64PW" => base64_std(password(&mut rng, param, "").as_bytes()), + "B64CREDURL" => base64_std( + format!( + "postgres://app:{}@db.internal:5432/app", + password(&mut rng, param, "") + ) + .as_bytes(), + ), + "DOCKERCFG" => docker_config_json(&mut rng, param), "B64" => rng.take(BASE64, param), "B64URL" => rng.take(BASE64URL, param), "HEX" => rng.take(HEX, param), @@ -491,6 +557,92 @@ fn word_password(rng: &mut Rng, length: usize, name: &str) -> String { format!("{word}{}", password(rng, tail, "")) } +/// A password of `length` probing the `=[^=]` allowlist (ticket #51): the +/// first character is a digit, exactly one literal `=` sits at an interior +/// position with a digit on either side, and the rest is mixed-case +/// alphanumeric with at least one lowercase letter and one uppercase letter. +/// Starting with a digit and flanking the `=` with digits defeats the +/// word-shaped-identifier-assignment allowlists, so the only pattern between +/// the value and a report is `=[^=]` itself. +fn eq_probe_password(rng: &mut Rng, length: usize) -> String { + assert!(length >= 8, "an eq-probe password needs room, got {length}"); + let mut value: Vec = rng.take(ALNUM, length).chars().collect(); + value[0] = rng.take(DIGIT, 1).chars().next().expect("one digit"); + // Interior with room on both sides: never first, never last. + let eq = 2 + rng.below(length - 4); + value[eq - 1] = rng.take(DIGIT, 1).chars().next().expect("one digit"); + value[eq] = '='; + value[eq + 1] = rng.take(DIGIT, 1).chars().next().expect("one digit"); + let mut slots: Vec = (1..length) + .filter(|slot| *slot + 1 < eq || *slot > eq + 1) + .collect(); + let lower = slots.remove(rng.below(slots.len())); + value[lower] = rng.take(LOWER, 1).chars().next().expect("one lower"); + let upper = slots.remove(rng.below(slots.len())); + value[upper] = rng.take(UPPER, 1).chars().next().expect("one upper"); + value.into_iter().collect() +} + +/// `length` characters of the standard base64 alphabet - guaranteed to carry +/// a digit and a `+` or `/`, so no letters-only or word-shape allowlist can +/// stand down on the draw - followed by literal `==` padding. `length` is +/// always congruent to 2 mod 4, so the padded total is a real base64 length. +/// The trailing `==` must survive `=[^=]` because no non-`=` follows either. +fn padded_base64(rng: &mut Rng, length: usize) -> String { + assert_eq!(length % 4, 2, "==-padded base64 bodies are 2 mod 4"); + let mut value: Vec = rng.take(BASE64, length).chars().collect(); + let digit = rng.below(length); + value[digit] = rng.take(DIGIT, 1).chars().next().expect("one digit"); + let mut symbol = rng.below(length); + if symbol == digit { + symbol = (symbol + 1) % length; + } + value[symbol] = if rng.below(2) == 0 { '+' } else { '/' }; + let mut out: String = value.into_iter().collect(); + out.push_str("=="); + out +} + +/// The compact JSON `kubectl create secret docker-registry` emits into +/// `.dockerconfigjson`, base64-encoded: two encoding layers, with the inner +/// `auth` field base64 of `user:password`. Only the outer encoding is on the +/// corpus line; the password itself never appears un-encoded anywhere. +fn docker_config_json(rng: &mut Rng, length: usize) -> String { + let value = password(rng, length, ""); + let auth = base64_std(format!("ci-deploy:{value}").as_bytes()); + let json = format!( + concat!( + "{{\"auths\":{{\"registry.internal:5000\":{{\"username\":\"ci-deploy\",", + "\"password\":\"{}\",\"auth\":\"{}\"}}}}}}" + ), + value, auth + ); + base64_std(json.as_bytes()) +} + +/// RFC 4648 standard-alphabet base64 with `=` padding. Implemented here so +/// the test needs no dependency for twelve lines of arithmetic. +fn base64_std(bytes: &[u8]) -> String { + const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); + for chunk in bytes.chunks(3) { + let b = [ + chunk[0], + *chunk.get(1).unwrap_or(&0), + *chunk.get(2).unwrap_or(&0), + ]; + let group = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + for slot in 0..4 { + if slot <= chunk.len() { + out.push(ALPHABET[(group >> (18 - 6 * slot)) as usize & 0x3f] as char); + } else { + out.push('='); + } + } + } + out +} + // -------------------------------------------------------------- measurement struct Measurement { @@ -629,6 +781,36 @@ impl Measurement { .count() } + /// Per family: (positives, positives reported). Families with no + /// positives - the pure-noise families exist to hold precision - do not + /// appear; they have no recall to measure. + fn recall_by_family(&self) -> BTreeMap<&str, (usize, usize)> { + let mut stats: BTreeMap<&str, (usize, usize)> = BTreeMap::new(); + for row in &self.rows { + if !row.expect.is_positive() { + continue; + } + let entry = stats.entry(family(&row.path)).or_insert((0, 0)); + entry.0 += 1; + if row.expect.satisfied_by(self.hits(&row.path, row.line)) { + entry.1 += 1; + } + } + stats + } + + fn family_table(&self) -> String { + let mut out = String::from("FAMILY RECALL REPORTED POSITIVES FLOOR\n"); + for (name, (positives, reported)) in self.recall_by_family() { + let recall = reported as f64 / positives as f64; + let floor = recall_floor(name); + out.push_str(&format!( + "{name:<20} {recall:<8.4} {reported:<9} {positives:<10} {floor:.4}\n" + )); + } + out + } + fn negatives(&self) -> usize { self.rows.len() - self.positives() } @@ -652,7 +834,7 @@ impl Measurement { self.negatives() )); out.push_str(&format!( - "RECALL {:.4} ({} of {} positives reported, floor {RECALL_FLOOR:.2})\n", + "RECALL {:.4} ({} of {} positives reported; floors are per family)\n", self.recall(), self.positives() - self.misses().len(), self.positives() @@ -663,6 +845,7 @@ impl Measurement { self.negatives() - self.spurious().len(), self.negatives() )); + out.push_str(&self.family_table()); for line in self.misses() { out.push_str(&line); out.push('\n'); @@ -778,14 +961,39 @@ fn no_corpus_file_spells_a_credential() { } } +/// Recall is measured and held per family, because the families are not +/// equal: `core` measures shapes the pack was tuned on, while `urls`, `xml` +/// and `k8s` deliberately contract known gaps whose floors start where the +/// gap leaves them. One global number would let a regression in a strong +/// family hide behind a fix in a weak one. #[test] -fn detection_recall_meets_its_floor() { +fn detection_recall_meets_its_floor_per_family() { let measurement = measure(); - let recall = measurement.recall(); + let stats = measurement.recall_by_family(); + println!("{}", measurement.family_table()); + + for (name, _) in RECALL_FLOORS { + assert!( + stats.contains_key(name), + "RECALL_FLOORS names {name}, which has no positives in the manifest" + ); + } + let mut failures = Vec::new(); + for (name, (positives, reported)) in &stats { + let recall = *reported as f64 / *positives as f64; + let floor = recall_floor(name); + if recall < floor { + failures.push(format!( + "family {name}: recall {recall:.4} is below its floor {floor:.4} \ + ({reported} of {positives} positives reported)" + )); + } + } assert!( - recall >= RECALL_FLOOR, - "{}\nrecall {recall:.4} is below the floor {RECALL_FLOOR:.2}", - measurement.report() + failures.is_empty(), + "{}\n{}", + measurement.report(), + failures.join("\n") ); } diff --git a/scripts/parity_gitleaks.py b/scripts/parity_gitleaks.py new file mode 100755 index 0000000..15eb76e --- /dev/null +++ b/scripts/parity_gitleaks.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""Parity harness: siloscan secret rules vs gitleaks (issue #50). + +Downloads a checksum-pinned gitleaks release binary, runs both scanners over +one or more target directories, joins findings by (file, line), and writes one +TSV of deltas per target with columns: + + file line gitleaks_rule siloscan_rule bucket + +Buckets: + siloscan-missing gitleaks fired on the line, siloscan did not + siloscan-extra siloscan fired on the line, gitleaks did not + both-different-rule both fired, no rule name in common after + stripping siloscan's "secrets." prefix + agree both fired with a matching rule name; counted in the + summary, not listed in the TSV + +Manual tooling. Never wired into PR CI. Stdlib only. + +Usage: + parity_gitleaks.py TARGET [TARGET ...] [--siloscan PATH] [--out-dir DIR] + [--cache-dir DIR] [--rule-prefix PREFIX] +""" + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request + +GITLEAKS_VERSION = "8.30.1" +# sha256 of gitleaks_8.30.1_linux_x64.tar.gz, verified 2026-08-07 against +# gitleaks_8.30.1_checksums.txt on the official GitHub release. +GITLEAKS_SHA256 = "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" +GITLEAKS_URL = ( + "https://github.com/gitleaks/gitleaks/releases/download/" + "v{v}/gitleaks_{v}_linux_x64.tar.gz".format(v=GITLEAKS_VERSION) +) + +DEFAULT_CACHE_DIR = os.path.join( + os.path.expanduser("~"), ".cache", "siloscan-parity" +) + +BUCKETS = ("siloscan-missing", "siloscan-extra", "both-different-rule", "agree") + + +def die(msg): + print("error: %s" % msg, file=sys.stderr) + sys.exit(2) + + +def ensure_gitleaks(cache_dir): + """Return the path to a verified gitleaks binary, downloading if needed.""" + bin_dir = os.path.join(cache_dir, "gitleaks-%s" % GITLEAKS_VERSION) + bin_path = os.path.join(bin_dir, "gitleaks") + if os.access(bin_path, os.X_OK): + return bin_path + + os.makedirs(bin_dir, exist_ok=True) + print("downloading gitleaks v%s ..." % GITLEAKS_VERSION, file=sys.stderr) + fd, archive = tempfile.mkstemp(suffix=".tar.gz", dir=bin_dir) + try: + with os.fdopen(fd, "wb") as out, urllib.request.urlopen( + GITLEAKS_URL + ) as resp: + shutil.copyfileobj(resp, out) + + digest = hashlib.sha256() + with open(archive, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + actual = digest.hexdigest() + if actual != GITLEAKS_SHA256: + die( + "gitleaks archive checksum mismatch:\n" + " expected %s\n actual %s\n" + "refusing to run an unverified binary" + % (GITLEAKS_SHA256, actual) + ) + + with tarfile.open(archive, "r:gz") as tar: + member = tar.getmember("gitleaks") + if not member.isfile(): + die("gitleaks archive entry is not a regular file") + src = tar.extractfile(member) + tmp_bin = bin_path + ".tmp" + with open(tmp_bin, "wb") as dst: + shutil.copyfileobj(src, dst) + os.chmod(tmp_bin, 0o755) + os.replace(tmp_bin, bin_path) + finally: + if os.path.exists(archive): + os.unlink(archive) + return bin_path + + +def run_gitleaks(bin_path, target): + """Run gitleaks detect --no-git over target. + + Returns a list of (relative_path, line, rule_id). + """ + fd, report = tempfile.mkstemp(suffix=".json") + os.close(fd) + try: + proc = subprocess.run( + [ + bin_path, + "detect", + "--no-git", + "--source", + target, + "--report-format", + "json", + "--report-path", + report, + "--exit-code", + "0", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + if proc.returncode != 0: + die( + "gitleaks failed on %s (exit %d):\n%s" + % (target, proc.returncode, proc.stderr.strip()) + ) + with open(report, encoding="utf-8") as f: + findings = json.load(f) + finally: + os.unlink(report) + + rows = [] + for f in findings: + path = f["File"] + rel = os.path.relpath(path, target) + if rel.startswith(".."): + rel = path + rows.append((rel, int(f["StartLine"]), f["RuleID"])) + return rows + + +def run_siloscan(bin_path, target, rule_prefix): + """Run siloscan --format json over target. + + Returns a list of (relative_path, line, rule_id), restricted to rules + matching rule_prefix (gitleaks parity only makes sense for secret rules). + """ + proc = subprocess.run( + [bin_path, target, "--format", "json"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + # 0 = clean, 1 = findings; anything else is a scan failure. + if proc.returncode not in (0, 1): + die( + "siloscan failed on %s (exit %d):\n%s" + % (target, proc.returncode, proc.stderr.strip()) + ) + report = json.loads(proc.stdout) + rows = [] + for f in report.get("findings", []): + if not f["rule_id"].startswith(rule_prefix): + continue + rows.append((f["path"], int(f["line"]), f["rule_id"])) + return rows + + +def strip_prefix(rule, prefix): + return rule[len(prefix):] if rule.startswith(prefix) else rule + + +def join_findings(gitleaks_rows, siloscan_rows, rule_prefix): + """Join by (file, line). Returns (delta_rows, bucket_counts). + + delta_rows are (file, line, gitleaks_rule, siloscan_rule, bucket) with + multiple rules on one line joined by ";". Agree keys are counted only. + """ + by_key_gl = {} + for path, line, rule in gitleaks_rows: + by_key_gl.setdefault((path, line), set()).add(rule) + by_key_ss = {} + for path, line, rule in siloscan_rows: + by_key_ss.setdefault((path, line), set()).add(rule) + + counts = {b: 0 for b in BUCKETS} + deltas = [] + for key in sorted(set(by_key_gl) | set(by_key_ss)): + gl = sorted(by_key_gl.get(key, ())) + ss = sorted(by_key_ss.get(key, ())) + if gl and not ss: + bucket = "siloscan-missing" + elif ss and not gl: + bucket = "siloscan-extra" + else: + ss_normed = {strip_prefix(r, rule_prefix) for r in ss} + bucket = "agree" if ss_normed & set(gl) else "both-different-rule" + counts[bucket] += 1 + if bucket != "agree": + deltas.append( + (key[0], key[1], ";".join(gl) or "-", ";".join(ss) or "-", bucket) + ) + return deltas, counts + + +def write_tsv(path, deltas): + with open(path, "w", encoding="utf-8") as out: + out.write("file\tline\tgitleaks_rule\tsiloscan_rule\tbucket\n") + for row in deltas: + out.write("%s\t%d\t%s\t%s\t%s\n" % row) + + +def tsv_name(target): + base = os.path.basename(os.path.normpath(target)) or "root" + return re.sub(r"[^A-Za-z0-9._-]+", "_", base) + ".tsv" + + +def main(): + ap = argparse.ArgumentParser( + description="Compare siloscan secret findings against gitleaks " + "v%s over one or more directories." % GITLEAKS_VERSION + ) + ap.add_argument("targets", nargs="+", help="directories to scan") + ap.add_argument( + "--siloscan", + default="target/release/siloscan", + help="path to the siloscan binary (default: %(default)s)", + ) + ap.add_argument( + "--out-dir", + default="parity-out", + help="directory for the per-target delta TSVs (default: %(default)s)", + ) + ap.add_argument( + "--cache-dir", + default=DEFAULT_CACHE_DIR, + help="cache directory for the gitleaks binary (default: %(default)s)", + ) + ap.add_argument( + "--rule-prefix", + default="secrets.", + help="siloscan rules compared, by id prefix; the prefix is stripped " + "before rule-name matching (default: %(default)s)", + ) + args = ap.parse_args() + + siloscan = os.path.abspath(args.siloscan) + if not os.access(siloscan, os.X_OK): + die("siloscan binary not found or not executable: %s" % siloscan) + for target in args.targets: + if not os.path.isdir(target): + die("target is not a directory: %s" % target) + + gitleaks = ensure_gitleaks(os.path.abspath(args.cache_dir)) + os.makedirs(args.out_dir, exist_ok=True) + + print( + "target\tsiloscan-missing\tsiloscan-extra\tboth-different-rule\t" + "agree\ttsv" + ) + for target in args.targets: + target = os.path.abspath(target) + gl_rows = run_gitleaks(gitleaks, target) + ss_rows = run_siloscan(siloscan, target, args.rule_prefix) + deltas, counts = join_findings(gl_rows, ss_rows, args.rule_prefix) + tsv_path = os.path.join(args.out_dir, tsv_name(target)) + write_tsv(tsv_path, deltas) + print( + "%s\t%d\t%d\t%d\t%d\t%s" + % ( + target, + counts["siloscan-missing"], + counts["siloscan-extra"], + counts["both-different-rule"], + counts["agree"], + tsv_path, + ) + ) + + +if __name__ == "__main__": + main() From 0855990786d88f4485fd496a0911059d0040afdd Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 7 Aug 2026 09:11:30 +0000 Subject: [PATCH 2/2] fix: extract the gitleaks binary owner-only --- scripts/parity_gitleaks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/parity_gitleaks.py b/scripts/parity_gitleaks.py index 15eb76e..a09073d 100755 --- a/scripts/parity_gitleaks.py +++ b/scripts/parity_gitleaks.py @@ -92,7 +92,9 @@ def ensure_gitleaks(cache_dir): tmp_bin = bin_path + ".tmp" with open(tmp_bin, "wb") as dst: shutil.copyfileobj(src, dst) - os.chmod(tmp_bin, 0o755) + # Owner-only: this is a private cache, and nothing but this script + # needs to read or run the binary. + os.chmod(tmp_bin, 0o700) os.replace(tmp_bin, bin_path) finally: if os.path.exists(archive):