diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d52a51b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Every text file checks out with LF line endings on every platform. The rule +# packs are embedded into the binary with include_str! and located by exact +# byte sequences; a CRLF checkout (git's autocrlf default on Windows) changed +# those bytes and left every Windows build panicking on its first scan. +* text=auto eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af4ad34..06c8c53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,46 @@ jobs: - name: Test run: cargo test --locked --workspace + windows: + name: Windows (build, test, smoke) + runs-on: windows-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master, 2026-07-16 + with: + toolchain: stable + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - name: Build + run: cargo build --locked --workspace + - name: Test + run: cargo test --locked --workspace + # Run the compiled binary against a generated credential. Unit tests + # cover the Windows cache internals; this proves a real scan on a real + # Windows filesystem finds a secret and writes its cache, including the + # NTFS alternate-data-stream salt no other CI job ever executes. + # The token is generated at runtime so no credential-shaped literal + # lives in this file. + - name: Smoke-test the binary + shell: bash + run: | + set -euo pipefail + smoke="$RUNNER_TEMP/smoke" + mkdir -p "$smoke" + python -c "import secrets; print('API_TOKEN=' + secrets.token_hex(20))" > "$smoke/config.env" + set +e + ./target/debug/siloscan.exe "$smoke" --format json > "$RUNNER_TEMP/out.json" + status=$? + set -e + echo "exit status: $status" + test "$status" -le 1 + python - "$RUNNER_TEMP/out.json" <<'EOF' + import json, sys + findings = json.load(open(sys.argv[1])).get("findings", []) + for f in findings: + print(f["rule_id"], f["path"], f["line"]) + assert findings, "the scan found nothing" + EOF + msrv: name: MSRV (1.96) runs-on: ubuntu-latest diff --git a/crates/siloscan-core/src/cache.rs b/crates/siloscan-core/src/cache.rs index 3b99b65..948b91d 100644 --- a/crates/siloscan-core/src/cache.rs +++ b/crates/siloscan-core/src/cache.rs @@ -623,20 +623,19 @@ impl Cache { /// one is the cache this run writes, which is the part that has to be out of /// the walk for a warm run to match a cold one. /// - /// The comparison is made on canonical paths, so `.`, `..` and symlinks in - /// either argument do not decide it, and the result is re-spelled against - /// `scan_root` as given because that is the spelling the walk produces. A - /// directory that does not exist yet canonicalizes to itself and is simply - /// not under the root; it is also not in the walk, so there is nothing to - /// exclude until the run after it appears. + /// The comparison is made on [`resolved`] paths, so `.`, `..`, symlinks + /// and Windows short names in either argument do not decide it, and the + /// result is re-spelled against `scan_root` as given because that is the + /// spelling the walk produces. A directory that does not exist yet + /// resolves through its nearest existing ancestor, so the exclusion holds + /// from the first run, not the one after the directory appears. pub fn exclusion_under(&self, scan_root: &Path) -> Option { - let canonical = |path: &Path| fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); - let base = canonical(scan_root); + let base = resolved(scan_root); [self.owned.as_deref(), self.root.as_deref()] .into_iter() .flatten() .find_map(|candidate| { - let rel = canonical(candidate).strip_prefix(&base).ok()?.to_path_buf(); + let rel = resolved(candidate).strip_prefix(&base).ok()?.to_path_buf(); // The candidate is the scan root itself. Not an exclusion, and // not this candidate's turn to answer. if rel.as_os_str().is_empty() { @@ -1165,6 +1164,28 @@ fn probe_uid(dir: &Path) -> Option { uid } +/// `fs::canonicalize`, extended to paths that do not exist yet: the nearest +/// existing ancestor is canonicalized and the remaining components are +/// appended as given. +/// +/// [`Cache::exclusion_under`] compares a cache directory that may not have +/// been created against a scan root that exists, and the two must land in one +/// spelling for `strip_prefix` to mean anything. Falling back to the path as +/// given is not enough on Windows: canonical paths there are verbatim long +/// form (`\\?\C:\Users\runneradmin\...`) while the environment may hand out +/// 8.3 short names (`RUNNER~1`), so a raw fallback never shares a prefix with +/// a canonicalized base and the exclusion silently vanished - a scan with +/// `--cache-dir` inside the root walked its own cache. +fn resolved(path: &Path) -> PathBuf { + if let Ok(canonical) = fs::canonicalize(path) { + return canonical; + } + match (path.parent(), path.file_name()) { + (Some(parent), Some(name)) if !parent.as_os_str().is_empty() => resolved(parent).join(name), + _ => path.to_path_buf(), + } +} + /// The directory under `base` that holds `scan_root`'s entries: a hash of the /// root's canonical path, so every scan root gets its own cache, its own salt /// and its own prune stamp. @@ -2618,8 +2639,10 @@ mod tests { // Now with the salt as well, as a wholesale copy of a cache directory // would arrive. A fresh instance is used because the first one has - // already resolved this directory as saltless. - fs::copy(salt_path(&theirs), salt_path(&mine)).unwrap(); + // already resolved this directory as saltless. Read and write rather + // than `fs::copy`: on Windows the salt lives in an NTFS alternate data + // stream, and `fs::copy` cannot copy attributes onto a stream. + fs::write(salt_path(&mine), fs::read(salt_path(&theirs)).unwrap()).unwrap(); set_owner_only(&salt_path(&mine)); let mine = mine_fx.open(); assert!( diff --git a/crates/siloscan-core/src/config.rs b/crates/siloscan-core/src/config.rs index 4b92da1..dd7abac 100644 --- a/crates/siloscan-core/src/config.rs +++ b/crates/siloscan-core/src/config.rs @@ -33,7 +33,7 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; -use globset::{Glob, GlobSet, GlobSetBuilder}; +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; use serde::{Deserialize, Serialize}; pub const CONFIG_NAME: &str = "siloscan.toml"; @@ -726,7 +726,14 @@ impl Config { for (name, patterns) in &self.silos { let mut builder = GlobSetBuilder::new(); for pattern in patterns { - let glob = Glob::new(pattern) + // Globs match the forward-slash relative paths every report + // uses, on every platform, so their syntax cannot be + // platform-dependent either: `\` escapes, everywhere, rather + // than globset's Windows default of treating it as a + // separator. + let glob = GlobBuilder::new(pattern) + .backslash_escape(true) + .build() .map_err(|e| format!("silo {name}: invalid glob {pattern:?}: {e}"))?; builder.add(glob); } diff --git a/crates/siloscan-core/src/default_pack.rs b/crates/siloscan-core/src/default_pack.rs index eed4217..88d980c 100644 --- a/crates/siloscan-core/src/default_pack.rs +++ b/crates/siloscan-core/src/default_pack.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::sync::OnceLock; /// Translated from the gitleaks default config (v8.30.1) by @@ -23,16 +24,34 @@ const RULES_HEADER: &str = "\nrules:\n"; pub fn default_rules() -> &'static str { static PACK: OnceLock = OnceLock::new(); PACK.get_or_init(|| { - let mut pack = String::with_capacity(GITLEAKS_DOCUMENT.len() + GENERIC_DOCUMENT.len()); - pack.push_str(GITLEAKS_DOCUMENT); + let gitleaks = normalize(GITLEAKS_DOCUMENT); + let generic = normalize(GENERIC_DOCUMENT); + let mut pack = String::with_capacity(gitleaks.len() + generic.len()); + pack.push_str(&gitleaks); if !pack.ends_with('\n') { pack.push('\n'); } - pack.push_str(rule_items(GENERIC_DOCUMENT)); + pack.push_str(rule_items(&generic)); pack }) } +/// The document with CRLF line endings replaced by LF. +/// +/// The documents are committed with LF and `.gitattributes` keeps them that +/// way, but a checkout that converted them anyway (git's `autocrlf` default on +/// Windows, before the attributes file existed) reaches `include_str!` with +/// CRLF and made [`rule_items`] panic on every scan. The pack's contents are +/// located by exact byte sequences, so the bytes are fixed here rather than +/// assumed. +fn normalize(document: &str) -> Cow<'_, str> { + if document.contains('\r') { + Cow::Owned(document.replace("\r\n", "\n")) + } else { + Cow::Borrowed(document) + } +} + /// The sequence items of a rule document, without its header. /// /// Panics when the document has no `rules:` list. Returning nothing instead diff --git a/crates/siloscan-core/src/rules.rs b/crates/siloscan-core/src/rules.rs index 4b98266..6f5205e 100644 --- a/crates/siloscan-core/src/rules.rs +++ b/crates/siloscan-core/src/rules.rs @@ -5,7 +5,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock}; -use globset::{Glob, GlobSet, GlobSetBuilder}; +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; use regex::Regex; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -958,10 +958,16 @@ fn compile_globs( let mut builder = GlobSetBuilder::new(); for pattern in &patterns { - let glob = Glob::new(pattern).map_err(|e| LoadError::BadGlob { - origin: origin.to_string(), - detail: e.to_string(), - })?; + // Matched against forward-slash relative paths on every platform, so + // the syntax is pinned too: `\` escapes everywhere, not globset's + // Windows default of treating it as a separator. + let glob = GlobBuilder::new(pattern) + .backslash_escape(true) + .build() + .map_err(|e| LoadError::BadGlob { + origin: origin.to_string(), + detail: e.to_string(), + })?; builder.add(glob); } diff --git a/crates/siloscan-core/src/walk.rs b/crates/siloscan-core/src/walk.rs index e09aaec..7f45e75 100644 --- a/crates/siloscan-core/src/walk.rs +++ b/crates/siloscan-core/src/walk.rs @@ -1413,9 +1413,15 @@ mod tests { /// above with `GIT_CONFIG_GLOBAL` pointed at it. fn spawn_global_ignore_child() { let dir = tempfile::tempdir().unwrap(); - let global_ignore = dir.path().join("global_ignore"); + // Canonical spelling, because the `ignore` crate expands every `~` in + // the configured path to the home directory - not only a leading one + // (`expand_tilde` in its gitignore.rs). Windows hands out temp paths + // with 8.3 short names (`RUNNER~1`), which that expansion corrupts + // into a path that opens nothing; the canonical form has no `~`. + let base = fs::canonicalize(dir.path()).unwrap(); + let global_ignore = base.join("global_ignore"); fs::write(&global_ignore, "secret.txt\n").unwrap(); - let gitconfig = dir.path().join("gitconfig"); + let gitconfig = base.join("gitconfig"); fs::write( &gitconfig, format!("[core]\nexcludesFile = {}\n", global_ignore.display()), diff --git a/crates/siloscan/tests/cli.rs b/crates/siloscan/tests/cli.rs index 9234003..6e1b2b9 100644 --- a/crates/siloscan/tests/cli.rs +++ b/crates/siloscan/tests/cli.rs @@ -105,6 +105,11 @@ const COVERAGE_RULE: &str = concat!( /// line the cursor sits on and `\r` returns to its start, so a report line /// written raw overwrites itself with whatever follows. A repository reaches /// this by pointing `rules` in its own `siloscan.toml` at a rule file it ships. +/// +/// The escape fixtures are unix-only: the second vector is a file whose name +/// carries the escape byte, and Windows refuses control characters in file +/// names, so neither the fixture nor the attack can exist there. +#[cfg(unix)] const ESC_MESSAGE_RULE: &str = concat!( "version: 1\n", "rules:\n", @@ -115,15 +120,17 @@ const ESC_MESSAGE_RULE: &str = concat!( " pattern: 'needle'\n", ); -/// The second vector, and the one needing no config at all: a file name. Any -/// byte but `/` and NUL is legal in one, so the escape arrives through the -/// walker without the repository configuring anything. +/// The second vector, and the one needing no config at all: a file name. On +/// unix any byte but `/` and NUL is legal in one, so the escape arrives +/// through the walker without the repository configuring anything. +#[cfg(unix)] const ESC_PATH: &str = "ev\u{1b}[2Kil.js"; /// The fingerprint `siloscan` 1.1.1 produced for the finding in [`ESC_PATH`], /// recorded before the terminal escaping existed. Escaping is a rendering /// concern, so this value may not move: a baseline written by an older release /// has to keep covering the same finding. +#[cfg(unix)] const ESC_FINGERPRINT: &str = "a5421000a7dd76b51bd5b139caaf6746668891ada868f7b47d0437039dea245a"; const SILO_CONFIG: &str = concat!( @@ -1066,6 +1073,7 @@ fn a_warm_run_over_a_hidden_tree_is_byte_identical_to_the_cold_one() { /// One file whose name carries an escape sequence, matched by a rule whose /// message carries another: the two ways a scanned repository reaches the /// operator's terminal. +#[cfg(unix)] fn esc_fixture() -> (TempDir, TempDir) { ( rules_dir(ESC_MESSAGE_RULE), @@ -1076,6 +1084,7 @@ fn esc_fixture() -> (TempDir, TempDir) { /// Written raw, `ESC [ 2 K` followed by a carriage return erases the report /// line and rewrites it, so a repository holding a live credential could render /// as a clean scan. The bytes are rendered instead, and the finding still shows. +#[cfg(unix)] #[test] fn no_escape_byte_from_a_scanned_repository_reaches_the_terminal() { let (rules, src) = esc_fixture(); @@ -1105,6 +1114,7 @@ fn no_escape_byte_from_a_scanned_repository_reaches_the_terminal() { /// Escaping is a rendering concern and stops at the human format: the JSON /// report still carries the bytes, and the fingerprint an older release wrote /// into a baseline still identifies the same finding. +#[cfg(unix)] #[test] fn escaping_leaves_the_json_report_and_its_fingerprints_where_they_were() { let (rules, src) = esc_fixture(); @@ -1270,8 +1280,10 @@ fn help_documents_only_the_forms_that_work() { assert_eq!(output.status.code(), Some(0)); let text = stdout(&output); - assert!(text.contains("siloscan [OPTIONS] [PATH]"), "stdout: {text}"); - assert!(text.contains("siloscan "), "stdout: {text}"); + // The binary name is platform-dependent (`siloscan.exe` on Windows); the + // assertion is about the usage forms, not the name. + assert!(text.contains("[OPTIONS] [PATH]"), "stdout: {text}"); + assert!(text.contains(""), "stdout: {text}"); assert!(!text.contains("[PATH] [COMMAND]"), "stdout: {text}"); }