Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Every text file checks out with LF line endings on every platform. The rule
# packs are embedded into the binary with include_str! and located by exact
# byte sequences; a CRLF checkout (git's autocrlf default on Windows) changed
# those bytes and left every Windows build panicking on its first scan.
* text=auto eol=lf
40 changes: 40 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,46 @@ jobs:
- name: Test
run: cargo test --locked --workspace

windows:
name: Windows (build, test, smoke)
runs-on: windows-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master, 2026-07-16
with:
toolchain: stable
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
- name: Build
run: cargo build --locked --workspace
- name: Test
run: cargo test --locked --workspace
# Run the compiled binary against a generated credential. Unit tests
# cover the Windows cache internals; this proves a real scan on a real
# Windows filesystem finds a secret and writes its cache, including the
# NTFS alternate-data-stream salt no other CI job ever executes.
# The token is generated at runtime so no credential-shaped literal
# lives in this file.
- name: Smoke-test the binary
shell: bash
run: |
set -euo pipefail
smoke="$RUNNER_TEMP/smoke"
mkdir -p "$smoke"
python -c "import secrets; print('API_TOKEN=' + secrets.token_hex(20))" > "$smoke/config.env"
set +e
./target/debug/siloscan.exe "$smoke" --format json > "$RUNNER_TEMP/out.json"
status=$?
set -e
echo "exit status: $status"
test "$status" -le 1
python - "$RUNNER_TEMP/out.json" <<'EOF'
import json, sys
findings = json.load(open(sys.argv[1])).get("findings", [])
for f in findings:
print(f["rule_id"], f["path"], f["line"])
assert findings, "the scan found nothing"
EOF

msrv:
name: MSRV (1.96)
runs-on: ubuntu-latest
Expand Down
45 changes: 34 additions & 11 deletions crates/siloscan-core/src/cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -623,20 +623,19 @@ impl Cache {
/// one is the cache this run writes, which is the part that has to be out of
/// the walk for a warm run to match a cold one.
///
/// The comparison is made on canonical paths, so `.`, `..` and symlinks in
/// either argument do not decide it, and the result is re-spelled against
/// `scan_root` as given because that is the spelling the walk produces. A
/// directory that does not exist yet canonicalizes to itself and is simply
/// not under the root; it is also not in the walk, so there is nothing to
/// exclude until the run after it appears.
/// The comparison is made on [`resolved`] paths, so `.`, `..`, symlinks
/// and Windows short names in either argument do not decide it, and the
/// result is re-spelled against `scan_root` as given because that is the
/// spelling the walk produces. A directory that does not exist yet
/// resolves through its nearest existing ancestor, so the exclusion holds
/// from the first run, not the one after the directory appears.
pub fn exclusion_under(&self, scan_root: &Path) -> Option<PathBuf> {
let canonical = |path: &Path| fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
let base = canonical(scan_root);
let base = resolved(scan_root);
[self.owned.as_deref(), self.root.as_deref()]
.into_iter()
.flatten()
.find_map(|candidate| {
let rel = canonical(candidate).strip_prefix(&base).ok()?.to_path_buf();
let rel = resolved(candidate).strip_prefix(&base).ok()?.to_path_buf();
// The candidate is the scan root itself. Not an exclusion, and
// not this candidate's turn to answer.
if rel.as_os_str().is_empty() {
Expand Down Expand Up @@ -1165,6 +1164,28 @@ fn probe_uid(dir: &Path) -> Option<u32> {
uid
}

/// `fs::canonicalize`, extended to paths that do not exist yet: the nearest
/// existing ancestor is canonicalized and the remaining components are
/// appended as given.
///
/// [`Cache::exclusion_under`] compares a cache directory that may not have
/// been created against a scan root that exists, and the two must land in one
/// spelling for `strip_prefix` to mean anything. Falling back to the path as
/// given is not enough on Windows: canonical paths there are verbatim long
/// form (`\\?\C:\Users\runneradmin\...`) while the environment may hand out
/// 8.3 short names (`RUNNER~1`), so a raw fallback never shares a prefix with
/// a canonicalized base and the exclusion silently vanished - a scan with
/// `--cache-dir` inside the root walked its own cache.
fn resolved(path: &Path) -> PathBuf {
if let Ok(canonical) = fs::canonicalize(path) {
return canonical;
}
match (path.parent(), path.file_name()) {
(Some(parent), Some(name)) if !parent.as_os_str().is_empty() => resolved(parent).join(name),
_ => path.to_path_buf(),
}
}

/// The directory under `base` that holds `scan_root`'s entries: a hash of the
/// root's canonical path, so every scan root gets its own cache, its own salt
/// and its own prune stamp.
Expand Down Expand Up @@ -2618,8 +2639,10 @@ mod tests {

// Now with the salt as well, as a wholesale copy of a cache directory
// would arrive. A fresh instance is used because the first one has
// already resolved this directory as saltless.
fs::copy(salt_path(&theirs), salt_path(&mine)).unwrap();
// already resolved this directory as saltless. Read and write rather
// than `fs::copy`: on Windows the salt lives in an NTFS alternate data
// stream, and `fs::copy` cannot copy attributes onto a stream.
fs::write(salt_path(&mine), fs::read(salt_path(&theirs)).unwrap()).unwrap();
set_owner_only(&salt_path(&mine));
let mine = mine_fx.open();
assert!(
Expand Down
11 changes: 9 additions & 2 deletions crates/siloscan-core/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};

use globset::{Glob, GlobSet, GlobSetBuilder};
use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
use serde::{Deserialize, Serialize};

pub const CONFIG_NAME: &str = "siloscan.toml";
Expand Down Expand Up @@ -726,7 +726,14 @@ impl Config {
for (name, patterns) in &self.silos {
let mut builder = GlobSetBuilder::new();
for pattern in patterns {
let glob = Glob::new(pattern)
// Globs match the forward-slash relative paths every report
// uses, on every platform, so their syntax cannot be
// platform-dependent either: `\` escapes, everywhere, rather
// than globset's Windows default of treating it as a
// separator.
let glob = GlobBuilder::new(pattern)
.backslash_escape(true)
.build()
.map_err(|e| format!("silo {name}: invalid glob {pattern:?}: {e}"))?;
builder.add(glob);
}
Expand Down
25 changes: 22 additions & 3 deletions crates/siloscan-core/src/default_pack.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::sync::OnceLock;

/// Translated from the gitleaks default config (v8.30.1) by
Expand All @@ -23,16 +24,34 @@ const RULES_HEADER: &str = "\nrules:\n";
pub fn default_rules() -> &'static str {
static PACK: OnceLock<String> = OnceLock::new();
PACK.get_or_init(|| {
let mut pack = String::with_capacity(GITLEAKS_DOCUMENT.len() + GENERIC_DOCUMENT.len());
pack.push_str(GITLEAKS_DOCUMENT);
let gitleaks = normalize(GITLEAKS_DOCUMENT);
let generic = normalize(GENERIC_DOCUMENT);
let mut pack = String::with_capacity(gitleaks.len() + generic.len());
pack.push_str(&gitleaks);
if !pack.ends_with('\n') {
pack.push('\n');
}
pack.push_str(rule_items(GENERIC_DOCUMENT));
pack.push_str(rule_items(&generic));
pack
})
}

/// The document with CRLF line endings replaced by LF.
///
/// The documents are committed with LF and `.gitattributes` keeps them that
/// way, but a checkout that converted them anyway (git's `autocrlf` default on
/// Windows, before the attributes file existed) reaches `include_str!` with
/// CRLF and made [`rule_items`] panic on every scan. The pack's contents are
/// located by exact byte sequences, so the bytes are fixed here rather than
/// assumed.
fn normalize(document: &str) -> Cow<'_, str> {
if document.contains('\r') {
Cow::Owned(document.replace("\r\n", "\n"))
} else {
Cow::Borrowed(document)
}
}

/// The sequence items of a rule document, without its header.
///
/// Panics when the document has no `rules:` list. Returning nothing instead
Expand Down
16 changes: 11 additions & 5 deletions crates/siloscan-core/src/rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock};

use globset::{Glob, GlobSet, GlobSetBuilder};
use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
use regex::Regex;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
Expand Down Expand Up @@ -958,10 +958,16 @@ fn compile_globs(

let mut builder = GlobSetBuilder::new();
for pattern in &patterns {
let glob = Glob::new(pattern).map_err(|e| LoadError::BadGlob {
origin: origin.to_string(),
detail: e.to_string(),
})?;
// Matched against forward-slash relative paths on every platform, so
// the syntax is pinned too: `\` escapes everywhere, not globset's
// Windows default of treating it as a separator.
let glob = GlobBuilder::new(pattern)
.backslash_escape(true)
.build()
.map_err(|e| LoadError::BadGlob {
origin: origin.to_string(),
detail: e.to_string(),
})?;
builder.add(glob);
}

Expand Down
10 changes: 8 additions & 2 deletions crates/siloscan-core/src/walk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1413,9 +1413,15 @@ mod tests {
/// above with `GIT_CONFIG_GLOBAL` pointed at it.
fn spawn_global_ignore_child() {
let dir = tempfile::tempdir().unwrap();
let global_ignore = dir.path().join("global_ignore");
// Canonical spelling, because the `ignore` crate expands every `~` in
// the configured path to the home directory - not only a leading one
// (`expand_tilde` in its gitignore.rs). Windows hands out temp paths
// with 8.3 short names (`RUNNER~1`), which that expansion corrupts
// into a path that opens nothing; the canonical form has no `~`.
let base = fs::canonicalize(dir.path()).unwrap();
let global_ignore = base.join("global_ignore");
fs::write(&global_ignore, "secret.txt\n").unwrap();
let gitconfig = dir.path().join("gitconfig");
let gitconfig = base.join("gitconfig");
fs::write(
&gitconfig,
format!("[core]\nexcludesFile = {}\n", global_ignore.display()),
Expand Down
22 changes: 17 additions & 5 deletions crates/siloscan/tests/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,11 @@ const COVERAGE_RULE: &str = concat!(
/// line the cursor sits on and `\r` returns to its start, so a report line
/// written raw overwrites itself with whatever follows. A repository reaches
/// this by pointing `rules` in its own `siloscan.toml` at a rule file it ships.
///
/// The escape fixtures are unix-only: the second vector is a file whose name
/// carries the escape byte, and Windows refuses control characters in file
/// names, so neither the fixture nor the attack can exist there.
#[cfg(unix)]
const ESC_MESSAGE_RULE: &str = concat!(
"version: 1\n",
"rules:\n",
Expand All @@ -115,15 +120,17 @@ const ESC_MESSAGE_RULE: &str = concat!(
" pattern: 'needle'\n",
);

/// The second vector, and the one needing no config at all: a file name. Any
/// byte but `/` and NUL is legal in one, so the escape arrives through the
/// walker without the repository configuring anything.
/// The second vector, and the one needing no config at all: a file name. On
/// unix any byte but `/` and NUL is legal in one, so the escape arrives
/// through the walker without the repository configuring anything.
#[cfg(unix)]
const ESC_PATH: &str = "ev\u{1b}[2Kil.js";

/// The fingerprint `siloscan` 1.1.1 produced for the finding in [`ESC_PATH`],
/// recorded before the terminal escaping existed. Escaping is a rendering
/// concern, so this value may not move: a baseline written by an older release
/// has to keep covering the same finding.
#[cfg(unix)]
const ESC_FINGERPRINT: &str = "a5421000a7dd76b51bd5b139caaf6746668891ada868f7b47d0437039dea245a";

const SILO_CONFIG: &str = concat!(
Expand Down Expand Up @@ -1066,6 +1073,7 @@ fn a_warm_run_over_a_hidden_tree_is_byte_identical_to_the_cold_one() {
/// One file whose name carries an escape sequence, matched by a rule whose
/// message carries another: the two ways a scanned repository reaches the
/// operator's terminal.
#[cfg(unix)]
fn esc_fixture() -> (TempDir, TempDir) {
(
rules_dir(ESC_MESSAGE_RULE),
Expand All @@ -1076,6 +1084,7 @@ fn esc_fixture() -> (TempDir, TempDir) {
/// Written raw, `ESC [ 2 K` followed by a carriage return erases the report
/// line and rewrites it, so a repository holding a live credential could render
/// as a clean scan. The bytes are rendered instead, and the finding still shows.
#[cfg(unix)]
#[test]
fn no_escape_byte_from_a_scanned_repository_reaches_the_terminal() {
let (rules, src) = esc_fixture();
Expand Down Expand Up @@ -1105,6 +1114,7 @@ fn no_escape_byte_from_a_scanned_repository_reaches_the_terminal() {
/// Escaping is a rendering concern and stops at the human format: the JSON
/// report still carries the bytes, and the fingerprint an older release wrote
/// into a baseline still identifies the same finding.
#[cfg(unix)]
#[test]
fn escaping_leaves_the_json_report_and_its_fingerprints_where_they_were() {
let (rules, src) = esc_fixture();
Expand Down Expand Up @@ -1270,8 +1280,10 @@ fn help_documents_only_the_forms_that_work() {

assert_eq!(output.status.code(), Some(0));
let text = stdout(&output);
assert!(text.contains("siloscan [OPTIONS] [PATH]"), "stdout: {text}");
assert!(text.contains("siloscan <COMMAND>"), "stdout: {text}");
// The binary name is platform-dependent (`siloscan.exe` on Windows); the
// assertion is about the usage forms, not the name.
assert!(text.contains("[OPTIONS] [PATH]"), "stdout: {text}");
assert!(text.contains("<COMMAND>"), "stdout: {text}");
assert!(!text.contains("[PATH] [COMMAND]"), "stdout: {text}");
}

Expand Down