From 9a02df3f6c6a7cc029ded169be78ae6397c619aa Mon Sep 17 00:00:00 2001 From: FGH-2 Date: Sat, 4 Jul 2026 00:48:36 +0300 Subject: [PATCH] Fix CVE sync reliability and per-subcommand CLI help (v0.2.1) Improve NVD fetch error messages and proxy-safe body decoding, make --years imply --full for year feed syncs, and route each top-level CLI subcommand through a dedicated parser so --help shows only relevant flags. Co-authored-by: Cursor --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/cli.rs | 547 ++++++++++++++++++++++++++++++++++++--- src/cve/providers/nvd.rs | 23 +- src/cve/rate_limit.rs | 136 +++++++++- src/cve/sync.rs | 11 +- src/main.rs | 8 +- 7 files changed, 670 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ae12d7..7e0491d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -239,7 +239,7 @@ dependencies = [ [[package]] name = "chronosphere" -version = "0.1.0" +version = "0.2.1" dependencies = [ "anyhow", "arboard", diff --git a/Cargo.toml b/Cargo.toml index b12ce7d..f474809 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chronosphere" -version = "0.2.0" +version = "0.2.1" edition = "2024" description = "Vim-flavored TUI for browsing, templating, and firing pentest commands from a per-engagement project directory." diff --git a/src/cli.rs b/src/cli.rs index 447006d..d836578 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -16,9 +16,10 @@ use clap::{Args, Parser, Subcommand}; use std::fs; use std::path::{Path, PathBuf}; -#[derive(Parser, Debug)] -#[command(name = "chronosphere", version, about = "Vim TUI + CLI for pentest engagement commands", long_about = None)] -pub struct Cli { +// Engagement / targeting flags shared by library and CRUD subcommands. +#[derive(Args, Debug, Clone, Default)] +#[command(next_help_heading = "Engagement")] +pub struct EngagementOpts { /// Engagement name (defaults to last-used or single existing one). #[arg(short = 'e', long, global = true)] pub engagement: Option, @@ -38,6 +39,19 @@ pub struct Cli { /// Access point name to apply for this invocation. #[arg(short = 'a', long, global = true)] pub ap: Option, +} + +impl EngagementOpts { + pub fn engagements_root(&self) -> PathBuf { + self.root.clone().unwrap_or_else(config::engagements_root) + } +} + +#[derive(Parser, Debug)] +#[command(name = "chronosphere", version, about = "Vim TUI + CLI for pentest engagement commands", long_about = None)] +pub struct Cli { + #[command(flatten)] + pub opts: EngagementOpts, #[command(subcommand)] pub command: Option, @@ -168,7 +182,7 @@ pub enum CveCmd { /// Import all year feeds (not just modified/recent). Mutually exclusive with `--month`. #[arg(long, conflicts_with = "month")] full: bool, - /// Year range or list (e.g. 2024-2026 or 2025,2026). Ignored when `--month` is set. + /// Year range or list (e.g. 2024-2026 or 2025,2026). Implies `--full`. Ignored when `--month` is set. #[arg(long, conflicts_with = "month")] years: Option, /// Sync one calendar month via NVD API (e.g. 2026-05). Smaller than `--full`. @@ -397,6 +411,478 @@ pub enum VariableCmd { }, } +// --- Per-subcommand parsers (clean `--help` without unrelated root flags) --- + +macro_rules! engagement_cli { + ($(#[$meta:meta])* $name:ident, $sub:literal, $($field:tt)*) => { + $(#[$meta])* + pub struct $name { + #[command(flatten)] + pub engagement: EngagementOpts, + $($field)* + } + }; +} + +macro_rules! standalone_cli { + ($(#[$meta:meta])* $name:ident, $sub:literal, $($field:tt)*) => { + $(#[$meta])* + pub struct $name { + $($field)* + } + }; +} + +engagement_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere list", about = "List engagements / categories / commands", disable_version_flag = true)] + ListCli, "list", + #[command(flatten)] + pub args: ListArgs, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere show", about = "Show a command (resolved with current target/creds)", disable_version_flag = true)] + ShowCli, "show", + pub id: String, + #[arg(short = 'v', long = "var", value_name = "KEY=VALUE")] + pub vars: Vec, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere render", about = "Print the resolved command (shell-safe, single line)", disable_version_flag = true)] + RenderCli, "render", + pub id: String, + #[arg(short = 'v', long = "var", value_name = "KEY=VALUE")] + pub vars: Vec, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere yank", about = "Render and copy a command to the clipboard", disable_version_flag = true)] + YankCli, "yank", + pub id: String, + #[arg(long)] + pub raw: bool, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere run", about = "Execute a command (runs in current shell, prints output)", disable_version_flag = true)] + RunCli, "run", + pub id: String, + #[arg(short = 'v', long = "var", value_name = "KEY=VALUE")] + pub vars: Vec, + #[arg(long)] + pub dry_run: bool, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere search", about = "Search the command library", disable_version_flag = true)] + SearchCli, "search", + pub query: String, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command( + name = "chronosphere targets", + about = "Target CRUD", + disable_version_flag = true, + subcommand_required = true, + arg_required_else_help = true, + )] + TargetsCli, "targets", + #[command(subcommand)] + pub command: TargetCmd, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command( + name = "chronosphere aps", + about = "WiFi access point CRUD", + disable_version_flag = true, + subcommand_required = true, + arg_required_else_help = true, + )] + ApsCli, "aps", + #[command(subcommand)] + pub command: ApCmd, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command( + name = "chronosphere pivots", + about = "Foothold pivot CRUD (ligolo tunnel + SSH remote exec)", + disable_version_flag = true, + subcommand_required = true, + arg_required_else_help = true, + )] + PivotsCli, "pivots", + #[command(subcommand)] + pub command: PivotCmd, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command( + name = "chronosphere creds", + about = "Credential CRUD", + disable_version_flag = true, + subcommand_required = true, + arg_required_else_help = true, + )] + CredsCli, "creds", + #[command(subcommand)] + pub command: CredsCmd, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command( + name = "chronosphere variables", + about = "Template variable placeholders for the active engagement", + disable_version_flag = true, + subcommand_required = true, + arg_required_else_help = true, + )] + VariablesCli, "variables", + #[command(subcommand)] + pub command: VariableCmd, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere doctor", about = "Check installed tools referenced by the library", disable_version_flag = true)] + DoctorCli, "doctor", + #[arg(long)] + pub missing: bool, +); + +engagement_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere mcp-serve", about = "Run as a Model Context Protocol server on stdio (for cursor-agent etc)", disable_version_flag = true)] + McpServeCli, "mcp-serve", +); + +standalone_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere new", about = "Create a new engagement directory", disable_version_flag = true)] + NewCli, "new", + #[arg(long)] + pub root: Option, + pub name: String, + #[arg(long)] + pub notes: Option, +); + +standalone_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere update-templates", about = "Extract embedded built-in commands to the user data dir", disable_version_flag = true)] + UpdateTemplatesCli, "update-templates", + #[arg(long)] + pub force: bool, +); + +standalone_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere where", about = "Print where chronosphere stores things", disable_version_flag = true)] + WhereCli, "where", +); + +standalone_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere path-install", about = "Print an `eval $(chronosphere path-install)` snippet for shell setup", disable_version_flag = true)] + PathInstallCli, "path-install", +); + +standalone_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere mcp-config", about = "Print an mcp.json snippet to wire chronosphere into Cursor / Claude clients", disable_version_flag = true)] + McpConfigCli, "mcp-config", + #[arg(long)] + pub ssh: Option, + #[arg(long, default_value_t = 22)] + pub port: u16, + #[arg(long, default_value = "/usr/local/bin/chronosphere")] + pub remote_path: String, + #[arg(long)] + pub identity: Option, +); + +standalone_cli!( + #[derive(Parser, Debug)] + #[command(name = "chronosphere deploy", about = "Ship the chronosphere binary to a remote host via scp (Pwnbox, lab box, etc)", disable_version_flag = true)] + DeployCli, "deploy", + #[command(flatten)] + pub args: crate::deploy::DeployArgs, +); + +#[derive(Parser, Debug)] +#[command( + name = "chronosphere cve", + about = "Local CVE index: sync, search, and browse vulnerability data", + long_about = None, + disable_version_flag = true, + subcommand_required = true, + arg_required_else_help = true, + after_help = "Examples: + chronosphere cve sync Incremental sync (modified + recent feeds) + chronosphere cve sync --month 2026-03 One month via NVD API + chronosphere cve sync --years 2025 Full year feed (implies --full) + chronosphere cve search nginx rce --limit 20 + chronosphere cve show CVE-2024-3400 + +Run `chronosphere cve sync --help` for all sync flags." +)] +pub struct CveCli { + #[command(subcommand)] + pub command: CveCmd, +} + +const TOP_LEVEL_SUBCOMMANDS: &[&str] = &[ + "new", + "list", + "show", + "render", + "yank", + "run", + "search", + "targets", + "aps", + "pivots", + "creds", + "variables", + "doctor", + "update-templates", + "where", + "path-install", + "mcp-serve", + "mcp-config", + "deploy", + "cve", + "tui", +]; + +fn engagement_flag_takes_value(flag: &str) -> bool { + matches!( + flag, + "-e" | "--engagement" | "--root" | "-t" | "--target" | "-c" | "--creds" | "-a" | "--ap" + ) +} + +fn is_engagement_flag(arg: &str) -> bool { + engagement_flag_takes_value(arg) +} + +fn find_top_level_subcommand() -> Option<(usize, &'static str)> { + let args: Vec = std::env::args().collect(); + let mut i = 1; + while i < args.len() { + let a = args[i].as_str(); + if a == "--" { + return None; + } + if a.starts_with('-') { + if engagement_flag_takes_value(a) && i + 1 < args.len() { + i += 2; + } else { + i += 1; + } + continue; + } + for sub in TOP_LEVEL_SUBCOMMANDS { + if a == *sub { + return Some((i, sub)); + } + } + return None; + } + None +} + +fn build_subcommand_argv(sub_idx: usize, sub: &str) -> Vec { + let args: Vec = std::env::args().collect(); + let mut out = vec![format!("chronosphere {sub}")]; + let mut i = 1; + while i < sub_idx { + if is_engagement_flag(&args[i]) { + out.push(args[i].clone()); + if engagement_flag_takes_value(&args[i]) && i + 1 < sub_idx { + i += 1; + out.push(args[i].clone()); + } + } + i += 1; + } + out.extend(args[sub_idx + 1..].iter().cloned()); + out +} + +fn cli_from(opts: EngagementOpts, command: Command) -> Cli { + Cli { + opts, + command: Some(command), + } +} + +/// Parse and run when argv names a top-level subcommand (clean per-command `--help`). +pub async fn try_early_dispatch() -> Result { + let Some((sub_idx, sub)) = find_top_level_subcommand() else { + return Ok(false); + }; + if sub == "tui" { + return Ok(false); + } + + let argv = build_subcommand_argv(sub_idx, sub); + let handled = match sub { + "new" => { + let c = NewCli::parse_from(&argv); + let mut opts = EngagementOpts::default(); + opts.root = c.root; + dispatch(cli_from( + opts, + Command::New { + name: c.name, + notes: c.notes, + }, + )) + .await? + } + "list" => { + let c = ListCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::List(c.args))).await? + } + "show" => { + let c = ShowCli::parse_from(&argv); + dispatch(cli_from( + c.engagement, + Command::Show { + id: c.id, + vars: c.vars, + }, + )) + .await? + } + "render" => { + let c = RenderCli::parse_from(&argv); + dispatch(cli_from( + c.engagement, + Command::Render { + id: c.id, + vars: c.vars, + }, + )) + .await? + } + "yank" => { + let c = YankCli::parse_from(&argv); + dispatch(cli_from( + c.engagement, + Command::Yank { + id: c.id, + raw: c.raw, + }, + )) + .await? + } + "run" => { + let c = RunCli::parse_from(&argv); + dispatch(cli_from( + c.engagement, + Command::Run { + id: c.id, + vars: c.vars, + dry_run: c.dry_run, + }, + )) + .await? + } + "search" => { + let c = SearchCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::Search { query: c.query })).await? + } + "targets" => { + let c = TargetsCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::Targets(c.command))).await? + } + "aps" => { + let c = ApsCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::Aps(c.command))).await? + } + "pivots" => { + let c = PivotsCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::Pivots(c.command))).await? + } + "creds" => { + let c = CredsCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::Creds(c.command))).await? + } + "variables" => { + let c = VariablesCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::Variables(c.command))).await? + } + "doctor" => { + let c = DoctorCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::Doctor { missing: c.missing })).await? + } + "update-templates" => { + let c = UpdateTemplatesCli::parse_from(&argv); + dispatch(cli_from( + EngagementOpts::default(), + Command::UpdateTemplates { force: c.force }, + )) + .await? + } + "where" => { + let _c = WhereCli::parse_from(&argv); + dispatch(cli_from(EngagementOpts::default(), Command::Where)).await? + } + "path-install" => { + let _c = PathInstallCli::parse_from(&argv); + dispatch(cli_from(EngagementOpts::default(), Command::PathInstall)).await? + } + "mcp-serve" => { + let c = McpServeCli::parse_from(&argv); + dispatch(cli_from(c.engagement, Command::McpServe)).await? + } + "mcp-config" => { + let c = McpConfigCli::parse_from(&argv); + dispatch(cli_from( + EngagementOpts::default(), + Command::McpConfig { + ssh: c.ssh, + port: c.port, + remote_path: c.remote_path, + identity: c.identity, + }, + )) + .await? + } + "deploy" => { + let c = DeployCli::parse_from(&argv); + dispatch(cli_from( + EngagementOpts::default(), + Command::Deploy(c.args), + )) + .await? + } + "cve" => { + let c = CveCli::parse_from(&argv); + dispatch_cve(c.command).await?; + true + } + _ => return Ok(false), + }; + Ok(handled) +} + /// Returns true if dispatching consumed the run (CLI mode); false to fall through to TUI. pub async fn dispatch(cli: Cli) -> Result { let cmd = match cli.command { @@ -409,10 +895,7 @@ pub async fn dispatch(cli: Cli) -> Result { crate::builtin::ensure_user_dir().context("ensure user templates")?; - let root = cli - .root - .clone() - .unwrap_or_else(config::engagements_root); + let root = cli.opts.engagements_root(); match cmd { Command::Tui => Ok(false), @@ -463,7 +946,7 @@ alias chronosphere='{bin}' } Command::McpServe => { let opts = crate::mcp::ServerOpts { - engagement: cli.engagement.clone(), + engagement: cli.opts.engagement.clone(), root: root.clone(), }; crate::mcp::serve(opts).await.context("mcp serve")?; @@ -497,7 +980,7 @@ alias chronosphere='{bin}' Ok(true) } Command::Variables(c) => { - let mut e = open_engagement(&root, cli.engagement.as_deref())?; + let mut e = open_engagement(&root, cli.opts.engagement.as_deref())?; let sources = library_sources(&root, Some(&e.meta.name))?; let lib = load_library(&sources)?; match c { @@ -518,7 +1001,7 @@ alias chronosphere='{bin}' Ok(true) } Command::Doctor { missing } => { - let sources = library_sources(root.as_path(), cli.engagement.as_deref())?; + let sources = library_sources(root.as_path(), cli.opts.engagement.as_deref())?; let lib = load_library(&sources)?; let tools = lib.all_tools_referenced(); let mut found = 0usize; @@ -545,7 +1028,7 @@ alias chronosphere='{bin}' Ok(true) } Command::List(args) => { - let sources = library_sources(root.as_path(), cli.engagement.as_deref())?; + let sources = library_sources(root.as_path(), cli.opts.engagement.as_deref())?; let lib = load_library(&sources)?; match args.kind { ListKind::Engagements => { @@ -577,7 +1060,7 @@ alias chronosphere='{bin}' } } ListKind::Jobs => { - let e = open_engagement(&root, cli.engagement.as_deref())?; + let e = open_engagement(&root, cli.opts.engagement.as_deref())?; for job in e.history.recent.iter().rev() { println!( "{} {:?} {} ({}/{})", @@ -593,7 +1076,7 @@ alias chronosphere='{bin}' Ok(true) } Command::Search { query } => { - let sources = library_sources(root.as_path(), cli.engagement.as_deref())?; + let sources = library_sources(root.as_path(), cli.opts.engagement.as_deref())?; let lib = load_library(&sources)?; let needle = query.to_lowercase(); for cat in &lib.categories { @@ -616,10 +1099,10 @@ alias chronosphere='{bin}' Command::Show { id, vars } | Command::Render { id, vars } => { let resolved = resolve( &root, - cli.engagement.as_deref(), - &cli.target, - &cli.ap, - &cli.creds, + cli.opts.engagement.as_deref(), + &cli.opts.target, + &cli.opts.ap, + &cli.opts.creds, &id, &vars, )?; @@ -628,14 +1111,14 @@ alias chronosphere='{bin}' } Command::Yank { id, raw } => { let text = if raw { - raw_template(&root, cli.engagement.as_deref(), &id)? + raw_template(&root, cli.opts.engagement.as_deref(), &id)? } else { resolve( &root, - cli.engagement.as_deref(), - &cli.target, - &cli.ap, - &cli.creds, + cli.opts.engagement.as_deref(), + &cli.opts.target, + &cli.opts.ap, + &cli.opts.creds, &id, &[], )? @@ -651,10 +1134,10 @@ alias chronosphere='{bin}' Command::Run { id, vars, dry_run } => { let resolved = resolve( &root, - cli.engagement.as_deref(), - &cli.target, - &cli.ap, - &cli.creds, + cli.opts.engagement.as_deref(), + &cli.opts.target, + &cli.opts.ap, + &cli.opts.creds, &id, &vars, )?; @@ -662,11 +1145,11 @@ alias chronosphere='{bin}' println!("{}", resolved); return Ok(true); } - run_with_history(&root, cli.engagement.as_deref(), &id, &resolved).await?; + run_with_history(&root, cli.opts.engagement.as_deref(), &id, &resolved).await?; Ok(true) } Command::Targets(c) => { - let mut e = open_engagement(&root, cli.engagement.as_deref())?; + let mut e = open_engagement(&root, cli.opts.engagement.as_deref())?; match c { TargetCmd::List => { let active = e.targets.active().map(|t| t.name.clone()); @@ -710,7 +1193,7 @@ alias chronosphere='{bin}' Ok(true) } Command::Aps(c) => { - let mut e = open_engagement(&root, cli.engagement.as_deref())?; + let mut e = open_engagement(&root, cli.opts.engagement.as_deref())?; match c { ApCmd::List => { let active = e.aps.active().map(|a| a.name.clone()); @@ -772,7 +1255,7 @@ alias chronosphere='{bin}' Ok(true) } Command::Pivots(c) => { - let mut e = open_engagement(&root, cli.engagement.as_deref())?; + let mut e = open_engagement(&root, cli.opts.engagement.as_deref())?; match c { PivotCmd::List => { let tun = e.pivots.active_tunnel.clone(); @@ -900,7 +1383,7 @@ alias chronosphere='{bin}' Ok(true) } Command::Creds(c) => { - let mut e = open_engagement(&root, cli.engagement.as_deref())?; + let mut e = open_engagement(&root, cli.opts.engagement.as_deref())?; match c { CredsCmd::List => { let active = e.profiles.active().map(|p| p.name.clone()); diff --git a/src/cve/providers/nvd.rs b/src/cve/providers/nvd.rs index 39acaca..1a150b6 100644 --- a/src/cve/providers/nvd.rs +++ b/src/cve/providers/nvd.rs @@ -20,8 +20,7 @@ pub struct NvdFeedMeta { pub async fn fetch_feed_meta(client: &HttpClient, feed_name: &str) -> Result { let url = format!("{NVD_FEED_BASE}/{feed_name}.meta"); - let resp = client.get("nvd", &url).await?; - let text = resp.text().await?; + let text = client.get_text("nvd", &url).await?; let mut sha256 = String::new(); let mut last_modified = None; for line in text.lines() { @@ -51,8 +50,7 @@ pub async fn download_and_import_feed( let url = format!("{NVD_FEED_BASE}/{feed_name}.json.gz"); tracing::info!(feed = feed_name, "downloading"); - let resp = client.get("nvd", &url).await?; - let bytes = resp.bytes().await?; + let bytes = client.get_bytes("nvd", &url).await?; let mut decoder = GzDecoder::new(&bytes[..]); let mut json = String::new(); decoder.read_to_string(&mut json)?; @@ -239,12 +237,11 @@ pub async fn fetch_single_cve( api_key: Option<&str>, ) -> Result { let url = format!("{NVD_API_BASE}?cveId={cve_id}"); - let resp = if let Some(key) = api_key { - client.get_with_api_key("nvd", &url, key).await? + let text = if let Some(key) = api_key { + client.get_text_with_api_key("nvd", &url, key).await? } else { - client.get("nvd", &url).await? + client.get_text("nvd", &url).await? }; - let text = resp.text().await?; let (added, updated) = import_nvd_json_stats(store, &text)?; Ok(added + updated > 0) } @@ -316,13 +313,13 @@ pub async fn sync_by_month( } } let url = build_month_api_url(&start, &end, by_modified, start_index, RESULTS_PER_PAGE); - let resp = if let Some(key) = api_key { - client.get_with_api_key("nvd", &url, key).await? + let text = if let Some(key) = api_key { + client.get_text_with_api_key("nvd", &url, key).await? } else { - client.get("nvd", &url).await? + client.get_text("nvd", &url).await? }; - let text = resp.text().await?; - let root: Value = serde_json::from_str(&text).context("parse nvd api response")?; + let root: Value = serde_json::from_str(&text) + .with_context(|| format!("parse NVD API JSON for {label} ({field}) page {page}"))?; if start_index == 0 { total_results = root.get("totalResults").and_then(|v| v.as_u64()).unwrap_or(0); tracing::info!(total = total_results, "NVD CVE count for month range"); diff --git a/src/cve/rate_limit.rs b/src/cve/rate_limit.rs index ea4ef84..af83adb 100644 --- a/src/cve/rate_limit.rs +++ b/src/cve/rate_limit.rs @@ -1,5 +1,5 @@ use anyhow::{Context, Result, bail}; -use reqwest::header::{HeaderMap, HeaderValue, RETRY_AFTER, USER_AGENT}; +use reqwest::header::{HeaderMap, HeaderValue, RETRY_AFTER, USER_AGENT, ACCEPT_ENCODING}; use reqwest::{Client, Response, StatusCode}; use std::collections::HashMap; use std::sync::Arc; @@ -127,7 +127,7 @@ impl HttpClient { } pub async fn get(&self, provider: &str, url: &str) -> Result { - self.execute(provider, || self.inner.get(url)).await + self.execute(provider, url, || self.inner.get(url)).await } pub async fn get_with_api_key( @@ -136,11 +136,98 @@ impl HttpClient { url: &str, api_key: &str, ) -> Result { - self.execute(provider, || self.inner.get(url).header("apiKey", api_key)) + self.execute(provider, url, || self.inner.get(url).header("apiKey", api_key)) .await } - async fn execute(&self, provider: &str, build: F) -> Result + /// Read a successful HTTP response body as text, with a proxy-safe identity-encoding retry. + pub async fn read_text( + &self, + provider: &str, + url: &str, + resp: Response, + api_key: Option<&str>, + ) -> Result { + let status = resp.status(); + match resp.text().await { + Ok(text) => Ok(text), + Err(first) => { + tracing::warn!(url, error = %first, "response body decode failed, retrying without compression"); + let resp = self + .execute(provider, url, || { + let req = self.inner.get(url).header(ACCEPT_ENCODING, "identity"); + if let Some(key) = api_key { + req.header("apiKey", key) + } else { + req + } + }) + .await?; + resp.text().await.with_context(|| { + format!( + "read response body from {url} (HTTP {status}) after decode error: {first}. \ + This usually means a truncated or garbled response through a proxy — not NVD rate limiting \ + (429 responses are retried automatically). Try without proxychains or set NVD_API_KEY." + ) + }) + } + } + } + + /// GET and return the response body as text. + pub async fn get_text(&self, provider: &str, url: &str) -> Result { + let resp = self.get(provider, url).await?; + self.read_text(provider, url, resp, None).await + } + + /// GET with an API key and return the response body as text. + pub async fn get_text_with_api_key( + &self, + provider: &str, + url: &str, + api_key: &str, + ) -> Result { + let resp = self.get_with_api_key(provider, url, api_key).await?; + self.read_text(provider, url, resp, Some(api_key)).await + } + + /// GET and return the response body as bytes (for gzip feeds). + pub async fn get_bytes(&self, provider: &str, url: &str) -> Result> { + let resp = self.get(provider, url).await?; + self.read_bytes(provider, url, resp).await + } + + /// Read a successful HTTP response body as bytes, with an identity-encoding retry. + pub async fn read_bytes( + &self, + provider: &str, + url: &str, + resp: Response, + ) -> Result> { + let status = resp.status(); + match resp.bytes().await { + Ok(bytes) => Ok(bytes.to_vec()), + Err(first) => { + tracing::warn!(url, error = %first, "response body decode failed, retrying without compression"); + let resp = self + .execute(provider, url, || { + self.inner + .get(url) + .header(ACCEPT_ENCODING, "identity") + }) + .await?; + let bytes = resp.bytes().await.with_context(|| { + format!( + "read response body from {url} (HTTP {status}) after decode error: {first}. \ + This usually means a truncated or garbled response through a proxy." + ) + })?; + Ok(bytes.to_vec()) + } + } + } + + async fn execute(&self, provider: &str, url: &str, build: F) -> Result where F: Fn() -> reqwest::RequestBuilder, { @@ -148,7 +235,10 @@ impl HttpClient { loop { self.wait_for_slot(provider).await; self.mark_request(provider).await; - let resp = build().send().await.context("http request")?; + let resp = build() + .send() + .await + .with_context(|| format!("{provider}: request to {url}"))?; let status = resp.status(); if status.is_success() { @@ -158,7 +248,10 @@ impl HttpClient { if status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::SERVICE_UNAVAILABLE { attempt += 1; if attempt >= MAX_RETRIES { - bail!("{provider}: rate limited after {MAX_RETRIES} retries ({status})"); + bail!( + "{provider}: rate limited after {MAX_RETRIES} retries (HTTP {status}) for {url}. \ + Set NVD_API_KEY or wait before retrying." + ); } let retry_secs = resp .headers() @@ -166,15 +259,42 @@ impl HttpClient { .and_then(|v| v.to_str().ok()) .and_then(|s| s.parse::().ok()) .unwrap_or(DEFAULT_RETRY_AFTER_SECS); - tracing::warn!(provider, attempt, retry_secs, "rate limited, backing off"); + tracing::warn!(provider, attempt, retry_secs, url, "rate limited, backing off"); self.set_backoff(provider, Duration::from_secs(retry_secs)).await; sleep(Duration::from_secs(retry_secs)).await; continue; } let body = resp.text().await.unwrap_or_default(); - bail!("{provider}: HTTP {status}: {body}"); + let body_hint = truncate_body_hint(&body); + let hint = http_error_hint(status, url); + bail!("HTTP {status} for {url}{body_hint}{hint}"); + } + } +} + +fn truncate_body_hint(body: &str) -> String { + let trimmed: String = body.chars().filter(|c| !c.is_control()).take(200).collect(); + if trimmed.is_empty() { + String::new() + } else { + format!(": {trimmed}") + } +} + +fn http_error_hint(status: StatusCode, url: &str) -> &'static str { + match status { + StatusCode::NOT_FOUND => { + if url.contains("/feeds/json/cve/") { + " (feed file not found — check the feed name; year feeds require `cve sync --full --years YYYY`)" + } else { + " (resource not found)" + } + } + StatusCode::FORBIDDEN | StatusCode::TOO_MANY_REQUESTS => { + " (likely rate limited — set NVD_API_KEY)" } + _ => "", } } diff --git a/src/cve/sync.rs b/src/cve/sync.rs index bb73e04..739503b 100644 --- a/src/cve/sync.rs +++ b/src/cve/sync.rs @@ -45,7 +45,7 @@ pub async fn sync(options: SyncOptions) -> Result { result.added += a; result.updated += u; } - Err(e) => result.errors.push(format!("nvd: {e}")), + Err(e) => result.errors.push(format!("NVD sync failed: {e:#}")), } } @@ -139,8 +139,15 @@ async fn sync_nvd( let mut added = 0u64; let mut updated = 0u64; + let full = options.full || !options.years.is_empty(); - if options.full { + if !options.years.is_empty() && !options.full && options.progress { + eprintln!( + "cve sync: --years implies --full (downloading year feeds, not modified/recent only)" + ); + } + + if full { let years = if options.years.is_empty() { cfg.sync.default_years.clone() } else { diff --git a/src/main.rs b/src/main.rs index 2546878..e304a14 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,10 +25,14 @@ async fn main() -> Result<()> { init_tracing().context("init tracing")?; tracing::info!("starting chronosphere"); + if cli::try_early_dispatch().await.context("dispatch subcommand cli")? { + return Ok(()); + } + let cli = cli::Cli::parse(); let boot = app::AppBoot { - engagement: cli.engagement.clone(), - root: cli.root.clone(), + engagement: cli.opts.engagement.clone(), + root: cli.opts.root.clone(), }; if cli::dispatch(cli).await.context("dispatch cli")? { return Ok(());