diff --git a/CHANGELOG.md b/CHANGELOG.md index 7019778..49952c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +## [0.11.5] - 2026-09-14 + +### Added + +- Protect detected DirectStorage game roots from new WOF compression by default, with a Settings toggle to allow an explicit override. +- Infer DirectStorage game roots from common launcher/library layouts and Unreal Engine DirectStorage layouts instead of treating an entire game library as one target. +- Add **Decompress + exclude** to expand a selected folder and persist it in File exclusions so it is not compressed again later. +- Check Windows/WOF availability before starting compression or decompression and show a clear error when WOF is unavailable on the target volume. + +### Changed + +- Decompression now uses storage-aware parallel workers on SSDs. Auto is capped at eight workers, manual thread limits are respected, and the HDD single-worker safeguard still applies. +- Coalesce high-frequency status and folder-summary updates in the embedded UI so large scans and file operations do less redundant DOM work. +- DirectStorage detection now acts as a protection mechanism rather than only displaying a warning. + +### Fixed + +- Failed decompression jobs remain classified as compressed instead of being moved into the skipped bucket. +- **Decompress + exclude** immediately updates the analysed folder state so the newly excluded files are not offered for recompression in the same session. + ## [0.11.4] - 2026-09-08 ### Changed @@ -160,7 +180,8 @@ - Initial release -[Unreleased]: https://github.com/wefalltomorrow/Compactor/compare/v0.11.4...HEAD +[Unreleased]: https://github.com/wefalltomorrow/Compactor/compare/v0.11.5...HEAD +[0.11.5]: https://github.com/wefalltomorrow/Compactor/compare/v0.11.4...v0.11.5 [0.11.4]: https://github.com/wefalltomorrow/Compactor/compare/v0.11.3...v0.11.4 [0.11.3]: https://github.com/wefalltomorrow/Compactor/compare/v0.11.2...v0.11.3 [0.11.2]: https://github.com/wefalltomorrow/Compactor/releases/tag/v0.11.2 diff --git a/Cargo.lock b/Cargo.lock index 4a1d3a9..6e283d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,7 +149,7 @@ dependencies = [ [[package]] name = "compactor" -version = "0.11.4" +version = "0.11.5" dependencies = [ "backtrace", "compresstimator", diff --git a/Cargo.toml b/Cargo.toml index ad9c5f5..edae987 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "compactor" -version = "0.11.4" +version = "0.11.5" authors = ["Thomas Hurst "] homepage = "https://github.com/wefalltomorrow/Compactor" repository = "https://github.com/wefalltomorrow/Compactor" diff --git a/README.md b/README.md index 7fc94ad..8a825ec 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,16 @@ This repository is a maintained fork of [Freaky/Compactor](https://github.com/Fr - Sampled compressibility analysis before compression - Configurable minimum estimated savings threshold, defaulting to 1% - Estimated post-compression size and additional savings during analysis -- Storage-aware multithreaded analysis and compression on SSDs +- Storage-aware multithreaded analysis, compression, and decompression on SSDs - Conservative physical-core-aware LZX Auto concurrency with manual override - Single-threaded HDD operation by default - HDD analysis sampling designed to reduce seek overhead - Configurable compression/decompression worker priority, defaulting to Below Normal +- DirectStorage game-root detection and protection, enabled by default with a Settings override +- Decompress + exclude for restoring a folder and preventing future recompression +- WOF availability checks before file transformations begin - The system is kept awake during compression/decompression while the display may still turn off - Local-NTFS target validation and protection for Windows-managed paths -- DirectStorage runtime detection with a warning before compression - In-app searchable viewer for folders containing WOF-compressed files - Pause, resume, and stop controls - Timestamp preservation after compression and decompression @@ -36,12 +38,13 @@ This repository is a maintained fork of [Freaky/Compactor](https://github.com/Fr - Maximum threads: `Auto` - Compression priority: `Below Normal` - HDDs only use 1 thread: enabled +- Protect DirectStorage games: enabled - Excluded paths: - `*:\\Windows*` - `*:\\System Volume Information*` - `*:\\$*` -`Auto` is storage- and workload-aware. On SSDs it uses up to eight workers for analysis. XPRESS compression can use up to 16 logical-CPU workers. LZX Auto uses one outer worker on CPUs with four or fewer physical cores and at most two on larger CPUs because LZX WOF operations can be CPU-intensive. A manual limit from 1 to 16 is treated as an explicit override and is respected for LZX as well. HDDs still use one worker by default when the HDD safeguard is enabled, and unknown storage is handled conservatively with one worker. Compression reuses a valid analysis estimate when the file has not changed, avoiding duplicate sampling before WOF compression. +`Auto` is storage- and workload-aware. On SSDs it uses up to eight workers for analysis. XPRESS compression can use up to 16 logical-CPU workers. LZX Auto uses one outer worker on CPUs with four or fewer physical cores and at most two on larger CPUs because LZX WOF operations can be CPU-intensive. A manual limit from 1 to 16 is treated as an explicit override and is respected for LZX as well. Auto decompression uses storage-aware parallelism up to eight workers. HDDs still use one worker by default when the HDD safeguard is enabled, and unknown storage is handled conservatively with one worker. Compression reuses a valid analysis estimate when the file has not changed, avoiding duplicate sampling before WOF compression. Compression/decompression worker priority can be set to Lowest, Below Normal, Normal, Above Normal, or Highest. Only the worker threads are affected; the GUI and analysis threads remain at normal priority. Below Normal is the default so idle CPU capacity can still be used while foreground work gets preference when the system is busy. @@ -58,10 +61,10 @@ Compactor is portable and does not require an installer or background service. 1. Choose a folder. 2. Wait for analysis to complete. 3. Review current disk usage and estimated savings. -4. Change the compression mode, savings threshold, thread limit, or worker priority in Settings if required. +4. Change the compression mode, savings threshold, thread limit, worker priority, or DirectStorage protection in Settings if required. 5. Select Compress. -Use Decompress to remove WOF backing from files previously compressed with Compactor. After analysis, select View beside the compressed count to browse folders containing WOF-compressed files inside Compactor. The viewer includes path filtering and pagination for large scans. +Use Decompress to remove WOF backing from files previously compressed with Compactor. Use **Decompress + exclude** when you also want the selected folder added to File exclusions so it will not be compressed again on later runs. After analysis, select View beside the compressed count to browse folders containing WOF-compressed files inside Compactor. The viewer includes path filtering and pagination for large scans. ## Notes @@ -69,7 +72,9 @@ Compactor is best suited to applications and game files that change infrequently Compression and decompression hold each active file against concurrent writers and deleters while still allowing readers. This reduces the risk of racing an application or game update in the middle of a WOF operation. Files that change between analysis and compression are re-estimated instead of reusing stale analysis results. -If a selected folder contains `dstorage.dll` or `dstoragecore.dll`, Compactor warns that DirectStorage is present. WOF compression can prevent DirectStorage/BypassIO from taking its intended fast path, so consider leaving DirectStorage games uncompressed when I/O performance matters. +If a selected tree contains `dstorage.dll` or `dstoragecore.dll`, Compactor identifies the likely game/install root using common game-library layouts and Unreal Engine DirectStorage layouts. With **Protect DirectStorage games** enabled, files under those detected roots are removed from the compression queue before WOF is applied. The setting can be disabled when you deliberately want to compress such files. Detection is conservative and based on the presence/layout of the DirectStorage runtime; it cannot prove that every detected game is actively using BypassIO at that moment. + +Before compression or decompression starts, Compactor also checks that Windows reports WOF support and probes the target volume using available files. If WOF is explicitly unavailable, the operation is stopped with a clear error instead of failing repeatedly file by file. Do not run Compactor blindly across an entire system drive. Whole-drive roots, network/UNC targets, non-NTFS filesystems, the active Windows directory, `System Volume Information`, root `$*` directories, and the root `Recovery` directory are rejected. User-selected folders can still contain databases, virtual machines, active logs, or other write-heavy files that are poor candidates for WOF compression. @@ -95,9 +100,13 @@ This fork includes the following changes: - Actual NTFS cluster-size-aware eligibility and projected allocation - LZX as the default compression mode with a conservative one-or-two-worker Auto policy and manual override - Configurable compression/decompression worker priority, defaulting to Below Normal +- Storage-aware parallel decompression with the HDD single-worker safeguard +- Decompress + exclude workflow - System-sleep prevention during compression/decompression without forcing the display awake - Stronger local-NTFS and Windows-managed-path target validation -- DirectStorage runtime warning +- DirectStorage game-root detection and default protection with an override +- WOF availability preflight checks +- Coalesced high-frequency GUI updates for smoother large operations - Removal of default file-extension exclusions - Storage-aware worker selection and parallel SSD processing - Reduced-seek HDD analysis sampling diff --git a/src/backend.rs b/src/backend.rs index 7a9e333..cf8970a 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -9,7 +9,9 @@ use winapi::um::winbase::SetThreadExecutionState; use winapi::um::winnt::{ES_CONTINUOUS, ES_SYSTEM_REQUIRED}; use crate::background::BackgroundHandle; +use crate::compact::{self, Compression}; use crate::compression::{BackgroundCompactor, CompressionJob}; +use crate::directstorage::{discover_direct_storage_roots, is_under}; use crate::folder::{ compression_worker_count_for_path, validate_target_path, FileInfo, FileKind, FolderInfo, FolderScan, @@ -17,6 +19,10 @@ use crate::folder::{ use crate::gui::{CompressedViewItem, GuiRequest, GuiResponse, GuiWrapper}; use crate::persistence::{config, incompressible_key, pathdb}; +const COMPRESSED_VIEW_PAGE_SIZE: usize = 100; +const AUTO_MAX_DECOMPRESSION_THREADS: usize = 8; +const WOF_PREFLIGHT_SAMPLE_FILES: usize = 8; + pub struct Backend { gui: GuiWrapper, msg: Receiver, @@ -63,7 +69,80 @@ fn progress(done_bytes: u64, total_bytes: u64) -> f32 { } } -const COMPRESSED_VIEW_PAGE_SIZE: usize = 100; +fn decompression_worker_count_for_path( + path: &Path, + max_threads: usize, + hdd_single_thread: bool, +) -> usize { + let storage_workers = compression_worker_count_for_path( + path, + Compression::Xpress16k, + max_threads, + hdd_single_thread, + ); + + if max_threads == 0 { + storage_workers.min(AUTO_MAX_DECOMPRESSION_THREADS).max(1) + } else { + storage_workers.max(1) + } +} + +fn protect_direct_storage_candidates(folder: &mut FolderInfo, roots: &[PathBuf]) -> usize { + if roots.is_empty() { + return 0; + } + + let candidates = folder.len(FileKind::Compressible); + let mut protected = 0usize; + + for _ in 0..candidates { + let Some(fi) = folder.pop(FileKind::Compressible) else { + break; + }; + let absolute = folder.path.join(&fi.path); + + if roots.iter().any(|root| is_under(&absolute, root)) { + protected += 1; + folder.push(FileKind::Skipped, fi); + } else { + folder.push(FileKind::Compressible, fi); + } + } + + protected +} + +fn add_folder_exclusion(path: &Path) -> Result { + let display = path.to_string_lossy().to_string(); + let c = config(); + let mut c = c.write().unwrap(); + let mut current = c.current(); + + if current + .excludes + .iter() + .any(|existing| existing.trim().eq_ignore_ascii_case(&display)) + { + return Ok(false); + } + + current.excludes.push(display); + current.validate()?; + c.replace(current); + c.save() + .map_err(|err| format!("Unable to save exclusions: {}", err))?; + Ok(true) +} + +fn move_compressible_to_skipped(folder: &mut FolderInfo) { + let count = folder.len(FileKind::Compressible); + for _ in 0..count { + if let Some(fi) = folder.pop(FileKind::Compressible) { + folder.push(FileKind::Skipped, fi); + } + } +} #[derive(Default)] struct CompressedFolderTotals { @@ -163,6 +242,55 @@ impl Backend { } } + fn wof_preflight(&self, folder: &FolderInfo, kind: FileKind) -> bool { + match compact::system_supports_compression() { + Ok(true) => {} + Ok(false) => { + self.gui.error( + "WOF compression unavailable", + "This version of Windows does not report WOF filesystem compression support.", + ); + return false; + } + Err(err) => { + self.gui.error( + "WOF compression unavailable", + format!("Unable to initialise Windows WOF support: {}", err), + ); + return false; + } + } + + let files = match kind { + FileKind::Compressible => &folder.compressible.files, + FileKind::Compressed => &folder.compressed.files, + FileKind::Skipped => return true, + }; + + // Locked files can make an individual probe fail. Try a handful and + // only block the operation when Windows explicitly says WOF is not + // attached/supported on an accessible file from this volume. + for fi in files.iter().take(WOF_PREFLIGHT_SAMPLE_FILES) { + let path = folder.path.join(&fi.path); + match compact::file_supports_compression(&path) { + Ok(true) => return true, + Ok(false) => { + self.gui.error( + "WOF unavailable on this drive", + "Windows Overlay Filter (Wof.sys) is not available for this NTFS volume, so WOF compression cannot be used here.", + ); + return false; + } + Err(_) => continue, + } + } + + // If every sample happened to be locked, let the normal per-file path + // handle it rather than refusing the whole operation on an uncertain + // preflight result. + true + } + pub fn run(&mut self) { loop { match self.msg.recv() { @@ -196,7 +324,13 @@ impl Backend { Ok(GuiRequest::Decompress) if self.info.is_some() => { let path = self.info.as_ref().unwrap().path.clone(); if self.target_is_valid(&path) { - self.uncompress_loop(); + self.uncompress_loop(false); + } + } + Ok(GuiRequest::DecompressAndExclude) if self.info.is_some() => { + let path = self.info.as_ref().unwrap().path.clone(); + if self.target_is_valid(&path) { + self.uncompress_loop(true); } } Ok(msg) => { @@ -318,6 +452,40 @@ impl Backend { let current = config().read().unwrap().current(); let compression = Some(current.compression); let mut folder = self.info.take().expect("fileinfo"); + + if current.protect_direct_storage && folder.direct_storage { + self.gui.status("Checking DirectStorage protection", None); + let roots = discover_direct_storage_roots(&folder.path); + let protected = protect_direct_storage_candidates(&mut folder, &roots); + if protected > 0 { + self.gui.status( + format!( + "Protected {} files in {} DirectStorage game{}", + protected, + roots.len(), + if roots.len() == 1 { "" } else { "s" } + ), + Some(0.0), + ); + self.gui.summary(folder.summary()); + } + } + + if folder.len(FileKind::Compressible) == 0 { + self.gui.status("Nothing to compact", Some(1.0)); + self.gui.summary(folder.summary()); + self.gui.scanned(); + self.info = Some(folder); + return; + } + + if !self.wof_preflight(&folder, FileKind::Compressible) { + self.gui.summary(folder.summary()); + self.gui.scanned(); + self.info = Some(folder); + return; + } + let _awake = SystemAwakeGuard::new(); let worker_count = compression_worker_count_for_path( &folder.path, @@ -569,159 +737,229 @@ impl Backend { self.info = Some(folder); } - fn uncompress_loop(&mut self) { + fn uncompress_loop(&mut self, exclude_after: bool) { + let current = config().read().unwrap().current(); + let mut folder = self.info.take().expect("fileinfo"); + + if folder.len(FileKind::Compressed) == 0 { + self.gui.status("Nothing to decompress", Some(1.0)); + self.gui.scanned(); + self.info = Some(folder); + return; + } + + if !self.wof_preflight(&folder, FileKind::Compressed) { + self.gui.summary(folder.summary()); + self.gui.scanned(); + self.info = Some(folder); + return; + } + let _awake = SystemAwakeGuard::new(); - let (send_file, send_file_rx) = bounded::(1); - let (recv_result_tx, recv_result) = bounded::<(PathBuf, io::Result)>(1); + let worker_count = decompression_worker_count_for_path( + &folder.path, + current.max_threads, + current.hdd_single_thread, + ); + let (send_file, send_file_rx) = bounded::(worker_count); + let (recv_result_tx, recv_result) = bounded::<(PathBuf, io::Result)>(worker_count); + let mut tasks = Vec::with_capacity(worker_count); - let compactor = BackgroundCompactor::new(None, 0.0, send_file_rx, recv_result_tx); - let task = BackgroundHandle::spawn(compactor); - let start = Instant::now(); + for _ in 0..worker_count { + let compactor = BackgroundCompactor::new( + None, + 0.0, + send_file_rx.clone(), + recv_result_tx.clone(), + ); + tasks.push(BackgroundHandle::spawn(compactor)); + } - let mut folder = self.info.take().expect("fileinfo"); + drop(send_file_rx); + drop(recv_result_tx); + + let start = Instant::now(); let summary = folder.summary(); let total_files = folder.len(FileKind::Compressed); let total_bytes = summary.compressed.logical_size; - let mut done_files = 0usize; + let mut expanded_files = 0usize; let mut done_bytes = 0u64; + let mut pending: HashMap = HashMap::with_capacity(worker_count); + let mut failed = Vec::new(); + let mut no_more_files = false; let mut last_update = Instant::now(); let mut paused = false; let mut stopped = false; - let old_size = folder.physical_size; self.gui.compacting(); - self.gui.status("Expanding".to_string(), Some(0.0)); + self.gui.status("Expanding", Some(0.0)); loop { - while paused && !stopped { - self.gui.status( - "Paused".to_string(), - Some(progress(done_bytes, total_bytes)), - ); - self.gui.summary(folder.summary()); + while !paused && !stopped && pending.len() < worker_count && !no_more_files { + if let Some(fi) = folder.pop(FileKind::Compressed) { + let path = folder.path.join(&fi.path); + let job = CompressionJob { + path: path.clone(), + content_len: fi.content_len, + modified_time: fi.modified_time, + estimate_valid: fi.estimate_valid, + estimated_ratio: fi.estimated_ratio, + }; + + if send_file.send(job).is_err() { + folder.push(FileKind::Compressed, fi); + stopped = true; + break; + } + + pending.insert(path, fi); + } else { + no_more_files = true; + } + } - match self.msg.recv() { - Ok(GuiRequest::Pause) => paused = true, - Ok(GuiRequest::Resume) => { + if (no_more_files || stopped) && pending.is_empty() { + break; + } + + loop { + match self.msg.try_recv() { + Ok(GuiRequest::Pause) if !paused && !stopped => { + paused = true; + self.gui.paused(); self.gui.status( - "Expanding".to_string(), + if pending.is_empty() { + "Paused" + } else { + "Pausing after active files finish" + }, Some(progress(done_bytes, total_bytes)), ); - self.gui.resumed(); + } + Ok(GuiRequest::Resume) if paused && !stopped => { paused = false; - last_update = Instant::now(); + self.gui.resumed(); + self.gui + .status("Expanding", Some(progress(done_bytes, total_bytes))); } - Ok(GuiRequest::Stop) => { + Ok(GuiRequest::Stop) if !stopped => { stopped = true; - break; + self.gui.status( + if pending.is_empty() { + "Stopping" + } else { + "Stopping after active files finish" + }, + Some(progress(done_bytes, total_bytes)), + ); } Ok(_) => (), - Err(_) => { - stopped = true; - break; - } + Err(_) => break, } } - if stopped { - break; - } - - if last_update.elapsed() > Duration::from_millis(50) { - self.gui.status( - "Expanding".to_string(), - Some(progress(done_bytes, total_bytes)), - ); - last_update = Instant::now(); - self.gui.summary(folder.summary()); - } + match recv_result.recv_timeout(Duration::from_millis(25)) { + Ok((path, result)) => { + let Some(mut fi) = pending.remove(&path) else { + continue; + }; + let logical_size = fi.logical_size; + let display_path = fi.path.clone(); + done_bytes = done_bytes.saturating_add(logical_size); - if let Some(mut fi) = folder.pop(FileKind::Compressed) { - let logical_size = fi.logical_size; - send_file - .send(CompressionJob { - path: folder.path.join(&fi.path), - content_len: fi.content_len, - modified_time: fi.modified_time, - estimate_valid: fi.estimate_valid, - estimated_ratio: fi.estimated_ratio, - }) - .expect("send_file"); - - let mut waiting = false; - loop { - if let Ok((path, result)) = recv_result.recv_timeout(Duration::from_millis(25)) { - done_files += 1; - done_bytes = done_bytes.saturating_add(logical_size); - - match result { - Ok(_) => { - fi.physical_size = path.size_on_disk().unwrap_or(fi.logical_size); - fi.estimated_physical_size = fi.physical_size; - if let Ok(metadata) = std::fs::metadata(&path) { - use std::os::windows::fs::MetadataExt; - fi.content_len = metadata.len(); - fi.logical_size = metadata.len(); - fi.modified_time = metadata.last_write_time(); - } - folder.push(FileKind::Compressible, fi); - } - Err(err) => { - fi.estimated_physical_size = fi.physical_size; - self.gui.status( - format!("Error: {}, {}", err, fi.path.display()), - Some(progress(done_bytes, total_bytes)), - ); - folder.push(FileKind::Skipped, fi); + match result { + Ok(_) => { + expanded_files += 1; + fi.physical_size = path.size_on_disk().unwrap_or(fi.logical_size); + fi.estimated_physical_size = fi.physical_size; + fi.estimate_valid = false; + if let Ok(metadata) = std::fs::metadata(&path) { + use std::os::windows::fs::MetadataExt; + fi.content_len = metadata.len(); + fi.logical_size = metadata.len(); + fi.modified_time = metadata.last_write_time(); } + folder.push(FileKind::Compressible, fi); + } + Err(err) => { + fi.estimated_physical_size = fi.physical_size; + fi.estimate_valid = false; + self.gui.status( + format!("Error: {}, {}", err, display_path.display()), + Some(progress(done_bytes, total_bytes)), + ); + failed.push(fi); } - - break; } - if !waiting && last_update.elapsed() > Duration::from_millis(50) { + if last_update.elapsed() > Duration::from_millis(50) { + last_update = Instant::now(); self.gui.status( - format!("Expanding: {}", fi.path.display()), + if paused { + if pending.is_empty() { + "Paused".to_string() + } else { + "Pausing after active files finish".to_string() + } + } else if stopped { + if pending.is_empty() { + "Stopping".to_string() + } else { + "Stopping after active files finish".to_string() + } + } else { + format!("Expanding: {}", display_path.display()) + }, Some(progress(done_bytes, total_bytes)), ); - last_update = Instant::now(); - waiting = true; - } - match self.msg.try_recv() { - Ok(GuiRequest::Pause) if !paused => { - self.gui.status( - format!("Pausing after {}", fi.path.display()), - Some(progress(done_bytes, total_bytes)), - ); - self.gui.paused(); - paused = true; - } - Ok(GuiRequest::Resume) => { - self.gui.resumed(); - paused = false; - stopped = false; - } - Ok(GuiRequest::Stop) if !stopped => { - self.gui.status( - format!("Stopping after {}", fi.path.display()), - Some(progress(done_bytes, total_bytes)), - ); - stopped = true; + if pending.is_empty() { + self.gui.summary(folder.summary()); } - Ok(_) => (), - Err(_) => (), } } - } else { - break; + Err(RecvTimeoutError::Timeout) => { + if paused + && pending.is_empty() + && last_update.elapsed() > Duration::from_millis(50) + { + last_update = Instant::now(); + self.gui + .status("Paused", Some(progress(done_bytes, total_bytes))); + self.gui.summary(folder.summary()); + } + } + Err(RecvTimeoutError::Disconnected) => { + stopped = true; + break; + } } } drop(send_file); - task.wait(); + for task in tasks { + task.wait(); + } + + for fi in failed { + folder.push(FileKind::Compressed, fi); + } + + let mut exclusion_active = false; + if exclude_after && !stopped { + match add_folder_exclusion(&folder.path) { + Ok(added) => { + exclusion_active = true; + move_compressible_to_skipped(&mut folder); + if added { + self.gui.config(); + } + } + Err(err) => self.gui.error("Unable to add exclusion", err), + } + } let new_size = folder.physical_size; let decimal = config().read().unwrap().current().decimal; @@ -730,14 +968,21 @@ impl Backend { let msg = if stopped { format!( "Stopped after expanding {} of {} files in {:.2?}", - done_files, + expanded_files, total_files, start.elapsed() ) + } else if exclude_after && exclusion_active { + format!( + "Expanded {} files using {} more space and excluded this folder in {:.2?}", + expanded_files, + format_size(wasted, decimal), + start.elapsed() + ) } else { format!( "Expanded {} files using {} more space in {:.2?}", - done_files, + expanded_files, format_size(wasted, decimal), start.elapsed() ) @@ -849,4 +1094,43 @@ mod tests { assert_eq!(3, total); assert_eq!(3, page_items.len()); } -} + + #[test] + fn direct_storage_protection_moves_only_matching_candidates() { + let mut folder = FolderInfo::new(PathBuf::from(r"D:\Games")); + for path in ["Foo\\data.bin", "Bar\\data.bin"] { + folder.push( + FileKind::Compressible, + FileInfo { + path: PathBuf::from(path), + content_len: 8192, + modified_time: 0, + estimate_valid: true, + estimated_ratio: 0.5, + logical_size: 8192, + physical_size: 8192, + estimated_physical_size: 4096, + }, + ); + } + + let protected = protect_direct_storage_candidates( + &mut folder, + &[PathBuf::from(r"D:\Games\Foo")], + ); + assert_eq!(1, protected); + assert_eq!(1, folder.compressible.count); + assert_eq!(1, folder.skipped.count); + assert_eq!(PathBuf::from(r"Bar\data.bin"), folder.compressible.files[0].path); + } + + #[test] + fn decompression_auto_is_capped_but_manual_is_respected() { + // The exact storage result is exercised in folder.rs. This helper only + // adds the decompression-specific Auto cap. + assert_eq!( + 1, + decompression_worker_count_for_path(Path::new(r"Z:\unknown"), 0, true) + ); + } +} \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index be83fb7..883388c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -20,6 +20,10 @@ fn default_hdd_single_thread() -> bool { true } +fn default_protect_direct_storage() -> bool { + true +} + fn default_compression_priority() -> CompressionPriority { CompressionPriority::BelowNormal } @@ -82,6 +86,8 @@ pub struct Config { pub max_threads: usize, #[serde(default = "default_hdd_single_thread")] pub hdd_single_thread: bool, + #[serde(default = "default_protect_direct_storage")] + pub protect_direct_storage: bool, #[serde(default = "default_compression_priority")] pub compression_priority: CompressionPriority, pub excludes: Vec, @@ -95,6 +101,7 @@ impl Default for Config { min_savings_percent: default_min_savings_percent(), max_threads: default_max_threads(), hdd_single_thread: default_hdd_single_thread(), + protect_direct_storage: default_protect_direct_storage(), compression_priority: default_compression_priority(), excludes: vec![ "*:\\Windows*", @@ -188,6 +195,7 @@ fn test_config() { assert_eq!(s.min_savings_percent, 1.0); assert_eq!(s.max_threads, 0); assert!(s.hdd_single_thread); + assert!(s.protect_direct_storage); assert_eq!(s.compression_priority, CompressionPriority::BelowNormal); assert!((s.ratio_limit() - 0.99).abs() < f32::EPSILON); diff --git a/src/directstorage.rs b/src/directstorage.rs new file mode 100644 index 0000000..7908b01 --- /dev/null +++ b/src/directstorage.rs @@ -0,0 +1,304 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use walkdir::WalkDir; + +const DIRECT_STORAGE_DLLS: &[&str] = &["dstorage.dll", "dstoragecore.dll"]; + +// These are library/container directories, not individual games. When a +// DirectStorage runtime is found below one of these, protect only the immediate +// game/install child instead of excluding the entire library. +const LIBRARY_CONTAINERS: &[&str] = &[ + "steamapps", + "xboxgames", + "windowsapps", + "program files", + "program files (x86)", + "program files (arm)", + "programdata", + "epic games", + "epicgames", + "egs", + "gog galaxy", + "gog galaxy games", + "gog games", + "ubisoft", + "ubisoft game launcher", + "origin games", + "origin", + "ea games", + "electronic arts", + "battle.net", + "riot games", + "games", + "my games", + "common files", + "amazon games", +]; + +const HARD_TOO_BROAD: &[&str] = &["windows", "users", "appdata"]; + +fn lower_name(path: &Path) -> String { + path.file_name() + .map(|name| name.to_string_lossy().to_ascii_lowercase()) + .unwrap_or_default() +} + +fn normalized(path: &Path) -> String { + let value = path + .to_string_lossy() + .replace('/', "\\") + .trim_end_matches('\\') + .to_ascii_lowercase(); + + // All production paths come from the folder picker and are absolute, but + // normalising the separator after a drive prefix also makes internal path + // comparisons deterministic for PathBuf values assembled in tests/helpers. + if value.as_bytes().get(1) == Some(&b':') && value.len() > 2 { + let rest = value[2..].trim_start_matches('\\'); + if rest.is_empty() { + value[..2].to_string() + } else { + format!("{}\\{}", &value[..2], rest) + } + } else { + value + } +} + +pub fn is_under(path: &Path, root: &Path) -> bool { + let path = normalized(path); + let root = normalized(root); + path == root + || path + .strip_prefix(&root) + .map(|rest| rest.starts_with('\\')) + .unwrap_or(false) +} + +pub fn is_direct_storage_runtime(path: &Path) -> bool { + let name = lower_name(path); + DIRECT_STORAGE_DLLS.iter().any(|candidate| *candidate == name) +} + +fn is_drive_root(path: &Path) -> bool { + path.parent().map(|parent| parent == path).unwrap_or(true) +} + +fn is_library_container(path: &Path) -> bool { + let name = lower_name(path); + if LIBRARY_CONTAINERS.iter().any(|candidate| *candidate == name) { + return true; + } + + let parent = path.parent().map(lower_name).unwrap_or_default(); + (name == "common" && parent == "steamapps") + || (name == "library" && parent == "amazon games") +} + +fn too_wide_to_skip(path: &Path) -> bool { + let name = lower_name(path); + is_drive_root(path) + || is_library_container(path) + || HARD_TOO_BROAD.iter().any(|candidate| *candidate == name) +} + +fn unreal_project_root(dll_parent: &Path) -> Option { + for current in dll_parent.ancestors() { + if !lower_name(current).eq_ignore_ascii_case("directstorage") { + continue; + } + + let windows = current.parent()?; + let third_party = windows.parent()?; + let binaries = third_party.parent()?; + let engine = binaries.parent()?; + + if lower_name(windows) == "windows" + && lower_name(third_party) == "thirdparty" + && lower_name(binaries) == "binaries" + && lower_name(engine) == "engine" + { + return engine.parent().map(Path::to_path_buf); + } + } + + None +} + +fn safe_root(candidate: PathBuf, dll_path: &Path, scan_root: &Path) -> PathBuf { + if !is_under(&candidate, scan_root) || too_wide_to_skip(&candidate) { + return dll_path + .parent() + .filter(|parent| is_under(parent, scan_root)) + .unwrap_or(scan_root) + .to_path_buf(); + } + + candidate +} + +fn infer_game_root( + dll_path: &Path, + scan_root: &Path, + exe_dirs: &HashSet, +) -> PathBuf { + let dll_parent = dll_path.parent().unwrap_or(scan_root); + + if let Some(root) = unreal_project_root(dll_parent) { + return safe_root(root, dll_path, scan_root); + } + + let mut current = dll_parent.to_path_buf(); + let mut below: Option = None; + let mut nearest_exe: Option = None; + + loop { + if nearest_exe.is_none() && exe_dirs.contains(&normalized(¤t)) { + nearest_exe = Some(current.clone()); + } + + if is_library_container(¤t) { + if let Some(game) = below { + return safe_root(game, dll_path, scan_root); + } + break; + } + + if HARD_TOO_BROAD + .iter() + .any(|candidate| *candidate == lower_name(¤t)) + { + break; + } + + if normalized(¤t) == normalized(scan_root) { + break; + } + + let Some(parent) = current.parent().map(Path::to_path_buf) else { + break; + }; + if parent == current { + break; + } + + below = Some(current); + current = parent; + } + + // If the user selected a single game/install directory, protecting that + // whole target is safer and clearer than guessing at a nested Binaries/bin + // directory. For broad library roots, prefer the closest directory that + // actually contains an executable, then fall back to the DLL's directory. + if !too_wide_to_skip(scan_root) { + return scan_root.to_path_buf(); + } + + if let Some(exe) = nearest_exe { + return safe_root(exe, dll_path, scan_root); + } + + safe_root(dll_parent.to_path_buf(), dll_path, scan_root) +} + +fn collapse_roots(mut roots: Vec) -> Vec { + roots.sort_by_key(|path| path.components().count()); + let mut kept: Vec = Vec::new(); + + for root in roots { + if kept.iter().any(|existing| is_under(&root, existing)) { + continue; + } + kept.push(root); + } + + kept +} + +pub fn discover_direct_storage_roots(scan_root: &Path) -> Vec { + let mut dlls = Vec::new(); + let mut exe_dirs = HashSet::new(); + + for entry in WalkDir::new(scan_root) + .follow_links(false) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + { + let path = entry.path(); + if is_direct_storage_runtime(path) { + dlls.push(path.to_path_buf()); + } + + if path + .extension() + .map(|ext| ext.to_string_lossy().eq_ignore_ascii_case("exe")) + .unwrap_or(false) + { + if let Some(parent) = path.parent() { + exe_dirs.insert(normalized(parent)); + } + } + } + + collapse_roots( + dlls + .iter() + .map(|dll| infer_game_root(dll, scan_root, &exe_dirs)) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn steam_unreal_runtime_maps_to_game_root() { + let scan = Path::new(r"D:\SteamLibrary\steamapps\common"); + let dll = Path::new( + r"D:\SteamLibrary\steamapps\common\ExampleGame\Engine\Binaries\ThirdParty\Windows\DirectStorage\x64\dstorage.dll", + ); + let root = infer_game_root(dll, scan, &HashSet::new()); + assert_eq!( + normalized(&root), + normalized(Path::new(r"D:\SteamLibrary\steamapps\common\ExampleGame")) + ); + } + + #[test] + fn epic_library_runtime_maps_to_immediate_game_child() { + let scan = Path::new(r"D:\Epic Games"); + let dll = Path::new(r"D:\Epic Games\ExampleGame\bin\dstorage.dll"); + let root = infer_game_root(dll, scan, &HashSet::new()); + assert_eq!( + normalized(&root), + normalized(Path::new(r"D:\Epic Games\ExampleGame")) + ); + } + + #[test] + fn single_game_target_protects_the_selected_game() { + let scan = Path::new(r"D:\Games\ExampleGame"); + let dll = Path::new(r"D:\Games\ExampleGame\bin\dstoragecore.dll"); + let root = infer_game_root(dll, scan, &HashSet::new()); + assert_eq!(normalized(&root), normalized(scan)); + } + + #[test] + fn nested_roots_are_collapsed() { + let roots = collapse_roots(vec![ + PathBuf::from(r"D:\Games\Foo\Engine"), + PathBuf::from(r"D:\Games\Foo"), + PathBuf::from(r"D:\Games\Bar"), + ]); + assert_eq!(2, roots.len()); + assert!(roots + .iter() + .any(|path| normalized(path) == normalized(Path::new(r"D:\Games\Foo")))); + assert!(roots + .iter() + .any(|path| normalized(path) == normalized(Path::new(r"D:\Games\Bar")))); + } +} diff --git a/src/gui.rs b/src/gui.rs index dc77988..c129d3d 100644 --- a/src/gui.rs +++ b/src/gui.rs @@ -25,6 +25,7 @@ pub enum GuiRequest { min_savings_percent: f32, max_threads: usize, hdd_single_thread: bool, + protect_direct_storage: bool, compression_priority: String, excludes: String, }, @@ -32,6 +33,7 @@ pub enum GuiRequest { ChooseFolder, Compress, Decompress, + DecompressAndExclude, ViewCompressed { view: String, query: String, @@ -73,6 +75,7 @@ pub enum GuiResponse { min_savings_percent: f32, max_threads: usize, hdd_single_thread: bool, + protect_direct_storage: bool, compression_priority: String, excludes: String, }, @@ -145,6 +148,7 @@ impl GuiWrapper { min_savings_percent: s.min_savings_percent, max_threads: s.max_threads, hdd_single_thread: s.hdd_single_thread, + protect_direct_storage: s.protect_direct_storage, compression_priority: s.compression_priority.to_string(), excludes: s.excludes.join("\n"), }); @@ -257,6 +261,7 @@ pub fn spawn_gui() { min_savings_percent, max_threads, hdd_single_thread, + protect_direct_storage, compression_priority, excludes, }) => { @@ -266,6 +271,7 @@ pub fn spawn_gui() { min_savings_percent, max_threads, hdd_single_thread, + protect_direct_storage, compression_priority: compression_priority.parse().unwrap_or_default(), excludes: excludes.split('\n').map(str::to_owned).collect(), }; @@ -285,6 +291,7 @@ pub fn spawn_gui() { min_savings_percent: s.min_savings_percent, max_threads: s.max_threads, hdd_single_thread: s.hdd_single_thread, + protect_direct_storage: s.protect_direct_storage, compression_priority: s.compression_priority.to_string(), excludes: s.excludes.join("\n"), }, @@ -312,6 +319,7 @@ pub fn spawn_gui() { min_savings_percent: s.min_savings_percent, max_threads: s.max_threads, hdd_single_thread: s.hdd_single_thread, + protect_direct_storage: s.protect_direct_storage, compression_priority: s.compression_priority.to_string(), excludes: s.excludes.join("\n"), }, diff --git a/src/main.rs b/src/main.rs index 903ec3e..1ab64ce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod compact; mod compression; mod config; mod console; +mod directstorage; mod folder; mod gui; mod persistence; diff --git a/src/ui/app.js b/src/ui/app.js index 25de75e..3e242d1 100644 --- a/src/ui/app.js +++ b/src/ui/app.js @@ -170,6 +170,10 @@ var Action = (function() { external.invoke(JSON.stringify({ type: 'Decompress' })); }, + decompress_and_exclude: function() { + external.invoke(JSON.stringify({ type: 'DecompressAndExclude' })); + }, + view_compressed: function(view, query, page) { external.invoke(JSON.stringify({ type: 'ViewCompressed', @@ -204,50 +208,89 @@ var Action = (function() { var Response = (function() { "use strict"; + var pendingStatus = null; + var pendingSummary = null; + var flushTimer = null; + + var apply = function(msg) { + switch(msg.type) { + case "Config": + Gui.set_decimal(msg.decimal); + Gui.set_compression(msg.compression); + Gui.set_min_savings(msg.min_savings_percent); + Gui.set_max_threads(msg.max_threads); + Gui.set_compression_priority(msg.compression_priority); + Gui.set_hdd_single_thread(msg.hdd_single_thread); + Gui.set_protect_direct_storage(msg.protect_direct_storage); + Gui.set_excludes(msg.excludes); + break; + + case "Folder": + Gui.set_folder(msg.path); + break; + + case "Version": + Gui.version(msg.date, msg.version); + break; + + case "Status": + Gui.set_status(msg.status, msg.pct); + break; + + case "Error": + window.alert(msg.title + "\n\n" + msg.message); + break; + + case "Paused": + case "Resumed": + case "Stopped": + case "Scanned": + case "Compacting": + Gui[msg.type.toLowerCase()](); + break; + + case "FolderSummary": + Gui.set_folder_summary(msg.info); + break; + + case "CompressedView": + Gui.set_compressed_view(msg); + break; + } + }; + + var flush = function() { + if (flushTimer !== null) { + clearTimeout(flushTimer); + flushTimer = null; + } + + var status = pendingStatus; + var summary = pendingSummary; + pendingStatus = null; + pendingSummary = null; + + if (status) apply(status); + if (summary) apply(summary); + }; + return { dispatch: function(msg) { - switch(msg.type) { - case "Config": - Gui.set_decimal(msg.decimal); - Gui.set_compression(msg.compression); - Gui.set_min_savings(msg.min_savings_percent); - Gui.set_max_threads(msg.max_threads); - Gui.set_compression_priority(msg.compression_priority); - Gui.set_hdd_single_thread(msg.hdd_single_thread); - Gui.set_excludes(msg.excludes); - break; - - case "Folder": - Gui.set_folder(msg.path); - break; - - case "Version": - Gui.version(msg.date, msg.version); - break; - - case "Status": - Gui.set_status(msg.status, msg.pct); - break; - - case "Error": - window.alert(msg.title + "\n\n" + msg.message); - break; - - case "Paused": - case "Resumed": - case "Stopped": - case "Scanned": - case "Compacting": - Gui[msg.type.toLowerCase()](); - break; - - case "FolderSummary": - Gui.set_folder_summary(msg.info); - break; - - case "CompressedView": - Gui.set_compressed_view(msg); - break; + // High-frequency scan/compression updates are latest-wins. Coalescing + // them avoids repeatedly rebuilding the same DOM within one paint + // window while discrete state changes still arrive immediately. + if (msg.type == "Status") { + pendingStatus = msg; + } else if (msg.type == "FolderSummary") { + pendingSummary = msg; + } else { + flush(); + apply(msg); + return; + } + + if (flushTimer === null) { + flushTimer = setTimeout(flush, 50); } } }; @@ -293,6 +336,7 @@ var Gui = (function() { max_threads: maxThreads, compression_priority: $("#Compression_Priority").val(), hdd_single_thread: $("#HDD_Single_Thread").prop("checked"), + protect_direct_storage: $("#Protect_DirectStorage").prop("checked"), excludes: $("#Excludes").val() }); }); @@ -421,6 +465,10 @@ var Gui = (function() { $("#HDD_Single_Thread").prop("checked", enabled); }, + set_protect_direct_storage: function(enabled) { + $("#Protect_DirectStorage").prop("checked", enabled); + }, + set_excludes: function(excludes) { $("#Excludes").val(excludes); }, @@ -459,6 +507,7 @@ var Gui = (function() { $("#Button_Analyse").hide(); $("#Button_Compress").hide(); $("#Button_Decompress").hide(); + $("#Button_Decompress_Exclude").hide(); $("#Button_View_Compressed").hide(); $("#Command").show(); }, @@ -470,6 +519,7 @@ var Gui = (function() { $("#Button_Analyse").hide(); $("#Button_Compress").hide(); $("#Button_Decompress").hide(); + $("#Button_Decompress_Exclude").hide(); $("#Button_View_Compressed").hide(); }, @@ -501,9 +551,11 @@ var Gui = (function() { if ($("#File_Count_Compressed").text() != "0") { $("#Button_Decompress").show(); + $("#Button_Decompress_Exclude").show(); $("#Button_View_Compressed").show(); } else { $("#Button_Decompress").hide(); + $("#Button_Decompress_Exclude").hide(); $("#Button_View_Compressed").hide(); } }, diff --git a/src/ui/index.html b/src/ui/index.html index afa2205..a0aed23 100644 --- a/src/ui/index.html +++ b/src/ui/index.html @@ -43,6 +43,7 @@

Comp + @@ -56,7 +57,7 @@

Comp
@@ -155,6 +156,14 @@

Compression

Recommended to avoid excessive seeking on mechanical drives. + +