diff --git a/AGENTS.md b/AGENTS.md index 0342b30c..2ae989d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,10 +73,11 @@ and `cargo build --release` directly; CI runs the gate on both systems. folders and depth; see `mft.rs`. That path counts every stream's allocation, alternate data streams included since they go when the file goes, so for a file with alternate streams it can exceed the walk's - number. -2. **Totals are derived, never tracked.** A leaf's `bytes` is the only - input: `tree::aggregate` computes every directory's `bytes`, `files` and - `dirs` from its children, and `own_bytes()`/`own_files()` read the direct + number. An APFS clone family is charged once, like a hardlinked file: + every copy's `st_blocks` counts the blocks they share; see `macos.rs`. +2. **Totals are derived, never tracked.** A leaf's `bytes` and `unshared` + are the only inputs: `tree::aggregate` computes every directory's + `bytes`, `unshared`, `files` and `dirs` from its children, and `own_bytes()`/`own_files()` read the direct children when asked. Hardlink de-duplication zeroes a duplicate leaf's `bytes` while that pass runs; anything that patches a directory's `bytes` directly will be overwritten. `Node` is paid for tens of millions of @@ -102,7 +103,9 @@ and `cargo build --release` directly; CI runs the gate on both systems. 8. **The view transform is the only thing zoom changes.** Layout runs in base-space pixels and is cached; `screen = (base - origin) * scale`. 9. **The status bar never claims a saving it cannot measure.** Projections come - from marked bytes *on the volume the meter measures*, which is the scan's own + from marked *unshared* bytes — an APFS clone's blocks stay while another + copy does, so they are never projected; see `Node::unshared` — + *on the volume the meter measures*, which is the scan's own notion of a volume — the mount *source* — so a mark on a disk mounted inside the tree, which `-X` and the media exception allow, is reported and never added to this volume's free space; see `space::attribution`. The final number diff --git a/README.md b/README.md index bf9bc21f..a1e00515 100644 --- a/README.md +++ b/README.md @@ -104,9 +104,12 @@ What is different from Linux: - **Free space** is what `df` reports. Finder's figure is larger: it counts purgeable space (caches and local snapshots macOS will clear on its own). -- **Cloned files** (copies APFS shares blocks between, as Finder's Duplicate - makes) are each counted in full, so a total can exceed what deleting them - frees. +- **Cloned files** (copies APFS shares blocks between, as `cp -c` and + Finder's Duplicate make) are counted once per family, like hardlinks. Each + copy holds none of those blocks alone, so marking one projects no saving: + deleting it frees nothing while another copy stays. A copy edited after + cloning still shares the rest of its blocks; it projects only what it + holds alone, or nothing if it is 64 KiB or smaller. - **Time Machine's local snapshots** are not files and do not appear; they are part of the gap between the scan and the disk's used space. - **Cloud-only folders** (iCloud Drive, Dropbox and the like, evicted to the @@ -402,7 +405,8 @@ escaped in the prompt, so it cannot pass for another path. - **Disk usage** by default: `st_blocks × 512`, the number `du` reports and the space that actually comes back when a file is deleted. Apparent size (what `ls -l` shows) is one toggle away. -- **Hardlinks once.** Two names for one inode cost one file. +- **Hardlinks once.** Two names for one inode cost one file. On macOS an + APFS clone family counts once too. - **Hidden entries included**, because `~/.cache` is often the biggest thing in a home directory. Symlinks are not followed. - **The projection counts one volume.** The meter shows the scanned volume's @@ -412,7 +416,8 @@ escaped in the prompt, so it cannot pass for another path. subvolume is left out of the projection — removing it gives its space back somewhere else, or, in a snapshot, shares its blocks with the live files — and the panel and the review screen say what was left out. A mark whose - volume cannot be told is not claimed either. + volume cannot be told is not claimed either, and neither are blocks a + marked APFS clone shares with copies that stay. The scan follows [dust](https://github.com/bootandy/dust)'s approach: one rayon scope per root, a completion counter per directory so no directory is built diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs index 7a03e476..4cd1928f 100644 --- a/crates/disktree-app/src/marks.rs +++ b/crates/disktree-app/src/marks.rs @@ -67,12 +67,14 @@ impl Marks { match find(root_path, root, &item.path) { Some(node) => resolved.push(Target { bytes: node.bytes, + unshared: node.unshared, is_dir: node.is_dir(), hidden: is_hidden(&item.path), path: item.path.clone(), }), None => resolved.push(Target { bytes: 0, + unshared: 0, ..item.clone() }), } @@ -133,7 +135,10 @@ mod tests { fn tree() -> Node { let mut root = Node::directory("home"); let mut cache = Node::directory(".cache"); - cache.children.push(file("blob.bin", 900)); + // A clone, mostly its family's: only 300 bytes are its own. + let mut blob = file("blob.bin", 900); + blob.unshared = 300; + cache.children.push(blob); root.children.push(cache); root.children.push(file("notes.bin", 100)); aggregate(&mut root, Metric::Bytes); @@ -144,6 +149,7 @@ mod tests { Target { path: PathBuf::from(path), bytes, + unshared: bytes, is_dir: false, hidden: false, } @@ -178,9 +184,11 @@ mod tests { marks.refresh(root_path, &root, Metric::Bytes); assert_eq!(marks.items()[0].bytes, 900); + assert_eq!(marks.items()[0].unshared, 300, "only what it frees"); assert!(marks.items()[0].is_dir); assert!(marks.items()[0].hidden, "the .cache mark is hidden"); assert_eq!(marks.items()[1].bytes, 0, "a path that no longer exists"); + assert_eq!(marks.items()[1].unshared, 0); } #[test] diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index efb9730a..5ed35bff 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -2174,6 +2174,7 @@ impl Disktree { hidden: is_hidden(&path), path, bytes: node.bytes, + unshared: node.unshared, is_dir: node.is_dir(), }) } diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index 0520e4cb..cf4a561f 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -1201,6 +1201,7 @@ fn the_review_screen_switches_removal_mode(cx: &mut TestAppContext) { app.marks.toggle(disktree_core::removal::Target { path: temp.path().join("junk"), bytes: 300_000, + unshared: 300_000, is_dir: true, hidden: false, }); @@ -1234,6 +1235,7 @@ fn the_review_screen_copies_the_list_as_an_agent_prompt( app.marks.toggle(disktree_core::removal::Target { path: junk.clone(), bytes: 300_000, + unshared: 300_000, is_dir: true, hidden: false, }); @@ -1276,6 +1278,7 @@ fn remote_review_copies_host_guarded_script_and_prompt( app.marks.toggle(disktree_core::removal::Target { path: temp.path().join("junk"), bytes: 300_000, + unshared: 300_000, is_dir: true, hidden: false, }); @@ -1316,6 +1319,7 @@ fn escape_in_the_delete_dialog_cancels(cx: &mut TestAppContext) { app.marks.toggle(disktree_core::removal::Target { path: temp.path().join("junk"), bytes: 300_000, + unshared: 300_000, is_dir: true, hidden: false, }); @@ -1864,6 +1868,65 @@ fn names_at(names: &[String], index: usize) -> String { names.get(index).cloned().unwrap_or_default() } +/// An APFS clone shares every block with its original, so the scan charges +/// the pair once, and marking it projects only what deleting it frees: the +/// meter never promises the shared blocks, and the review and the delete +/// dialog say where they went. +#[cfg(target_os = "macos")] +#[gpui_kit::test] +fn a_marked_apfs_clone_projects_only_what_it_holds_alone( + cx: &mut TestAppContext, +) { + cx.update(gpui_omarchy::init); + let temp = fixture(); + let copied = std::process::Command::new("/bin/cp") + .arg("-c") + .arg(temp.path().join("junk/blob.bin")) + .arg(temp.path().join("keep/copy.bin")) + .status() + .expect("cp"); + if !copied.success() { + eprintln!("skipped: no clones on {}", temp.path().display()); + return; + } + let (view, cx) = view_over(temp.path(), cx); + draw(cx); + let (keep, junk) = read(&view, cx, |app| { + let at = |rel: &str| app.crumbs_for_path(&temp.path().join(rel)); + (at("keep").expect("keep"), at("junk").expect("junk")) + }); + let tree_bytes = read(&view, cx, |app| app.tree().expect("tree").bytes); + assert_eq!(tree_bytes, 1_000 + 200_000 + 100_000 + 300_000, "once"); + + update(&view, cx, |app, cx| { + app.toggle_mark(&keep, cx); + app.toggle_mark(&junk, cx); + }); + let plan = read(&view, cx, Disktree::plan); + assert_eq!(plan.bytes(), 1_000 + 200_000 + 100_000); + assert_eq!(plan.shared(), 200_000, "the family's blocks"); + assert_eq!(plan.frees(), 1_000 + 100_000); + + update(&view, cx, |app, cx| { + app.screen = Screen::Review; + cx.notify(); + }); + draw(cx); + assert_eq!(read(&view, cx, |app| app.screen), Screen::Review); + let said = crate::views::not_counted(&plan).expect("the shared bytes"); + assert!(said.contains("shared with clones"), "{said}"); + update(&view, cx, |app, cx| { + app.removal_mode = RemovalMode::Permanent; + app.commit(cx); + }); + draw(cx); + assert!( + read(&view, cx, |app| app.confirm_open), + "the dialog is drawn" + ); + assert!(temp.path().join("keep/copy.bin").exists(), "only asked"); +} + #[gpui_kit::test] fn marking_a_directory_marks_everything_inside_it(cx: &mut TestAppContext) { cx.update(gpui_omarchy::init); diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rs index 32aedf5f..ad27f8b0 100644 --- a/crates/disktree-app/src/views.rs +++ b/crates/disktree-app/src/views.rs @@ -345,9 +345,19 @@ fn delete_dialog( ), targets => format!("Delete {} items permanently?", targets.len()), }; + // Only what is sure to come back: a clone's blocks stay with its + // family, so deleting it frees what it alone holds. + let shared = if plan.shared() > 0 { + format!( + "; {} more is shared with clones and stays", + human_bytes(plan.shared()) + ) + } else { + String::new() + }; let body = format!( - "This frees {}. Deleted files can\u{2019}t be recovered; move them to the trash if you might need them again.", - human_bytes(plan.bytes()) + "This frees {}{shared}. Deleted files can\u{2019}t be recovered; move them to the trash if you might need them again.", + human_bytes(plan.frees()) ); let confirm = cx.entity().downgrade(); let cancel = confirm.clone(); @@ -2288,11 +2298,16 @@ fn disk_section( } /// What the volume meter leaves out, and why, when it leaves anything out: -/// bytes marked on another volume come back there, and bytes no volume can -/// be found for are not claimed at all. `None` when every marked byte is -/// this volume's, which is the usual case. -fn not_counted(plan: &Plan) -> Option { +/// bytes shared with clones stay on the disk, bytes marked on another +/// volume come back there, and bytes no volume can be found for are not +/// claimed at all. `None` when every marked byte is this volume's own, +/// which is the usual case. +pub fn not_counted(plan: &Plan) -> Option { let mut parts: Vec = Vec::new(); + if plan.shared() > 0 { + parts + .push(format!("{} shared with clones", human_bytes(plan.shared()))); + } if plan.foreign() > 0 { parts.push(format!("{} on other volumes", human_bytes(plan.foreign()))); } diff --git a/crates/disktree-core/src/export.rs b/crates/disktree-core/src/export.rs index 5fba2cd2..2f64af64 100644 --- a/crates/disktree-core/src/export.rs +++ b/crates/disktree-core/src/export.rs @@ -331,6 +331,7 @@ mod tests { Target { path: PathBuf::from(path), bytes, + unshared: bytes, is_dir, hidden: false, } @@ -514,6 +515,7 @@ mod tests { let targets = [Target { path: PathBuf::from("/tmp").join(name), bytes: 1, + unshared: 1, is_dir: false, hidden: false, }]; diff --git a/crates/disktree-core/src/insights.rs b/crates/disktree-core/src/insights.rs index 589029c5..021b6bc7 100644 --- a/crates/disktree-core/src/insights.rs +++ b/crates/disktree-core/src/insights.rs @@ -33,7 +33,9 @@ pub enum Finding { pub struct Candidate { /// Where it is, from the scanned root. pub crumbs: Vec, - /// What clearing it frees. For stale experiments, only the stale ones. + /// What clearing it is sure to free: its unshared bytes, so a folder of + /// APFS clones is not offered for space its copies elsewhere keep. For + /// stale experiments, only the stale ones. pub bytes: u64, pub finding: Finding, } @@ -62,14 +64,14 @@ fn visit( ) { // Nothing beneath a small directory can reach `MIN_BYTES` either, so a // scan of millions of files looks at a few thousand directories. - if !node.is_dir() || node.bytes < MIN_BYTES { + if !node.is_dir() || node.unshared < MIN_BYTES { return; } // Topmost only: everything beneath a reclaimable directory goes with it. if let Some(reason) = node.reclaim { found.push(Candidate { crumbs: crumbs.clone(), - bytes: node.bytes, + bytes: node.unshared, finding: Finding::Reclaimable(reason), }); return; @@ -91,7 +93,7 @@ fn visit( .unwrap_or(now); found.push(Candidate { crumbs: crumbs.clone(), - bytes: node.bytes, + bytes: node.unshared, finding: Finding::Worktrees { count: trees.len(), oldest_days: (now - oldest).max(0) / DAY, @@ -111,7 +113,7 @@ fn visit( && now - child.modified > STALE_DAYS * DAY; if is_stale { stale.0 += 1; - stale.1 += child.bytes; + stale.1 += child.unshared; // A stale experiment is judged whole; its caches go with it. continue; } @@ -215,6 +217,36 @@ mod tests { assert_eq!(found[2].bytes, 4 * GIB, "only the stale experiment counts"); } + /// Worktrees cloned with `cp -c` share nearly every block with the + /// checkout they came from: offered for what deleting them frees, not + /// for what each copy reports. + #[test] + fn a_finding_offers_only_what_clearing_it_frees() { + let mut copy = file("x", 3 * GIB, 41); + copy.unshared = GIB / 2; + let mut twin = file("y", 3 * GIB, 41); + twin.unshared = 0; + let root = scan(dir( + "tobi", + vec![dir( + ".codex", + vec![dir( + "worktrees", + vec![dir("a1", vec![copy]), dir("b2", vec![twin])], + )], + )], + )); + let found = worth_a_look(&root, NOW, 10); + assert_eq!(found.len(), 1); + assert_eq!(found[0].bytes, GIB / 2); + + // All of it shared: nothing to offer at all. + let mut alone = file("blob", 5 * GIB, 1); + alone.unshared = 0; + let cache = scan(dir("tobi", vec![dir(".cache", vec![alone])])); + assert!(worth_a_look(&cache, NOW, 10).is_empty()); + } + #[test] fn documents_are_never_suggested() { let found = worth_a_look(&home(), NOW, 10); diff --git a/crates/disktree-core/src/macos.rs b/crates/disktree-core/src/macos.rs index 6cdac06d..08079bed 100644 --- a/crates/disktree-core/src/macos.rs +++ b/crates/disktree-core/src/macos.rs @@ -5,6 +5,14 @@ //! entry's full path. `getattrlistbulk(2)` returns every entry's size, //! identity, age and flags for a whole buffer of entries, without the kernel //! setting up per-file state for each one as a stat must. +//! +//! It also says what a stat cannot: how much of a file APFS shares with +//! other files. A clone (`cp -c`, Finder's Duplicate) reports every block +//! it shares as its own in `st_blocks`, so three copies of 200 MB read as +//! 600 MB while the disk spends 200 MB, and deleting one gives back +//! nothing. The listing asks for each file's clone family and whether it +//! shares any blocks, so a total can charge the family once and a +//! projection can count only what deleting is sure to free. #![allow( unsafe_code, @@ -18,6 +26,7 @@ use std::io; use std::os::fd::{AsRawFd as _, OwnedFd}; use std::os::unix::ffi::OsStrExt as _; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; use rustix::fs::{AtFlags, FileType, Mode, OFlags}; @@ -45,6 +54,49 @@ const VDIR: u32 = 2; const VLNK: u32 = 5; /// `SF_DATALESS` from . const SF_DATALESS: u32 = 0x4000_0000; +/// Lets the fork attribute word carry `ATTR_CMNEXT_*` attributes. +const FSOPT_ATTR_CMN_EXTENDED: u64 = 0x0000_0020; +/// Read a link at the path itself, not what it leads to. +const FSOPT_NOFOLLOW: u64 = 0x0000_0001; +/// Bytes of the file no other file shares: what deleting it alone is sure +/// to give back. Never asked of a whole listing: APFS walks the file's +/// extents to answer, which doubled the time to list a 3.1 million file +/// `~/Library` (15 s to 31 s, 8 workers). Asked of the few files that +/// share only some of their blocks, which no cheaper attribute measures. +const ATTR_CMNEXT_PRIVATESIZE: u32 = 0x0000_0008; +/// The clone family: every file cloned from one another, unchanged since, +/// has the same one. A write gives the written file a family of its own. +const ATTR_CMNEXT_CLONEID: u32 = 0x0000_0100; +/// `EF_*` flags from , as a `u64`. +const ATTR_CMNEXT_EXT_FLAGS: u32 = 0x0000_0200; +/// How many files are in the clone family. +const ATTR_CMNEXT_CLONE_REFCNT: u32 = 0x0000_1000; +/// Below this, a file that shares some of its blocks is not asked how many +/// are its own, and claims none. In a 3.1 million file `~/Library`, 203,000 +/// files shared some of their blocks; 195,000 of them were at most 64 KiB +/// and held 1.4 GB between them, while the 8,000 larger ones held 17 GB. +/// Asking all of them cost a 15 s scan 2 to 6 s more. +const PRIVATE_SIZE_FLOOR: u64 = 64 << 10; +/// The file shares, or once shared, blocks with another: it was cloned. +const EF_MAY_SHARE_BLOCKS: u64 = 0x0000_0001; +/// The file shares every block it has. +const EF_SHARES_ALL_BLOCKS: u64 = 0x0000_0040; +/// What each listing asks for in the fork word, most first. A kernel older +/// than an attribute refuses the whole request with `EINVAL` rather than +/// leaving it out, so a refused request is retried with less; an APFS +/// volume answers all of them, and others return none. +const SHARING_REQUESTS: [u32; 3] = [ + ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_EXT_FLAGS | ATTR_CMNEXT_CLONE_REFCNT, + ATTR_CMNEXT_EXT_FLAGS, + 0, +]; +/// The first of [`SHARING_REQUESTS`] this kernel accepted, so only the +/// first listing pays for finding out. +static SHARING_LEVEL: AtomicUsize = AtomicUsize::new(0); +/// Set in a clone family's number to keep it apart from inode numbers, +/// which never reach it: the first file of a family shares its number with +/// its own inode, and a file edited away from the family keeps that inode. +const CLONE_FAMILY: u64 = 1 << 63; const COMMON_ATTRIBUTES: u32 = ATTR_CMN_RETURNED_ATTRS | ATTR_CMN_NAME @@ -83,6 +135,23 @@ unsafe extern "C" { attr_buf_size: usize, options: u64, ) -> c_int; + + fn getattrlistat( + dir: c_int, + path: *const std::ffi::c_char, + attr_list: *mut AttrList, + attr_buf: *mut c_void, + attr_buf_size: usize, + options: u64, + ) -> c_int; + + fn getattrlist( + path: *const std::ffi::c_char, + attr_list: *mut AttrList, + attr_buf: *mut c_void, + attr_buf_size: usize, + options: u32, + ) -> c_int; } thread_local! { @@ -116,6 +185,27 @@ pub struct Entry { inode: u64, links: u32, flags: u32, + sharing: Sharing, +} + +/// What a file shares with others, where the file system says. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Sharing { + own: Own, + /// [`ATTR_CMNEXT_CLONEID`], only when the family has another member. + family: Option, +} + +/// How much of a file's allocation is its own. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum Own { + /// All of it: it shares no blocks, or the file system does not say. + #[default] + All, + /// This many bytes; it shares the rest. + Bytes(u64), + /// Some of it: only [`ATTR_CMNEXT_PRIVATESIZE`] can say how much. + Unknown, } impl Entry { @@ -170,9 +260,39 @@ impl Entry { self.device } - /// Whether the file has more than one name. + /// Whether the walk could meet this file's blocks again: under + /// another name, or in a clone. pub const fn shared(&self) -> bool { - self.links > 1 + self.links > 1 || self.sharing.family.is_some() + } + + /// The identity a total charges once: the clone family's when the file + /// has clones, since every member reports the family's blocks, and + /// otherwise [`Self::identity`]. + pub const fn charged_as(&self) -> (u64, u64) { + match self.sharing.family { + Some(family) => (self.device, CLONE_FAMILY | family), + None => self.identity(), + } + } + + /// Whether only [`ATTR_CMNEXT_PRIVATESIZE`] can say how much of the file + /// is its own, and the answer is worth what asking costs. + fn needs_private_size(&self) -> bool { + self.sharing.own == Own::Unknown && self.allocated > PRIVATE_SIZE_FLOOR + } + + /// The part of `size` no other file shares, so deleting + /// the file is sure to free it; all of `size` where the file system + /// does not say. + pub fn unshared(&self, size: u64) -> u64 { + match self.sharing.own { + Own::All => size, + Own::Bytes(own) => own.min(size), + // Shares some blocks, and nobody could say how many are its + // own: claim none rather than claim what may not come back. + Own::Unknown => 0, + } } /// Whether this is a directory iCloud Drive or a File Provider has @@ -210,6 +330,7 @@ impl Entry { inode: stat.st_ino, links: u32::from(stat.st_nlink), flags: stat.st_flags, + sharing: Sharing::default(), } } } @@ -226,6 +347,8 @@ pub struct ReadDir { /// Where the next of them starts. offset: usize, done: bool, + /// The [`SHARING_REQUESTS`] entry this listing asks with. + level: usize, } /// List `dir`, with each entry's size, identity and age. A link at `dir` @@ -245,6 +368,7 @@ pub fn read_dir(dir: &Path) -> io::Result { remaining: 0, offset: 0, done: false, + level: SHARING_LEVEL.load(Ordering::Relaxed), }) } @@ -297,38 +421,24 @@ impl Iterator for ReadDir { impl ReadDir { /// Ask the kernel for the next buffer of records; how many it wrote. fn fill(&mut self) -> io::Result { + let fd = self.fd.as_raw_fd(); let buffer = self.buffer.as_mut().expect("held until drop"); - let mut request = AttrList { - bitmapcount: ATTR_BIT_MAP_COUNT, - reserved: 0, - commonattr: COMMON_ATTRIBUTES, - volattr: 0, - dirattr: 0, - fileattr: FILE_ATTRIBUTES, - forkattr: 0, - }; - loop { + let common = COMMON_ATTRIBUTES; + request(&mut self.level, &SHARING_LEVEL, common, |list, options| { // SAFETY: the descriptor is an open directory owned by `self`; - // `request` is a valid attrlist for the call's duration; and the + // `list` is a valid attrlist for the call's duration; and the // pointer and length describe `buffer`, which is writable for // that many bytes and not otherwise borrowed during the call. - let count = unsafe { + unsafe { getattrlistbulk( - self.fd.as_raw_fd(), - &raw mut request, + fd, + list, buffer.as_mut_ptr().cast(), BUFFER_BYTES, - 0, + options, ) - }; - if let Ok(count) = u32::try_from(count) { - return Ok(count); - } - let error = io::Error::last_os_error(); - if error.kind() != io::ErrorKind::Interrupted { - return Err(error); } - } + }) } /// The buffer as the bytes the kernel wrote. @@ -345,40 +455,30 @@ impl ReadDir { } } - /// The entry at `self.offset` and the record's length. Every length and - /// offset the kernel wrote is checked against the buffer before it is - /// used. - /// - /// Layout per getattrlist(2): a `u32` length, the returned - /// `attribute_set_t`, `ATTR_CMN_ERROR` when it is set, then every other - /// attribute in bit order, each packed at 4-byte alignment. + /// The entry at `self.offset` and the record's length. The length is + /// checked against the buffer, and [`parse`] checks the rest. fn record(&self) -> io::Result<(io::Result, usize)> { let bytes = self.bytes(); let length = read_u32(bytes, self.offset)? as usize; let record = bytes .get(self.offset..self.offset + length) .ok_or_else(malformed)?; - let mut cursor = Cursor { record, at: 4 }; - let common = cursor.u32()?; - cursor.at += 8; - let file = cursor.u32()?; - cursor.at += 4; - let error = if common & ATTR_CMN_ERROR == 0 { - 0 - } else { - cursor.u32()? - }; - let name = cursor.name()?; - if error != 0 { - let error = io::Error::from_raw_os_error(error.cast_signed()); - return Ok((Err(error), length)); - } - let entry = if common & MEASURED == MEASURED { - cursor.entry(file, name)? - } else { - None + let entry = match parse(record)? { + Parsed::Failed(error) => Err(error), + Parsed::Partial(name) => self.stat(name), + Parsed::Measured(mut entry) => { + if entry.needs_private_size() { + let name = + std::ffi::CString::new(entry.file_name().as_bytes()); + if let Ok(name) = name { + entry.sharing.own = + private_size(self.fd.as_raw_fd(), &name, false); + } + } + Ok(entry) + } }; - Ok((entry.map_or_else(|| self.stat(name), Ok), length)) + Ok((entry, length)) } /// An entry the file system described only in part, read the ordinary @@ -390,6 +490,210 @@ impl ReadDir { } } +/// Make one attribute request with the most sharing attributes this kernel +/// takes, from `level` on, and leave `level` at the request that worked, +/// which `remembered` learns too. `call` makes the request and returns +/// what the system call did; the answer is how many records it wrote. +fn request( + level: &mut usize, + remembered: &AtomicUsize, + common: u32, + mut call: impl FnMut(&mut AttrList, u64) -> c_int, +) -> io::Result { + loop { + let sharing = SHARING_REQUESTS[*level]; + let mut attributes = AttrList { + bitmapcount: ATTR_BIT_MAP_COUNT, + reserved: 0, + commonattr: common, + volattr: 0, + dirattr: 0, + fileattr: FILE_ATTRIBUTES, + forkattr: sharing, + }; + let options = if sharing == 0 { + 0 + } else { + FSOPT_ATTR_CMN_EXTENDED + }; + if let Ok(count) = u32::try_from(call(&mut attributes, options)) { + // Remembered only once a smaller request has worked, so a + // directory that refuses for its own reasons cannot cost every + // later listing its sharing. + remembered.fetch_max(*level, Ordering::Relaxed); + return Ok(count); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc_einval()) + && *level + 1 < SHARING_REQUESTS.len() + { + *level += 1; + } else if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } +} + +/// `path` measured as a listing measures an entry, following a link at +/// `path` itself: for a file reached through a followed link, which no +/// listing of its own directory measured, so a clone met that way is still +/// charged as its family. +pub fn stat(path: &Path) -> io::Result { + let path_c = std::ffi::CString::new(path.as_os_str().as_bytes()) + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + // A name is at most 255 UTF-16 units, so 765 bytes of UTF-8; the rest + // is a few dozen bytes of attributes. + let mut buffer = [0_u64; 128]; + let mut level = SHARING_LEVEL.load(Ordering::Relaxed); + // `ATTR_CMN_ERROR` is for bulk listings only; one path reports its + // error as the call's. + request( + &mut level, + &SHARING_LEVEL, + COMMON_ATTRIBUTES & !ATTR_CMN_ERROR, + |attributes, options| { + // SAFETY: `path_c` is a NUL-terminated path; `attributes` is a + // valid attrlist for the call's duration; and the pointer and + // length describe `buffer`, writable for that many bytes and + // not otherwise borrowed during the call. The options fit: + // only the low bits are flags. + unsafe { + getattrlist( + path_c.as_ptr(), + attributes, + buffer.as_mut_ptr().cast(), + size_of_val(&buffer), + options as u32, + ) + } + }, + )?; + // SAFETY: as in `ReadDir::bytes`: `u64`s read as their bytes. + let bytes = unsafe { + std::slice::from_raw_parts( + buffer.as_ptr().cast::(), + size_of_val(&buffer), + ) + }; + let length = read_u32(bytes, 0)? as usize; + let record = bytes.get(..length).ok_or_else(malformed)?; + match parse(record)? { + Parsed::Failed(error) => Err(error), + Parsed::Partial(name) => rustix::fs::stat(path) + .map(|stat| Entry::from_stat(name, &stat)) + .map_err(Into::into), + Parsed::Measured(mut entry) => { + if entry.needs_private_size() { + let cwd = rustix::fs::CWD.as_raw_fd(); + entry.sharing.own = private_size(cwd, &path_c, true); + } + Ok(entry) + } + } +} + +/// [`ATTR_CMNEXT_PRIVATESIZE`] of `name` in the directory `dir`, for a file +/// that shares some of its blocks; following a link at `name` itself when +/// `follow` says. [`Own::Unknown`] when the file system will not say. +fn private_size(dir: c_int, name: &std::ffi::CStr, follow: bool) -> Own { + let mut attributes = AttrList { + bitmapcount: ATTR_BIT_MAP_COUNT, + reserved: 0, + commonattr: ATTR_CMN_RETURNED_ATTRS, + volattr: 0, + dirattr: 0, + fileattr: 0, + forkattr: ATTR_CMNEXT_PRIVATESIZE, + }; + // The length, the returned attributes and the size. + let mut buffer = [0_u64; 5]; + let options = + FSOPT_ATTR_CMN_EXTENDED | if follow { 0 } else { FSOPT_NOFOLLOW }; + loop { + // SAFETY: `dir` is an open directory or `AT_FDCWD`, `name` is + // NUL-terminated, `attributes` is a valid attrlist for the call's + // duration, and the pointer and length describe `buffer`, writable + // for that many bytes and not otherwise borrowed during the call. + let status = unsafe { + getattrlistat( + dir, + name.as_ptr(), + &raw mut attributes, + buffer.as_mut_ptr().cast(), + size_of_val(&buffer), + options, + ) + }; + if status == 0 { + break; + } + if io::Error::last_os_error().kind() != io::ErrorKind::Interrupted { + return Own::Unknown; + } + } + // SAFETY: as in `ReadDir::bytes`: `u64`s read as their bytes. + let record = unsafe { + std::slice::from_raw_parts( + buffer.as_ptr().cast::(), + size_of_val(&buffer), + ) + }; + // The returned fork word follows the length and four other words, and + // the size follows the returned attributes. + let mut cursor = Cursor { record, at: 20 }; + match (cursor.u32(), cursor.u64()) { + (Ok(fork), Ok(own)) if fork & ATTR_CMNEXT_PRIVATESIZE != 0 => { + Own::Bytes(own) + } + _ => Own::Unknown, + } +} + +/// What one record says. +enum Parsed<'a> { + /// The entry could not be read. + Failed(io::Error), + /// The file system left out something the walk needs: a directory, or + /// a file it measured only in part. The name, to read it another way. + Partial(&'a [u8]), + Measured(Entry), +} + +/// Read one record, every length and offset checked against it first. +/// +/// Layout per getattrlist(2): a `u32` length, the returned +/// `attribute_set_t`, `ATTR_CMN_ERROR` when it is set, then every other +/// attribute in bit order, each packed at 4-byte alignment. +fn parse(record: &[u8]) -> io::Result> { + let mut cursor = Cursor { record, at: 4 }; + let common = cursor.u32()?; + // The volume and directory words: nothing asked for there. + cursor.at += 8; + let file = cursor.u32()?; + let fork = cursor.u32()?; + let error = if common & ATTR_CMN_ERROR == 0 { + 0 + } else { + cursor.u32()? + }; + let name = cursor.name()?; + if error != 0 { + let error = io::Error::from_raw_os_error(error.cast_signed()); + return Ok(Parsed::Failed(error)); + } + let entry = if common & MEASURED == MEASURED { + cursor.entry(file, fork, name)? + } else { + None + }; + Ok(entry.map_or(Parsed::Partial(name), Parsed::Measured)) +} + +/// `EINVAL`, which rustix spells for its own error type only. +const fn libc_einval() -> i32 { + rustix::io::Errno::INVAL.raw_os_error() +} + fn malformed() -> io::Error { io::Error::other("the kernel returned a malformed record") } @@ -448,7 +752,12 @@ impl<'a> Cursor<'a> { /// The attributes after the name, every one of them asked for; `None` /// for a file whose sizes the file system left out. - fn entry(&mut self, file: u32, name: &[u8]) -> io::Result> { + fn entry( + &mut self, + file: u32, + fork: u32, + name: &[u8], + ) -> io::Result> { let device = self.u32()?.cast_signed(); let kind = match self.u32()? { VDIR => Kind::Directory, @@ -472,6 +781,35 @@ impl<'a> Cursor<'a> { } else { return Ok(None); }; + // After the file attributes, in bit order, whichever came back. + let clone = if fork & ATTR_CMNEXT_CLONEID == 0 { + None + } else { + Some(self.u64()?) + }; + let shares = if fork & ATTR_CMNEXT_EXT_FLAGS == 0 { + 0 + } else { + self.u64()? + }; + let members = if fork & ATTR_CMNEXT_CLONE_REFCNT == 0 { + 0 + } else { + self.u32()? + }; + // A file never cloned counts no members, and one whose clones are + // gone counts one: either way there is nothing to charge once. + let family = clone.filter(|_| members > 1); + // A family's members have the same blocks, so none has any of its + // own. A file edited away from its family keeps sharing the rest, + // and only its private size says how much is its own. + let own = if shares & EF_MAY_SHARE_BLOCKS == 0 { + Own::All + } else if family.is_some() || shares & EF_SHARES_ALL_BLOCKS != 0 { + Own::Bytes(0) + } else { + Own::Unknown + }; let (name, exact) = Entry::named(name); Ok(Some(Entry { name, @@ -484,12 +822,44 @@ impl<'a> Cursor<'a> { inode, links, flags, + sharing: Sharing { own, family }, })) } } +/// What tests of clones need, here and in the scan's tests. +#[cfg(test)] +pub mod testing { + use std::path::Path; + + /// Whether `dir` is on APFS, the only file system here with clones. + pub fn on_apfs(dir: &Path) -> bool { + rustix::fs::statfs(dir).is_ok_and(|info| { + let name: Vec = info + .f_fstypename + .iter() + .take_while(|&&byte| byte != 0) + .map(|&byte| byte.cast_unsigned()) + .collect(); + name == b"apfs" + }) + } + + /// `cp -c`, which clones with clonefile(2). + pub fn clone_file(from: &Path, to: &Path) { + let status = std::process::Command::new("/bin/cp") + .arg("-c") + .arg(from) + .arg(to) + .status() + .expect("cp"); + assert!(status.success(), "cp -c {}", from.display()); + } +} + #[cfg(test)] mod tests { + use super::testing::{clone_file, on_apfs}; use super::*; use std::fs; use std::os::unix::fs::symlink; @@ -614,6 +984,172 @@ mod tests { } } + fn named<'a>(entries: &'a [Entry], name: &str) -> &'a Entry { + entries + .iter() + .find(|entry| entry.name() == name) + .unwrap_or_else(|| panic!("no entry {name}")) + } + + #[test] + fn a_clone_family_is_one_identity_and_holds_nothing_of_its_own() { + let temp = TempDir::new().expect("tempdir"); + let root = temp.path(); + if !on_apfs(root) { + eprintln!("skipped: {} is not on APFS", root.display()); + return; + } + fs::write(root.join("original"), vec![7_u8; 1 << 20]).expect("write"); + clone_file(&root.join("original"), &root.join("copy")); + fs::write(root.join("plain"), vec![7_u8; 1 << 20]).expect("write"); + + let entries = listed(root); + let (original, copy) = + (named(&entries, "original"), named(&entries, "copy")); + assert_ne!(original.identity(), copy.identity(), "two files"); + assert_eq!(original.charged_as(), copy.charged_as(), "one family"); + assert!(original.shared() && copy.shared()); + for member in [original, copy] { + assert!(member.allocated() >= 1 << 20, "each reports it all"); + assert_eq!( + member.unshared(member.allocated()), + 0, + "every block of {} is the family's", + member.name() + ); + } + let plain = named(&entries, "plain"); + assert!(!plain.shared()); + assert_eq!(plain.charged_as(), plain.identity()); + assert_eq!(plain.unshared(plain.allocated()), plain.allocated()); + + // A write gives the copy blocks of its own and a family of its own; + // the original, alone in the old one, is charged as itself again. + let mut file = fs::OpenOptions::new() + .write(true) + .open(root.join("copy")) + .expect("open"); + std::io::Write::write_all(&mut file, &vec![9_u8; 64 << 10]) + .expect("write"); + file.sync_all().expect("sync"); + let entries = listed(root); + let (original, copy) = + (named(&entries, "original"), named(&entries, "copy")); + assert_ne!(original.charged_as(), copy.charged_as()); + assert_eq!(original.charged_as(), original.identity()); + let own = copy.unshared(copy.allocated()); + assert!(own >= 64 << 10, "what was written is its own: {own}"); + assert!(own < copy.allocated(), "the rest is still shared: {own}"); + } + + #[test] + fn a_path_reads_as_its_listing_reads_it() { + let temp = TempDir::new().expect("tempdir"); + let root = temp.path(); + fs::write(root.join("original"), vec![7_u8; 1 << 20]).expect("write"); + fs::write(root.join("edited"), vec![7_u8; 1 << 20]).expect("write"); + if on_apfs(root) { + clone_file(&root.join("original"), &root.join("copy")); + // Shares most of its blocks with the file it was cloned from, + // which only its private size measures. + clone_file(&root.join("edited"), &root.join("source")); + let mut file = fs::OpenOptions::new() + .write(true) + .open(root.join("edited")) + .expect("open"); + std::io::Write::write_all(&mut file, &vec![9_u8; 64 << 10]) + .expect("write"); + file.sync_all().expect("sync"); + } + symlink("original", root.join("to-original")).expect("symlink"); + symlink("edited", root.join("to-edited")).expect("symlink"); + + let entries = listed(root); + for (name, link) in + [("original", "to-original"), ("edited", "to-edited")] + { + let listed = named(&entries, name); + for path in [root.join(name), root.join(link)] { + let stated = stat(&path).expect("stat"); + assert_eq!(stated.kind(), Kind::File, "{}", path.display()); + assert_eq!(stated.identity(), listed.identity()); + assert_eq!(stated.charged_as(), listed.charged_as()); + assert_eq!(stated.allocated(), listed.allocated()); + assert_eq!( + stated.unshared(stated.allocated()), + listed.unshared(listed.allocated()), + "{}", + path.display() + ); + } + } + if on_apfs(root) { + let edited = named(&entries, "edited"); + let own = edited.unshared(edited.allocated()); + assert!((64 << 10..edited.allocated()).contains(&own), "{own}"); + } + let error = stat(&root.join("gone")).expect_err("missing"); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } + + /// A kernel that knows only some of the sharing attributes refuses a + /// request for the rest with `EINVAL`; the request is made again with + /// less, and the level that worked is remembered. An attribute bit no + /// kernel knows stands in for one this kernel does not. + #[test] + fn a_refused_sharing_request_is_made_again_with_less() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("file"); + fs::write(&path, b"data").expect("write"); + let path_c = + std::ffi::CString::new(path.as_os_str().as_bytes()).expect("path"); + let mut buffer = [0_u64; 128]; + let mut call = |list: &mut AttrList, options: u64| { + if list.forkattr & ATTR_CMNEXT_CLONE_REFCNT != 0 { + list.forkattr |= 0x8000_0000; + } + // SAFETY: as in `stat`. + unsafe { + getattrlist( + path_c.as_ptr(), + list, + buffer.as_mut_ptr().cast(), + size_of_val(&buffer), + options as u32, + ) + } + }; + let remembered = AtomicUsize::new(0); + let mut level = 0; + let common = COMMON_ATTRIBUTES & !ATTR_CMN_ERROR; + request(&mut level, &remembered, common, &mut call).expect("less"); + assert_eq!(level, 1, "the sharing flags alone"); + assert_eq!(remembered.load(Ordering::Relaxed), 1); + + // Any other failure is the answer, not a reason to ask for less. + let missing = std::ffi::CString::new( + temp.path().join("gone").as_os_str().as_bytes(), + ) + .expect("path"); + let mut level = 0; + let error = + request(&mut level, &remembered, common, |list, options| { + // SAFETY: as in `stat`. + unsafe { + getattrlist( + missing.as_ptr(), + list, + buffer.as_mut_ptr().cast(), + size_of_val(&buffer), + options as u32, + ) + } + }) + .expect_err("missing"); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + assert_eq!(level, 0); + } + #[test] fn a_missing_directory_is_an_error() { let temp = TempDir::new().expect("tempdir"); diff --git a/crates/disktree-core/src/removal.rs b/crates/disktree-core/src/removal.rs index 60277529..c4d0ed97 100644 --- a/crates/disktree-core/src/removal.rs +++ b/crates/disktree-core/src/removal.rs @@ -28,8 +28,12 @@ use std::thread; #[derive(Clone, Debug, PartialEq, Eq)] pub struct Target { pub path: PathBuf, - /// Bytes measured when the path was marked; used for the projection. + /// Bytes measured when the path was marked: what it spends on disk. pub bytes: u64, + /// The part of `bytes` no other file shares, measured with them: what + /// removing the path is sure to give back, and so all the projection + /// counts. See [`crate::tree::Node::unshared`]. + pub unshared: u64, pub is_dir: bool, pub hidden: bool, } @@ -78,7 +82,8 @@ pub struct Plan { pub blocked: Vec, /// The scanned root the targets were judged against. pub root: PathBuf, - /// Bytes the targets free on `root`'s volume; see [`Self::reclaim`]. + /// Bytes the targets are sure to free on `root`'s volume; see + /// [`Self::reclaim`]. reclaim: u64, /// Bytes the targets free on other volumes; see [`Self::foreign`]. foreign: u64, @@ -89,13 +94,14 @@ impl Plan { self.targets.iter().map(|target| target.bytes).sum() } - /// Bytes the targets free on the volume `root` is on: the only ones the - /// free space meter may add to that volume's projection. + /// Bytes the targets are sure to free on the volume `root` is on: the + /// only ones the free space meter may add to that volume's projection. /// /// [`Self::bytes`] counts every target, which is what a delete list /// wants. A mark on another volume — `-X` crosses filesystems, and a /// disk mounted under the root can be marked — gives its space back /// there, so projecting it here would lend one disk another's bytes. + /// And only unshared bytes count anywhere: see [`Self::shared`]. pub const fn reclaim(&self) -> u64 { self.reclaim } @@ -105,11 +111,30 @@ impl Plan { self.foreign } - /// Bytes whose volume could not be told: no saving is claimed for them - /// either, since a projection the meter cannot stand behind is worse - /// than a smaller one. + /// Bytes the targets share with other files: an APFS clone's blocks + /// are its family's, so deleting one copy frees none of them. No saving + /// is claimed for them. Deleting every copy does free them, which the + /// measured result then shows; nothing short of the volume's own extent + /// map can say so in advance, since a copy edited away from its family + /// still shares most of its blocks. + pub fn shared(&self) -> u64 { + self.targets + .iter() + .map(|target| target.bytes.saturating_sub(target.unshared)) + .sum() + } + + /// What removing the targets is sure to free, wherever it comes back: + /// [`Self::bytes`] less [`Self::shared`]. + pub fn frees(&self) -> u64 { + self.bytes().saturating_sub(self.shared()) + } + + /// Unshared bytes whose volume could not be told: no saving is claimed + /// for them either, since a projection the meter cannot stand behind + /// is worse than a smaller one. pub fn unattributed(&self) -> u64 { - self.bytes().saturating_sub(self.reclaim + self.foreign) + self.frees().saturating_sub(self.reclaim + self.foreign) } pub const fn is_empty(&self) -> bool { @@ -184,7 +209,7 @@ fn plan_against(targets: &[Target], root: &Path, mounts: &MountTable) -> Plan { // Which volume each target's bytes come back to. Only what comes back to // the scanned volume may be shown against its free space. for target in &plan.targets { - let bytes = target.bytes; + let bytes = target.unshared.min(target.bytes); match crate::space::attribution( mounts.mounts.as_deref(), &plan.root, @@ -1600,6 +1625,7 @@ mod tests { Target { path: path.to_path_buf(), bytes, + unshared: bytes, is_dir: path.is_dir(), hidden: false, } @@ -1831,6 +1857,43 @@ mod tests { assert_eq!(plan.unattributed(), 50, "no saving may be claimed"); } + /// An APFS clone's blocks are its family's: deleting one copy frees + /// only what it holds alone, so only that is projected, on any volume, + /// and the rest is said to be shared rather than silently dropped. + #[test] + fn only_unshared_bytes_are_projected() { + let table = "\ +/dev/sda2 / ext4 rw 0 0 +/dev/sdb1 /data ext4 rw 0 0 +"; + let mounts = MountTable { + points: crate::space::parse_mounts(table) + .iter() + .map(|mount| guard_key(&mount.point)) + .collect(), + media: Vec::new(), + mounts: Some(crate::space::parse_mounts(table)), + }; + let shared = |path: &str, bytes, unshared| Target { + unshared, + ..target(Path::new(path), bytes) + }; + let marked = [ + shared("/home/tobi/copy", 1000, 0), + shared("/home/tobi/edited", 1000, 100), + target(Path::new("/home/tobi/plain"), 500), + shared("/data/copy", 200, 50), + ]; + let plan = plan_against(&marked, Path::new("/"), &mounts); + assert!(plan.blocked.is_empty(), "{:?}", plan.blocked); + assert_eq!(plan.bytes(), 2700, "the list still says what goes"); + assert_eq!(plan.shared(), 1000 + 900 + 150); + assert_eq!(plan.frees(), 650); + assert_eq!(plan.reclaim(), 100 + 500, "this disk's own bytes"); + assert_eq!(plan.foreign(), 50); + assert_eq!(plan.unattributed(), 0); + } + #[test] fn the_root_and_home_are_refused() { let temp = tree(); diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs index 5ffe81a0..79ec8963 100644 --- a/crates/disktree-core/src/scan.rs +++ b/crates/disktree-core/src/scan.rs @@ -57,7 +57,9 @@ pub struct ScanOptions { /// Stop descending past this depth. Totals below that depth are then /// unknown, which makes an overview scan of a huge tree cheap. pub max_depth: Option, - /// Count a hardlinked file once instead of once per link. + /// Count a hardlinked file once instead of once per link, and an APFS + /// clone family once instead of once per copy: every copy reports the + /// blocks the family shares. pub dedup_hardlinks: bool, /// Whether children are ranked by bytes or by file count. pub metric: Metric, @@ -501,6 +503,13 @@ impl WalkContext { // What the link leads to, so a file that is also reached directly is // charged once. facts.identity = identity_of(path, &meta); + // And as its listing would charge it: a clone as its family. + #[cfg(target_os = "macos")] + if let Ok(entry) = crate::macos::stat(path) { + facts.identity = Some(entry.charged_as()); + facts.unshared = entry.unshared(facts.size); + facts.shared |= entry.shared(); + } self.leaf(name, kind_of(&meta, meta.file_type()), &facts) } @@ -574,6 +583,9 @@ enum Listing { /// What a leaf contributes. struct Facts { size: u64, + /// The part of `size` no other file shares, which removing the file is + /// sure to give back; all of it where nothing says otherwise. + unshared: u64, identity: Option<(u64, u64)>, modified: i64, /// The file may have another name the walk could meet: its identity is @@ -583,8 +595,10 @@ struct Facts { impl Facts { fn of(meta: &Metadata, apparent_size: bool) -> Self { + let size = measure(meta, apparent_size); Self { - size: measure(meta, apparent_size), + size, + unshared: size, identity: file_identity(meta), modified: modified_seconds(meta), shared: shares_inode(meta), @@ -664,13 +678,17 @@ impl Listed for crate::macos::Entry { } fn facts(&self, apparent_size: bool) -> io::Result { + let size = if apparent_size { + self.apparent() + } else { + self.allocated() + }; Ok(Facts { - size: if apparent_size { - self.apparent() - } else { - self.allocated() - }, - identity: Some(self.identity()), + size, + unshared: self.unshared(size), + // A clone is charged as its family, so the family is charged + // once however many copies the walk meets. + identity: Some(self.charged_as()), modified: self.modified(), shared: self.shared(), }) @@ -707,12 +725,14 @@ impl Listed for crate::windows::Entry { } fn facts(&self, apparent_size: bool) -> io::Result { + let size = if apparent_size { + self.apparent() + } else { + self.allocated() + }; Ok(Facts { - size: if apparent_size { - self.apparent() - } else { - self.allocated() - }, + size, + unshared: size, identity: self.identity(), modified: self.modified(), // The listing has no link count; a file id is free here, so @@ -834,6 +854,7 @@ impl PendingDir { name: partial.name, kind: NodeKind::Directory, bytes: 0, + unshared: 0, files: 0, dirs: 1, inode: None, @@ -1225,6 +1246,7 @@ fn leaf_node( track: bool, ) -> Node { let mut node = Node::entry(name, kind, facts.size); + node.unshared = facts.unshared; if track { node.inode = facts .identity @@ -1563,6 +1585,66 @@ mod tests { assert_eq!(counted.bytes, 8192); } + #[cfg(target_os = "macos")] + #[test] + fn an_apfs_clone_family_is_charged_once_and_frees_only_its_own() { + use crate::macos::testing::{clone_file, on_apfs}; + let temp = TempDir::new().expect("tempdir"); + let root = temp.path(); + if !on_apfs(root) { + eprintln!("skipped: {} is not on APFS", root.display()); + return; + } + // The copy in another folder, so the folder that is not charged + // shows what a mark on it would wrongly have claimed. + let original = write(root, "data/original.bin", 1 << 20); + fs::create_dir(root.join("copies")).expect("mkdir"); + clone_file(&original, &root.join("copies/copy.bin")); + write(root, "plain.bin", 1 << 20); + let allocated = ScanOptions::default(); + + let tree = scan_dir(root, &allocated); + let plain = child(&tree, "plain.bin").bytes; + let family = { + use std::os::unix::fs::MetadataExt as _; + fs::metadata(&original).expect("stat").blocks() * 512 + }; + assert!(family >= 1 << 20); + let (data, copies) = (child(&tree, "data"), child(&tree, "copies")); + assert_eq!(data.bytes + copies.bytes, family, "charged once"); + assert_eq!(tree.bytes, family + plain); + assert_eq!(data.unshared + copies.unshared, 0, "all of it shared"); + assert_eq!(tree.unshared, plain, "only the plain file is sure"); + assert_eq!(child(&tree, "plain.bin").unshared, plain); + + let counted = scan_dir( + root, + &ScanOptions { + dedup_hardlinks: false, + ..allocated + }, + ); + assert_eq!(counted.bytes, 2 * family + plain, "once per copy"); + assert_eq!(counted.unshared, plain); + + // Reached again through a followed link, the copy is still its + // family, not a third file. + std::os::unix::fs::symlink( + root.join("copies/copy.bin"), + root.join("link.bin"), + ) + .expect("symlink"); + let followed = scan_dir( + root, + &ScanOptions { + follow_links: true, + ..allocated + }, + ); + assert_eq!(followed.bytes, family + plain); + assert_eq!(followed.unshared, plain); + } + #[test] fn hidden_entries_are_included_by_default_and_excluded_on_request() { let temp = TempDir::new().expect("tempdir"); diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rs index 4bbeda44..bbc3718e 100644 --- a/crates/disktree-core/src/tree.rs +++ b/crates/disktree-core/src/tree.rs @@ -76,6 +76,13 @@ pub struct Node { /// Subtree total: direct contents plus every descendant. A file's own /// size, or nothing for a second name of a file already charged. pub bytes: u64, + /// The part of `bytes` no other file shares: what removing this node is + /// sure to give back, and all a projection may count. A directory's is + /// derived. A leaf's is all of its `bytes` unless its file system says + /// it shares blocks: an APFS clone holds nothing of its own, so marking + /// one copy of three projects nothing, which is what deleting it frees; + /// see [`crate::macos`] for a copy edited away from its family. + pub unshared: u64, /// Files at or beneath this node; `1` for a file. pub files: u64, /// Directories at or beneath this node; `1` for a directory. Four @@ -109,6 +116,7 @@ impl Node { name: name.into(), kind: NodeKind::Directory, bytes: 0, + unshared: 0, files: 0, dirs: 1, inode: None, @@ -130,6 +138,7 @@ impl Node { name: name.into(), kind, bytes, + unshared: bytes, files: u64::from(kind == NodeKind::File), dirs: 0, inode: None, @@ -264,20 +273,21 @@ impl Node { } } -/// Recompute `bytes`, `files` and `dirs` bottom-up, then order children by -/// `metric`, largest first. +/// Recompute `bytes`, `unshared`, `files` and `dirs` bottom-up, then order +/// children by `metric`, largest first. /// -/// A leaf's `bytes` is its measured size and the only input; every total is -/// derived from the leaves, as are the direct figures [`Node::own_bytes`] -/// and [`Node::own_files`]. Deriving them is what keeps them consistent: -/// hardlink de-duplication rewrites a leaf's weight, and every total above -/// it follows without a second pass. +/// A leaf's `bytes` and `unshared` are its measured sizes and the only +/// inputs; every total is derived from the leaves, as are the direct +/// figures [`Node::own_bytes`] and [`Node::own_files`]. Deriving them is +/// what keeps them consistent: hardlink de-duplication rewrites a leaf's +/// weight, and every total above it follows without a second pass. pub fn aggregate(node: &mut Node, metric: Metric) { aggregate_at(node, metric, 0, None); } /// [`aggregate`], charging a hardlinked file once: a leaf whose identity -/// `seen` already holds weighs nothing. Which of a file's names is charged +/// `seen` already holds weighs nothing. On APFS a clone's identity is its +/// family's, so the family is charged once too. Which of a file's names is charged /// is whichever a worker reaches first, so two scans of an unchanged tree /// can split it differently between folders; the totals are the same. pub(crate) fn aggregate_deduped(node: &mut Node, metric: Metric, seen: &Seen) { @@ -369,12 +379,17 @@ fn aggregate_at( && let Some(key) = node.identity() && !seen.insert(key) { + // Never more freed than charged: a second name's private bytes + // are the first name's, and a clone family's members share + // every block, so none has any of its own. node.bytes = 0; + node.unshared = 0; } return; } let mut bytes = 0; + let mut unshared = 0; let mut files = 0; let mut dirs: u32 = 1; let mut modified = 0; @@ -391,10 +406,12 @@ fn aggregate_at( modified = modified.max(child.modified); // Saturating: a corrupt volume's file table can claim any size. bytes = child.bytes.saturating_add(bytes); + unshared = child.unshared.saturating_add(unshared); files += child.files; dirs += child.dirs; } node.bytes = bytes; + node.unshared = unshared; node.files = files; node.dirs = dirs; node.modified = modified; @@ -441,7 +458,7 @@ mod tests { #[test] #[cfg(target_pointer_width = "64")] fn a_node_stays_small() { - assert_eq!(size_of::(), 88); + assert_eq!(size_of::(), 96); } #[test] @@ -515,6 +532,42 @@ mod tests { .unwrap_or_else(|| panic!("no child named {name}")) } + #[test] + fn what_is_sure_to_be_freed_is_derived_and_charged_once() { + let family = Some((1, NonZeroU64::new(7).unwrap())); + let mut clones = Node::directory("clones"); + for name in ["a", "b"] { + // Each copy reports the family's blocks and none of its own. + let mut copy = leaf(name, 1000); + copy.unshared = 0; + copy.inode = family; + clones.children.push(copy); + } + let mut edited = leaf("edited", 1000); + edited.unshared = 100; + let mut linked = Node::directory("linked"); + for name in ["first", "second"] { + let mut link = leaf(name, 400); + link.inode = Some((1, NonZeroU64::new(9).unwrap())); + linked.children.push(link); + } + let mut root = Node::directory("root"); + root.children.push(clones); + root.children.push(edited); + root.children.push(linked); + root.children.push(leaf("plain", 500)); + + aggregate_deduped(&mut root, Metric::Bytes, &Seen::new()); + assert_eq!(root.bytes, 1000 + 1000 + 400 + 500); + assert_eq!(root.unshared, 100 + 400 + 500); + assert_eq!(child_named(&root, "clones").bytes, 1000, "charged once"); + assert_eq!(child_named(&root, "clones").unshared, 0); + // A second name is charged nothing, so it frees nothing either. + assert_eq!(child_named(&root, "linked").unshared, 400); + // A leaf nobody measured for sharing is sure to free all of itself. + assert_eq!(leaf("x", 5).unshared, 5); + } + #[test] fn aggregate_can_rank_by_file_count() { let mut root = Node::directory("root");