diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56b3cac..478efc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,8 +32,8 @@ jobs: - name: make clippy run: make clippy - - name: cargo test - run: cargo test + - name: make test + run: make test - name: cargo fmt -- --check run: cargo fmt -- --check diff --git a/Makefile b/Makefile index abad980..f3c5739 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,20 @@ UNAME := $(shell uname -s) +# Which workspace members to check, lint, and test. +# +# `default-members = ["."]` in Cargo.toml means a bare `cargo test` only +# covers the root package, so the unit tests in ember-core and the +# platform crate never run. `--workspace` is not the fix: it selects the +# other platform's backend too, and that backend does not compile here +# (ember-macos needs `clonefile`, ember-linux's image tests need +# `mkfs.ext4`). Naming the packages is what actually works. +ifeq ($(UNAME),Darwin) +PACKAGES := -p ember -p ember-core -p ember-macos +else +PACKAGES := -p ember -p ember-core -p ember-linux +endif + .PHONY: build release clean fmt check clippy test udeps build: @@ -32,13 +46,13 @@ fmt: cargo fmt check: - cargo check + cargo check --all-targets $(PACKAGES) clippy: - cargo clippy --all-targets -- -D warnings + cargo clippy --all-targets $(PACKAGES) -- -D warnings test: - cargo test + cargo test $(PACKAGES) udeps: cargo machete diff --git a/README.md b/README.md index 9fc778a..c29c626 100644 --- a/README.md +++ b/README.md @@ -216,14 +216,17 @@ ember cp ./local-file.txt myvm:/tmp/ ember cp myvm:/var/log/syslog ./syslog.txt ``` -## Storage efficiency +## Storage usage -Both platforms use copy-on-write storage, so VMs and forks share disk blocks with their parent image. Check actual disk usage: +Both platforms use copy-on-write storage, so VMs and forks share disk blocks with their parent image, and every backend that compresses shrinks them further. `ember storage usage` reports what each VM and image actually occupies against what it was provisioned: ```bash -ember debug storage-efficiency +ember storage usage +ember storage usage --format json ``` +`EXCLUSIVE` is what a volume holds on its own, `SHARED` is what it still has in common with the image or fork it came from. `ember vm list` carries the same exclusive figure in its `USED` column. + ## Building a custom kernel The stock kernel (auto-downloaded on first use) works for most use cases. However, it **lacks full Docker networking support** — the iptables `raw` table and nftables modules are missing, so Docker bridge networking doesn't work inside guest VMs. diff --git a/crates/ember-core/src/backend.rs b/crates/ember-core/src/backend.rs index 33cd2e2..91a1506 100644 --- a/crates/ember-core/src/backend.rs +++ b/crates/ember-core/src/backend.rs @@ -7,8 +7,11 @@ //! The active implementation is selected at compile time in the binary crate //! and re-exported as type aliases (`Vm`, `Storage`, `Network`). +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +use serde::Serialize; + use crate::config::size::ByteSize; use crate::config::{DmThinMode, GlobalConfig}; use crate::error::Result; @@ -52,6 +55,113 @@ impl VolumeHandle { } } +/// Space accounting for a single volume, in bytes. +/// +/// This is an occupancy model: it answers where space has gone, not +/// what a delete would give back. Those differ enough on ZFS to be +/// worth stating. A zvol is also charged for its refreservation and for +/// blocks held only by its snapshots, and neither appears in +/// `exclusive`. Backends whose forks are independent (dm-thin, APFS) +/// have no such gap. +/// +/// Backends fill in what they can measure. `exclusive` is the only +/// field all of them produce. +#[derive(Clone, Copy, Debug, Serialize)] +pub struct VolumeUsage { + /// Virtual size presented to the guest. + pub provisioned: u64, + /// Physical bytes this volume holds that are not shared with an + /// origin. Always within `referenced` when that is known. + pub exclusive: u64, + /// Physical bytes reachable from this volume, including blocks + /// shared with an origin. `None` when the backend cannot tell + /// shared and exclusive blocks apart. + pub referenced: Option, + /// Uncompressed size of `referenced`. `None` when the backend does + /// not compress. + pub logical: Option, +} + +impl VolumeUsage { + /// Bytes shared with an origin volume, when the backend can tell. + pub fn shared(&self) -> Option { + self.referenced.map(|r| r.saturating_sub(self.exclusive)) + } + + /// Compression ratio over the referenced blocks, when the backend + /// compresses. `None` for an empty volume, where the ratio would be + /// a division by zero rather than a meaningful 1.0. + pub fn compression_ratio(&self) -> Option { + ratio(self.logical, self.referenced) + } +} + +/// Usage of a backend's dedicated metadata device, in bytes. +#[derive(Clone, Copy, Debug, Serialize)] +pub struct MetadataUsage { + pub capacity: u64, + pub used: u64, +} + +/// Pool-wide capacity, in bytes. +#[derive(Clone, Copy, Debug, Serialize)] +pub struct PoolUsage { + pub capacity: u64, + pub allocated: u64, + /// Part of `allocated` that is reserved but holds no data, so it + /// compresses to nothing and must be kept out of the ratio. Zero + /// for backends without reservations. + pub reserved: u64, + /// Uncompressed size of the data within `allocated`. `None` when + /// the backend does not compress. + pub logical: Option, + /// Present only for backends that keep a separate metadata device. + pub metadata: Option, +} + +impl PoolUsage { + pub fn free(&self) -> u64 { + self.capacity.saturating_sub(self.allocated) + } + + /// Bytes of `allocated` that actually hold data. + pub fn occupied(&self) -> u64 { + self.allocated.saturating_sub(self.reserved) + } + + /// Compression ratio over occupied space, when the backend + /// compresses. + /// + /// Measured against [`occupied`](Self::occupied) rather than + /// `allocated`: empty reservation is charged to the pool but has no + /// logical counterpart, so including it would understate the ratio. + pub fn compression_ratio(&self) -> Option { + ratio(self.logical, Some(self.occupied())) + } +} + +/// Shared by the two `compression_ratio` accessors. A zero +/// denominator yields `None` rather than an infinity that would render +/// as `inf` in the CLI. +fn ratio(logical: Option, physical: Option) -> Option { + match (logical, physical) { + (Some(logical), Some(physical)) if physical > 0 => Some(logical as f64 / physical as f64), + _ => None, + } +} + +/// Space accounting for a whole installation, produced in one pass. +#[derive(Clone, Debug, Serialize)] +pub struct StorageUsage { + pub pool: PoolUsage, + /// Keyed by [`VmMetadata::name`]. A missing key means the backend + /// could not account for that VM, which the CLI renders as `-` + /// rather than as zero. + pub vms: BTreeMap, + /// Keyed by [`ImageEntry::local_name`], same missing-key rule. + pub images: BTreeMap, +} + /// Configuration for storage backend initialization during `ember init`. /// /// Carries the subset of init arguments that the storage backend needs. @@ -256,6 +366,18 @@ pub trait StorageBackend { /// destroyed. Empty for backends whose forks are independent. fn storage_dependents(&self, vm: &VmMetadata) -> Result>; + /// Measure actual space usage across the installation. + /// + /// Takes the state records rather than discovering volumes itself, + /// because the name-to-volume mapping lives in state and not in the + /// backend. Returns the whole set in one value so that backends + /// which have to walk pool-wide metadata do that walk once instead + /// of once per volume. + /// + /// Volumes the backend cannot account for are left out of the maps + /// instead of being reported as zero. + fn usage(&self, vms: &[VmMetadata], images: &[ImageEntry]) -> Result; + /// Mount a disk image and return the mount point path. /// /// Linux: mounts the zvol block device. @@ -484,3 +606,87 @@ pub trait Platform { /// callers are expected to soft-fail rather than block on this. fn host_ram_mib() -> anyhow::Result; } + +#[cfg(test)] +mod tests { + use super::*; + + fn volume(exclusive: u64, referenced: Option, logical: Option) -> VolumeUsage { + VolumeUsage { + provisioned: 1024, + exclusive, + referenced, + logical, + } + } + + #[test] + fn shared_is_the_gap_between_referenced_and_exclusive() { + assert_eq!(volume(80, Some(100), None).shared(), Some(20)); + assert_eq!(volume(100, Some(100), None).shared(), Some(0)); + } + + /// Backends that cannot separate shared from exclusive report + /// nothing rather than claiming zero sharing. + #[test] + fn shared_is_unknown_without_referenced() { + assert_eq!(volume(80, None, None).shared(), None); + } + + /// `exclusive` is contractually within `referenced`, but the + /// saturating subtraction keeps a backend bug from producing a + /// wrapped, astronomically large shared figure. + #[test] + fn shared_saturates_instead_of_wrapping() { + assert_eq!(volume(120, Some(100), None).shared(), Some(0)); + } + + #[test] + fn compression_ratio_divides_logical_by_referenced() { + let r = volume(80, Some(100), Some(200)).compression_ratio(); + assert_eq!(r, Some(2.0)); + } + + /// An untouched volume would divide by zero. `None` renders as `-` + /// where an infinity would render as `inf`. + #[test] + fn compression_ratio_guards_empty_volume() { + assert_eq!(volume(0, Some(0), Some(0)).compression_ratio(), None); + assert_eq!(volume(0, None, Some(200)).compression_ratio(), None); + assert_eq!(volume(0, Some(100), None).compression_ratio(), None); + } + + fn pool(allocated: u64, reserved: u64, logical: Option) -> PoolUsage { + PoolUsage { + capacity: 1000, + allocated, + reserved, + logical, + metadata: None, + } + } + + #[test] + fn free_is_capacity_minus_allocated() { + assert_eq!(pool(400, 0, None).free(), 600); + // A pool reporting more allocated than capacity must not wrap. + assert_eq!(pool(1200, 0, None).free(), 0); + } + + /// Empty reservation is charged to the pool but has no logical + /// counterpart, so leaving it in the denominator understates + /// compression. + #[test] + fn pool_ratio_excludes_reservation() { + let p = pool(300, 100, Some(400)); + assert_eq!(p.occupied(), 200); + assert_eq!(p.compression_ratio(), Some(2.0)); + } + + #[test] + fn pool_ratio_guards_fully_reserved_pool() { + assert_eq!(pool(100, 100, Some(0)).compression_ratio(), None); + assert_eq!(pool(0, 0, Some(0)).compression_ratio(), None); + assert_eq!(pool(100, 0, None).compression_ratio(), None); + } +} diff --git a/crates/ember-core/src/config.rs b/crates/ember-core/src/config.rs index 79bd57f..5c0e86c 100644 --- a/crates/ember-core/src/config.rs +++ b/crates/ember-core/src/config.rs @@ -152,6 +152,13 @@ pub fn fnv1a_32(bytes: &[u8]) -> u32 { } impl GlobalConfig { + /// Root of the ZFS dataset tree ember owns (e.g. `ember/ember`), + /// the common parent of [`images_dataset`](Self::images_dataset) + /// and [`vms_dataset`](Self::vms_dataset). + pub fn base_dataset(&self) -> String { + format!("{}/{}", self.pool, self.dataset) + } + /// Full ZFS dataset path for images (e.g. `ember/ember/images`). pub fn images_dataset(&self) -> String { format!("{}/{}/images", self.pool, self.dataset) diff --git a/crates/ember-linux/src/dm_thin.rs b/crates/ember-linux/src/dm_thin.rs index cce5ba3..b548b82 100644 --- a/crates/ember-linux/src/dm_thin.rs +++ b/crates/ember-linux/src/dm_thin.rs @@ -64,6 +64,23 @@ pub fn is_already_exists(err: &ember_core::error::Error) -> bool { ) } +/// Whether an [`Error`](ember_core::error::Error) reports a kernel `EBUSY` +/// from a `dmsetup message` operation. +/// +/// The only place we act on this is `reserve_metadata_snap`, where the +/// kernel returns `-EBUSY` when the pool already holds a snapshot +/// (`__reserve_metadata_snap` in `drivers/md/dm-thin-metadata.c`). +/// Same stability argument as [`is_already_exists`]: `dmsetup` embeds +/// the libc strerror, and `strerror(EBUSY)` is `"Device or resource +/// busy"` on glibc and musl. +pub fn is_busy(err: &ember_core::error::Error) -> bool { + matches!( + err, + ember_core::error::Error::Command { stderr, .. } + if stderr.contains("Device or resource busy") + ) +} + #[cfg(test)] mod tests { use super::*; @@ -99,4 +116,19 @@ mod tests { let err = Error::Vm("File exists somewhere else in the system".to_string()); assert!(!is_already_exists(&err)); } + + /// Companion to [`matches_dmsetup_eexist_message`], pinning the + /// wording `reserve_metadata_snap` fails with when a snapshot is + /// already held. + #[test] + fn matches_dmsetup_ebusy_message() { + let err = Error::Command { + command: "dmsetup".to_string(), + exit_code: 1, + stderr: "device-mapper: message ioctl on ember-pool failed: Device or resource busy\n" + .to_string(), + }; + assert!(is_busy(&err)); + assert!(!is_already_exists(&err)); + } } diff --git a/crates/ember-linux/src/dm_thin/pool.rs b/crates/ember-linux/src/dm_thin/pool.rs index 2f405a6..1f8a667 100644 --- a/crates/ember-linux/src/dm_thin/pool.rs +++ b/crates/ember-linux/src/dm_thin/pool.rs @@ -39,6 +39,11 @@ pub const DEFAULT_BLOCK_SIZE_SECTORS: u32 = 128; /// raises a `dmeventd` notification. pub const DEFAULT_LOW_WATER_BLOCKS: u64 = 32_768; +/// Size of one metadata block, in bytes. Fixed by the kernel and +/// independent of the pool's data block size. Needed to turn the +/// metadata block counts in [`PoolStatus`] into bytes. +pub const METADATA_BLOCK_SIZE: u64 = 4096; + /// Operating mode reported by `dmsetup status`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PoolMode { @@ -211,6 +216,50 @@ pub fn message(name: &str, msg: &str) -> Result<()> { Ok(()) } +/// A reserved pool metadata snapshot, released when dropped. +/// +/// Reading thin-volume mappings needs a snapshot, because the live +/// metadata device belongs to the kernel and cannot be walked +/// underneath it. A pool holds at most one snapshot at a time, which is +/// why this is a guard rather than a pair of calls: the release has to +/// happen on every path out, including an early `?` return and a panic. +/// A leaked reservation blocks the next reader and pins metadata blocks +/// that the pool would otherwise be free to reuse. +pub struct MetadataSnap { + pool: String, +} + +impl MetadataSnap { + /// Reserve the pool's metadata snapshot. + /// + /// Fails if one is already held. That is usually a reservation + /// stranded by a killed process rather than a live reader, so the + /// error spells out the manual release. + pub fn reserve(pool_name: &str) -> Result { + match message(pool_name, "reserve_metadata_snap") { + Ok(()) => Ok(Self { + pool: pool_name.to_string(), + }), + Err(e) if super::is_busy(&e) => Err(Error::Pool(format!( + "dm-thin pool '{pool_name}' already holds a metadata snapshot. \ + If nothing else is reading the pool, an earlier run left it behind: \ + release it with `dmsetup message {pool_name} 0 release_metadata_snap`" + ))), + Err(e) => Err(e), + } + } +} + +impl Drop for MetadataSnap { + fn drop(&mut self) { + // We never force-release a snapshot we did not take, and by the + // same token there is nothing useful to do if releasing our own + // fails. Swallowing it here at least keeps the failure from + // masking whatever error is already unwinding. + let _ = message(&self.pool, "release_metadata_snap"); + } +} + /// Reload the pool table with new parameters (typically a larger /// `data_sectors` after growing the data device). /// diff --git a/crates/ember-linux/src/dm_thin/tools.rs b/crates/ember-linux/src/dm_thin/tools.rs index 46eb23f..1336054 100644 --- a/crates/ember-linux/src/dm_thin/tools.rs +++ b/crates/ember-linux/src/dm_thin/tools.rs @@ -1,5 +1,5 @@ //! Wrappers around the `thin-provisioning-tools` package: `thin_check`, -//! `thin_repair`, `thin_metadata_size`, `thin_dump`. +//! `thin_repair`, `thin_metadata_size`, `thin_dump`, `thin_ls`. //! //! These are recommended (and in some cases required) for safe pool //! activation and capacity planning. They live in their own module so @@ -80,6 +80,75 @@ pub fn repair(input: &Path, output: &Path) -> Result<()> { Ok(()) } +/// Per-volume accounting for one thin device. +#[derive(Debug, PartialEq)] +pub struct ThinRow { + pub dev_id: u64, + /// Bytes mapped by this device, blocks shared with an origin + /// included. + pub mapped_bytes: u64, + /// Bytes mapped only by this device. Freed when it is deleted. + pub exclusive_bytes: u64, +} + +/// List per-volume accounting for every thin device in a pool. +/// +/// Reads through a reserved metadata snapshot (`-m`), which is the only +/// way to inspect metadata while the kernel owns the live device. The +/// caller must hold a [`pool::MetadataSnap`](super::pool::MetadataSnap) +/// for the duration. +/// +/// Reporting through metadata rather than `dmsetup status` also covers +/// volumes that are not currently activated, which is the common case +/// given that ember activates thin devices lazily. +pub fn list_thins(metadata_dev: &Path) -> Result> { + let output = Command::new("thin_ls") + .args([ + "-m", + "--no-headers", + "-o", + "DEV,MAPPED_BYTES,EXCLUSIVE_BYTES", + ]) + .arg(metadata_dev) + .output() + .map_err(|e| Error::CommandExec { + command: "thin_ls".to_string(), + source: e, + })?; + let output = Error::check_command("thin_ls", output)?; + parse_thin_ls(&String::from_utf8_lossy(&output.stdout)) +} + +fn parse_thin_ls(stdout: &str) -> Result> { + stdout + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| { + let fields: Vec<&str> = line.split_whitespace().collect(); + if fields.len() != 3 { + return Err(Error::Command { + command: "thin_ls".to_string(), + exit_code: 0, + stderr: format!("expected 3 fields per row, got {}: {line}", fields.len()), + }); + } + Ok(ThinRow { + dev_id: parse_field(fields[0], "DEV")?, + mapped_bytes: parse_field(fields[1], "MAPPED_BYTES")?, + exclusive_bytes: parse_field(fields[2], "EXCLUSIVE_BYTES")?, + }) + }) + .collect() +} + +fn parse_field(s: &str, field: &str) -> Result { + s.parse::().map_err(|e| Error::Command { + command: "thin_ls".to_string(), + exit_code: 0, + stderr: format!("non-numeric {field} value {s:?}: {e}"), + }) +} + /// Dump the metadata device's contents as XML. /// /// Useful for recovery (cross-checking ember's recorded thin ids @@ -96,3 +165,49 @@ pub fn dump(metadata_dev: &Path) -> Result { let output = Error::check_command("thin_dump", output)?; Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_thin_ls_rows() { + let out = " 1234 10737418240 8589934592\n\ + 5678 2147483648 1073741824\n"; + let rows = parse_thin_ls(out).unwrap(); + assert_eq!( + rows, + vec![ + ThinRow { + dev_id: 1234, + mapped_bytes: 10_737_418_240, + exclusive_bytes: 8_589_934_592, + }, + ThinRow { + dev_id: 5678, + mapped_bytes: 2_147_483_648, + exclusive_bytes: 1_073_741_824, + }, + ] + ); + } + + /// A pool that holds no thin devices yet. + #[test] + fn parses_empty_listing() { + assert_eq!(parse_thin_ls("").unwrap(), vec![]); + assert_eq!(parse_thin_ls("\n\n").unwrap(), vec![]); + } + + /// If `--no-headers` ever stops suppressing the header we want a + /// hard failure, not a row with a garbage device id. + #[test] + fn rejects_header_row() { + assert!(parse_thin_ls("DEV MAPPED_BYTES EXCLUSIVE_BYTES\n").is_err()); + } + + #[test] + fn rejects_short_row() { + assert!(parse_thin_ls("1234 10737418240\n").is_err()); + } +} diff --git a/crates/ember-linux/src/dm_thin_storage.rs b/crates/ember-linux/src/dm_thin_storage.rs index 067b4e8..b0d2ce5 100644 --- a/crates/ember-linux/src/dm_thin_storage.rs +++ b/crates/ember-linux/src/dm_thin_storage.rs @@ -8,11 +8,14 @@ //! //! See `docs/DM-THIN-SPEC.md` for the design. +use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command as ProcessCommand; -use ember_core::backend::{InitConfig, StorageBackend, VolumeHandle}; +use ember_core::backend::{ + InitConfig, MetadataUsage, PoolUsage, StorageBackend, StorageUsage, VolumeHandle, VolumeUsage, +}; use ember_core::config::size::ByteSize; use ember_core::config::{DmThinMode, GlobalConfig}; use ember_core::error::{Error, Result}; @@ -587,6 +590,43 @@ impl StorageBackend for DmThinStorage { Ok(Vec::new()) } + /// Pool figures come from the status line we already parse for + /// health checks. Per-volume figures need a metadata snapshot, so + /// the whole installation is measured under one reservation. + /// + /// Unlike the rest of the backend this does not activate the pool. + /// Measuring is a query, and callers include `ember vm list`, which + /// has no business loading a pool table, running `thin_check`, and + /// attaching loop devices as a side effect of listing VMs. + fn usage(&self, vms: &[VmMetadata], images: &[ImageEntry]) -> Result { + if !dm_device_exists(&self.pool_name)? { + return Err(Error::Pool(format!( + "dm-thin pool '{}' is not active, so its usage cannot be measured. \ + Any command that touches storage will activate it.", + self.pool_name + ))); + } + let status = pool::status(&self.pool_name)?; + let block_bytes = (self.block_size_sectors as u64) * SECTOR_SIZE; + + let by_id = { + let metadata_loop = loop_device::find_for(&self.metadata_file())?.ok_or_else(|| { + Error::Config(format!( + "metadata device {} is not attached to a loop device", + self.metadata_file().display() + )) + })?; + let _snap = pool::MetadataSnap::reserve(&self.pool_name)?; + tools::list_thins(&metadata_loop)? + }; + + Ok(StorageUsage { + pool: pool_usage(&status, block_bytes), + vms: join_vms(vms, &by_id), + images: join_images(images, &by_id), + }) + } + fn deinit(&self, purge: bool) -> Result<()> { // 1. Deactivate every thin volume that belongs to *this* // installation so the pool can be removed cleanly. Other @@ -717,6 +757,75 @@ impl StorageBackend for DmThinStorage { // Helpers // --------------------------------------------------------------------------- +/// Turn a thin-pool status line into pool-level byte figures. +/// +/// `dmsetup status` counts data in pool blocks and metadata in the +/// kernel's fixed 4 KiB metadata blocks, so the two need different +/// multipliers. +fn pool_usage(status: &pool::PoolStatus, block_bytes: u64) -> PoolUsage { + PoolUsage { + capacity: status.total_data_blocks * block_bytes, + allocated: status.used_data_blocks * block_bytes, + // dm-thin never over-allocates against a volume's virtual size, + // so nothing is reserved-but-empty the way a zvol is. + reserved: 0, + // dm-thin stores blocks verbatim. Reporting `None` rather than + // a figure equal to `allocated` keeps the CLI from printing a + // meaningless 1.00x ratio. + logical: None, + metadata: Some(MetadataUsage { + capacity: status.total_metadata_blocks * pool::METADATA_BLOCK_SIZE, + used: status.used_metadata_blocks * pool::METADATA_BLOCK_SIZE, + }), + } +} + +/// Project a `thin_ls` row onto a record's accounting. +/// +/// Returns `None` for a record with no thin id, and for an id the pool +/// no longer knows about. Both are reported as absent rather than as an +/// empty volume, since zero bytes and "cannot say" are different +/// answers. +fn volume_usage( + thin_id: Option, + provisioned: u64, + rows: &[tools::ThinRow], +) -> Option { + let thin_id = thin_id?; + let row = rows.iter().find(|r| r.dev_id == thin_id)?; + Some(VolumeUsage { + provisioned, + exclusive: row.exclusive_bytes, + referenced: Some(row.mapped_bytes), + logical: None, + }) +} + +fn join_vms(vms: &[VmMetadata], rows: &[tools::ThinRow]) -> BTreeMap { + vms.iter() + .filter_map(|vm| { + let provisioned = DmThinStorage::vm_size_sectors(vm) * SECTOR_SIZE; + Some(( + vm.name.clone(), + volume_usage(vm.thin_id, provisioned, rows)?, + )) + }) + .collect() +} + +fn join_images(images: &[ImageEntry], rows: &[tools::ThinRow]) -> BTreeMap { + images + .iter() + .filter_map(|img| { + let provisioned = img.size_mib * 1024 * 1024; + Some(( + img.local_name.clone(), + volume_usage(img.thin_id, provisioned, rows)?, + )) + }) + .collect() +} + /// Decide where the metadata + data backing live based on the /// caller-resolved [`DmThinMode`]. /// @@ -908,4 +1017,95 @@ mod tests { assert_eq!(format_bytes(2 * 1024 * 1024), "2.0 MiB"); assert_eq!(format_bytes(3u64 * 1024 * 1024 * 1024), "3.0 GiB"); } + + fn status( + used_data: u64, + total_data: u64, + used_meta: u64, + total_meta: u64, + ) -> pool::PoolStatus { + pool::PoolStatus { + used_metadata_blocks: used_meta, + total_metadata_blocks: total_meta, + used_data_blocks: used_data, + total_data_blocks: total_data, + mode: pool::PoolMode::ReadWrite, + } + } + + /// Data blocks scale by the pool's block size, metadata blocks by + /// the kernel's fixed 4 KiB. Mixing the two multipliers up would + /// misreport metadata by a factor of 16 at the default block size. + #[test] + fn pool_usage_scales_data_and_metadata_separately() { + let block_bytes = pool::DEFAULT_BLOCK_SIZE_SECTORS as u64 * SECTOR_SIZE; + assert_eq!(block_bytes, 65536); + + let u = pool_usage(&status(100, 1000, 5, 2048), block_bytes); + assert_eq!(u.allocated, 100 * 65536); + assert_eq!(u.capacity, 1000 * 65536); + assert_eq!(u.free(), 900 * 65536); + assert_eq!(u.reserved, 0); + assert_eq!(u.logical, None); + assert_eq!(u.compression_ratio(), None); + + let meta = u.metadata.expect("dm-thin has a metadata device"); + assert_eq!(meta.used, 5 * 4096); + assert_eq!(meta.capacity, 2048 * 4096); + } + + fn row(dev_id: u64, mapped: u64, exclusive: u64) -> tools::ThinRow { + tools::ThinRow { + dev_id, + mapped_bytes: mapped, + exclusive_bytes: exclusive, + } + } + + fn vm_record(name: &str, thin_id: Option, disk_size_gib: u32) -> VmMetadata { + let mut m = VmMetadata::default_for_teardown(); + m.name = name.to_string(); + m.thin_id = thin_id; + m.disk_size_gib = disk_size_gib; + m + } + + #[test] + fn join_matches_records_to_rows_by_thin_id() { + let rows = vec![row(42, 3000, 2000), row(7, 500, 500)]; + let vms = [vm_record("a", Some(42), 1)]; + + let joined = join_vms(&vms, &rows); + let a = joined.get("a").expect("matched by thin id"); + assert_eq!(a.exclusive, 2000); + assert_eq!(a.referenced, Some(3000)); + assert_eq!(a.shared(), Some(1000)); + assert_eq!(a.provisioned, 1024 * 1024 * 1024); + } + + /// A record the pool cannot account for is absent from the map, not + /// present with zeroes. The CLI renders absent as `-`. + #[test] + fn join_omits_unaccountable_records() { + let rows = vec![row(42, 3000, 2000)]; + let vms = [ + // Never got a thin id (ZFS record, or a half-created VM). + vm_record("no-id", None, 1), + // Has an id the pool no longer knows about. + vm_record("stale", Some(999), 1), + ]; + + let joined = join_vms(&vms, &rows); + assert!(joined.is_empty(), "{joined:?}"); + } + + /// Thin ids the pool holds but no record claims (leaked staging + /// volumes, another install) contribute to the pool figure and must + /// not invent rows. + #[test] + fn join_ignores_rows_no_record_claims() { + let rows = vec![row(42, 3000, 2000), row(7, 500, 500)]; + assert_eq!(join_vms(&[vm_record("a", Some(42), 1)], &rows).len(), 1); + assert!(join_images(&[], &rows).is_empty()); + } } diff --git a/crates/ember-linux/src/lib.rs b/crates/ember-linux/src/lib.rs index 7286f31..7355fca 100644 --- a/crates/ember-linux/src/lib.rs +++ b/crates/ember-linux/src/lib.rs @@ -23,25 +23,33 @@ use ember_core::backend::{InitConfig, StorageBackend}; use ember_core::config::{GlobalConfig, StorageKind}; use ember_core::error::{Error, Result}; -/// Construct the active storage backend. +/// Construct the active storage backend, or explain why we cannot. /// -/// Returns the implementation indicated by [`GlobalConfig::storage_backend`]. -/// btrfs is not yet implemented; rather than silently routing through -/// the ZFS path with garbage inputs, the call panics so a hand-edited -/// `config.json` fails loudly. `init_storage` returns the same shape -/// of error from the init side. -pub fn create_storage(config: &GlobalConfig) -> Arc { +/// btrfs is not yet implemented. Rather than silently routing through +/// the ZFS path with garbage inputs, a hand-edited `config.json` naming +/// it gets a real error. `init_storage` returns the same shape of error +/// from the init side. +pub fn try_create_storage(config: &GlobalConfig) -> Result> { match config.storage_backend { - StorageKind::Zfs => Arc::new(LinuxStorage::new(config)), - StorageKind::DmThin => Arc::new(DmThinStorage::new(config)), - StorageKind::Btrfs => panic!( - "btrfs storage backend is not yet implemented; \ - config.json has storage_backend = btrfs but no \ - implementation exists yet" - ), + StorageKind::Zfs => Ok(Arc::new(LinuxStorage::new(config))), + StorageKind::DmThin => Ok(Arc::new(DmThinStorage::new(config))), + StorageKind::Btrfs => Err(Error::Config( + "config.json has storage_backend = btrfs, but the btrfs \ + storage backend is not implemented" + .to_string(), + )), } } +/// [`try_create_storage`] for callers that are about to operate on +/// storage anyway, where an unusable backend is fatal regardless. +/// +/// Read-only paths that must survive a broken config use the fallible +/// form instead. +pub fn create_storage(config: &GlobalConfig) -> Arc { + try_create_storage(config).unwrap_or_else(|e| panic!("{e}")) +} + /// Initialize storage during `ember init`. /// /// Dispatches to the concrete backend's `init` associated function. The diff --git a/crates/ember-linux/src/storage.rs b/crates/ember-linux/src/storage.rs index 7c7623e..9d5cf31 100644 --- a/crates/ember-linux/src/storage.rs +++ b/crates/ember-linux/src/storage.rs @@ -7,11 +7,14 @@ //! The struct holds the ZFS dataset paths (derived from [`GlobalConfig`]) so //! trait methods can construct full zvol paths from short names. +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::process::Command as ProcessCommand; use crate::zfs; -use ember_core::backend::{InitConfig, StorageBackend, VolumeHandle}; +use ember_core::backend::{ + InitConfig, PoolUsage, StorageBackend, StorageUsage, VolumeHandle, VolumeUsage, +}; use ember_core::config::size::ByteSize; use ember_core::config::GlobalConfig; use ember_core::error::{Error, Result}; @@ -24,6 +27,8 @@ pub struct LinuxStorage { /// ZFS pool name (e.g., "tank"). Cached so `deinit` can call /// `zpool destroy` without re-reading the config. pool: String, + /// Root of ember's dataset tree (e.g., "tank/ember"). + base_dataset: String, /// ZFS images dataset path (e.g., "tank/ember/images"). images_dataset: String, /// ZFS VMs dataset path (e.g., "tank/ember/vms"). @@ -37,6 +42,7 @@ impl LinuxStorage { pub fn new(config: &GlobalConfig) -> Self { Self { pool: config.pool.clone(), + base_dataset: config.base_dataset(), images_dataset: config.images_dataset(), vms_dataset: config.vms_dataset(), } @@ -277,6 +283,50 @@ impl StorageBackend for LinuxStorage { .collect()) } + fn usage(&self, vms: &[VmMetadata], images: &[ImageEntry]) -> Result { + let totals = zfs::usage::totals(&self.base_dataset)?; + let rows = zfs::usage::volumes(&self.base_dataset)?; + let by_name: HashMap<&str, &zfs::usage::VolumeRow> = + rows.iter().map(|r| (r.name.as_str(), r)).collect(); + + // `logicalused` covers data only, while `used` also carries + // every zvol's refreservation. Summing the reservations here + // lets the pool ratio compare like with like. There is no + // aggregate ZFS property for this, so it comes from the rows. + let reserved: u64 = rows.iter().map(|r| r.used_by_refreservation).sum(); + + // `disk_path` holds the dataset name on the ZFS backend, but a + // record written by a different backend (or a half-created VM) + // may not resolve. Those are dropped rather than zeroed. + let volume_usage = |dataset: &str| -> Option { + let row = by_name.get(dataset)?; + Some(VolumeUsage { + provisioned: row.volsize, + exclusive: row.used_by_dataset, + referenced: Some(row.referenced), + logical: Some(row.logical_referenced), + }) + }; + + Ok(StorageUsage { + pool: PoolUsage { + capacity: totals.used.saturating_add(totals.available), + allocated: totals.used, + reserved, + logical: Some(totals.logical_used), + metadata: None, + }, + vms: vms + .iter() + .filter_map(|vm| Some((vm.name.clone(), volume_usage(&vm.disk_path)?))) + .collect(), + images: images + .iter() + .filter_map(|img| Some((img.local_name.clone(), volume_usage(&img.disk_path)?))) + .collect(), + }) + } + /// Mount a block device (zvol) at a temporary directory. /// /// Waits for the device to appear if needed (ZFS zvols may take a moment diff --git a/crates/ember-linux/src/zfs.rs b/crates/ember-linux/src/zfs.rs index c45f153..cc89b05 100644 --- a/crates/ember-linux/src/zfs.rs +++ b/crates/ember-linux/src/zfs.rs @@ -1,6 +1,7 @@ pub mod dataset; pub mod pool; pub mod snapshot; +pub mod usage; pub mod volume; use std::process::Command; diff --git a/crates/ember-linux/src/zfs/usage.rs b/crates/ember-linux/src/zfs/usage.rs new file mode 100644 index 0000000..ac08527 --- /dev/null +++ b/crates/ember-linux/src/zfs/usage.rs @@ -0,0 +1,224 @@ +//! Space accounting queries against ZFS. +//! +//! Both queries use `-p` so ZFS emits exact byte counts rather than the +//! human-readable abbreviations `zfs list` prints by default. + +use std::process::Command; + +use ember_core::error::{Error, Result}; + +/// Per-volume accounting for one zvol. +/// +/// The occupancy figure is [`used_by_dataset`](Self::used_by_dataset) +/// and deliberately not the `used` property, even though `used` is what +/// a destroy would return to the pool. Two things inflate `used` past +/// what the volume physically holds: +/// +/// * A zvol from `zfs create -V` carries a refreservation for its whole +/// virtual size, which `used` counts as consumed. Our image volumes +/// report a `used` of 8.4 GiB against a `referenced` of 1.9 GiB. +/// * `usedbysnapshots` is by definition space the live volume no longer +/// references, so adding it would push occupancy past `referenced`. +/// +/// Clones carry no reservation and ember's fork snapshots hold almost +/// nothing, so in practice the two agree for VMs and diverge for images. +#[derive(Debug, PartialEq)] +pub struct VolumeRow { + /// Full dataset path, e.g. `tank/ember/vms/myvm`. + pub name: String, + pub volsize: u64, + /// Blocks referenced by the live volume and by nothing else. A + /// subset of `referenced` by ZFS's own definition. + pub used_by_dataset: u64, + /// Reserved but unwritten space, charged to the pool by + /// `refreservation`. Zero for clones. + pub used_by_refreservation: u64, + /// Addressable space including blocks shared with an origin. + pub referenced: u64, + /// Uncompressed size of `referenced`. + pub logical_referenced: u64, +} + +/// Dataset-tree totals, used for pool-level reporting. +#[derive(Debug, PartialEq)] +pub struct DatasetTotals { + pub used: u64, + pub available: u64, + pub logical_used: u64, +} + +/// Accounting for every zvol under `base`, recursively. +/// +/// One call covers both the `images/` and `vms/` subtrees. +pub fn volumes(base: &str) -> Result> { + let output = Command::new("zfs") + .args([ + "list", + "-Hp", + "-r", + "-t", + "volume", + "-o", + "name,volsize,usedbydataset,usedbyrefreservation,referenced,logicalreferenced", + base, + ]) + .output() + .map_err(|e| Error::CommandExec { + command: "zfs list".to_string(), + source: e, + })?; + let output = Error::check_command("zfs list volumes", output)?; + parse_volumes(&String::from_utf8_lossy(&output.stdout)) +} + +fn parse_volumes(stdout: &str) -> Result> { + stdout + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| { + let fields: Vec<&str> = line.split('\t').collect(); + if fields.len() != 6 { + return Err(Error::Zfs(format!( + "expected 6 tab-separated fields from `zfs list`, got {}: {line}", + fields.len() + ))); + } + Ok(VolumeRow { + name: fields[0].to_string(), + volsize: super::parse_u64(fields[1], "volsize")?, + used_by_dataset: super::parse_u64(fields[2], "usedbydataset")?, + used_by_refreservation: super::parse_u64(fields[3], "usedbyrefreservation")?, + referenced: super::parse_u64(fields[4], "referenced")?, + logical_referenced: super::parse_u64(fields[5], "logicalreferenced")?, + }) + }) + .collect() +} + +/// Totals for the dataset tree ember owns. +/// +/// We report against the dataset rather than the raw vdev (`zpool +/// list`) so that per-volume numbers sum into the pool figure, and so +/// quotas and sibling datasets on a shared pool are accounted for. +pub fn totals(base: &str) -> Result { + let output = Command::new("zfs") + .args([ + "get", + "-Hp", + "-o", + "value", + "used,available,logicalused", + base, + ]) + .output() + .map_err(|e| Error::CommandExec { + command: "zfs get".to_string(), + source: e, + })?; + let output = Error::check_command("zfs get totals", output)?; + parse_totals(&String::from_utf8_lossy(&output.stdout)) +} + +/// `zfs get` emits one line per property, in the order requested. +fn parse_totals(stdout: &str) -> Result { + let values: Vec<&str> = stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect(); + if values.len() != 3 { + return Err(Error::Zfs(format!( + "expected 3 property values from `zfs get`, got {}", + values.len() + ))); + } + Ok(DatasetTotals { + used: super::parse_u64(values[0], "used")?, + available: super::parse_u64(values[1], "available")?, + logical_used: super::parse_u64(values[2], "logicalused")?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Rows captured verbatim from a live pool: a clone (no + /// reservation) and an image volume (reserved by `zfs create -V`). + #[test] + fn parses_volume_rows() { + let out = + "ember/ember/vms/aj-dev\t214748364800\t104418334720\t0\t105790773760\t208040643072\n\ + ember/ember/images/ubuntu-dev\t6810501120\t2079834112\t6919027712\t2079834112\t4660178944\n"; + let rows = parse_volumes(out).unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].name, "ember/ember/vms/aj-dev"); + assert_eq!(rows[0].volsize, 214_748_364_800); + assert_eq!(rows[0].used_by_dataset, 104_418_334_720); + assert_eq!(rows[0].used_by_refreservation, 0); + assert_eq!(rows[0].referenced, 105_790_773_760); + assert_eq!(rows[0].logical_referenced, 208_040_643_072); + assert_eq!(rows[1].used_by_refreservation, 6_919_027_712); + } + + /// Regression, on the exact rows the live pool produces. An earlier + /// cut summed `usedbydataset + usedbysnapshots` for occupancy, and + /// since snapshot-only space is by definition outside `referenced`, + /// both image rows shipped with exclusive above referenced. + #[test] + fn occupancy_never_exceeds_referenced() { + let out = "\ +ember/ember/vms/aj-dev\t214748364800\t104418334720\t0\t105790773760\t208040643072 +ember/ember/vms/mz-dev\t214748364800\t8803586560\t0\t10869327872\t22587842560 +ember/ember/images/ubuntu-dev\t6810501120\t2079834112\t6919027712\t2079834112\t4660178944 +ember/ember/images/ubuntu-dev-new\t7147094016\t2158472704\t7260863488\t2158472704\t4885119488 +"; + for row in parse_volumes(out).unwrap() { + assert!( + row.used_by_dataset <= row.referenced, + "{}: occupancy {} exceeds referenced {}", + row.name, + row.used_by_dataset, + row.referenced + ); + } + } + + /// A pool with no zvols yet is not an error. + #[test] + fn parses_empty_listing() { + assert_eq!(parse_volumes("").unwrap(), vec![]); + assert_eq!(parse_volumes("\n").unwrap(), vec![]); + } + + /// ZFS emits `-` for properties that do not apply. We would rather + /// fail loudly than silently record a zero-sized volume. + #[test] + fn rejects_non_numeric_field() { + let out = "ember/ember/vms/x\t100\t-\t0\t100\t100\n"; + assert!(parse_volumes(out).is_err()); + } + + #[test] + fn rejects_short_row() { + assert!(parse_volumes("ember/ember/vms/x\t100\t100\n").is_err()); + } + + #[test] + fn parses_totals() { + let out = "320637513728\n196273849856\n643565009920\n"; + assert_eq!( + parse_totals(out).unwrap(), + DatasetTotals { + used: 320_637_513_728, + available: 196_273_849_856, + logical_used: 643_565_009_920, + } + ); + } + + #[test] + fn rejects_truncated_totals() { + assert!(parse_totals("320637513728\n196273849856\n").is_err()); + } +} diff --git a/crates/ember-macos/src/extents.rs b/crates/ember-macos/src/extents.rs new file mode 100644 index 0000000..e2e0d28 --- /dev/null +++ b/crates/ember-macos/src/extents.rs @@ -0,0 +1,275 @@ +//! Physical extent maps, and what a set of files really occupies. +//! +//! APFS clones share physical blocks, and `st_blocks` cannot see it: it +//! counts the blocks a file maps, not the ones it owns, so a fresh clone +//! reports its origin's full figure while costing nothing. The sharing +//! is visible one level down. `fcntl(F_LOG2PHYS_EXT)` maps a logical +//! offset to the physical byte range backing it, so two files that map +//! the same physical bytes are demonstrably sharing them. +//! +//! [`scan`] reads one file's extents. [`occupancy`] sweeps several +//! files' extents together and splits them into what each file holds +//! alone and what the whole set occupies. The split only means anything +//! across a whole set, which is why the caller hands us every volume at +//! once rather than asking per file. + +use std::os::unix::io::AsRawFd; +use std::path::Path; + +use ember_core::error::{Error, Result}; + +/// A contiguous run of physical bytes on the volume. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Extent { + /// Byte offset on the underlying device. + pub start: u64, + pub len: u64, +} + +/// What a set of files occupies once shared blocks are counted once. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Occupancy { + /// Bytes mapped by exactly one file, indexed as the input was. + pub exclusive: Vec, + /// Bytes mapped by at least one file, each counted a single time. + /// This is what the set actually costs on disk. + pub union: u64, +} + +/// Physical extents of `path`, or `None` if it no longer exists. +/// +/// Holes are skipped with `SEEK_DATA` rather than probed block by +/// block, which matters because a VM rootfs is mostly hole: an 8 GiB +/// image holding a 300 MiB filesystem would otherwise cost two million +/// syscalls to walk. +pub(crate) fn scan(path: &Path) -> Result>> { + let file = match std::fs::File::open(path) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(Error::Io { + path: path.to_path_buf(), + source: e, + }) + } + }; + let size = file + .metadata() + .map_err(|e| Error::Io { + path: path.to_path_buf(), + source: e, + })? + .len(); + + let fd = file.as_raw_fd(); + let mut extents = Vec::new(); + let mut offset = 0u64; + + while offset < size { + // SEEK_DATA lands on the next byte that is actually backed. + // ENXIO means there is none left, which is the normal way out + // of a file that ends in a hole. + let data = unsafe { nix::libc::lseek(fd, offset as i64, nix::libc::SEEK_DATA) }; + if data < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(nix::libc::ENXIO) { + break; + } + return Err(Error::Io { + path: path.to_path_buf(), + source: err, + }); + } + let data = data as u64; + if data >= size { + break; + } + + // On the way in, `l2p_devoffset` is the logical offset we are + // asking about and `l2p_contigbytes` is how much we would like + // covered. On the way out both describe the physical side. + let mut l2p: nix::libc::log2phys = unsafe { std::mem::zeroed() }; + l2p.l2p_devoffset = data as i64; + l2p.l2p_contigbytes = (size - data) as i64; + let rc = unsafe { nix::libc::fcntl(fd, nix::libc::F_LOG2PHYS_EXT, &mut l2p) }; + if rc < 0 { + let err = std::io::Error::last_os_error(); + // ERANGE means the range is not mappable, which we treat as + // the end of what we can account for rather than an error. + // SEEK_DATA promised data here, so this is not expected. + if err.raw_os_error() == Some(nix::libc::ERANGE) { + break; + } + return Err(Error::Io { + path: path.to_path_buf(), + source: err, + }); + } + + // A non-positive length would leave `offset` where it is and + // spin forever, so it ends the walk instead. + if l2p.l2p_contigbytes <= 0 { + break; + } + let len = l2p.l2p_contigbytes as u64; + extents.push(Extent { + start: l2p.l2p_devoffset as u64, + len, + }); + offset = data.saturating_add(len); + } + + Ok(Some(extents)) +} + +/// Split the physical bytes of several files into per-file exclusive +/// holdings and the union across all of them. +/// +/// A byte mapped by exactly one file is exclusive to it. A byte mapped +/// by several belongs to none of them exclusively, which is what makes +/// an image's `exclusive` fall to nothing while its clones live. +/// +/// Pure interval arithmetic, so it is testable without APFS underneath. +pub(crate) fn occupancy(files: &[Vec]) -> Occupancy { + // Sweep boundaries left to right, tracking how many files cover the + // segment we are standing on. + let mut events: Vec<(u64, i8, usize)> = Vec::new(); + for (idx, extents) in files.iter().enumerate() { + for e in extents { + if e.len == 0 { + continue; + } + events.push((e.start, 1, idx)); + events.push((e.start.saturating_add(e.len), -1, idx)); + } + } + events.sort_unstable(); + + let mut depth = vec![0i32; files.len()]; + let mut exclusive = vec![0u64; files.len()]; + let mut union = 0u64; + let mut covering = 0usize; + // While exactly one file covers the segment, the sum of the indices + // of the covering files is that file's index. Cheaper than scanning + // `depth` for the survivor at every boundary. + let mut index_sum = 0usize; + + let mut i = 0; + while i < events.len() { + let pos = events[i].0; + while i < events.len() && events[i].0 == pos { + let (_, delta, idx) = events[i]; + if delta > 0 { + depth[idx] += 1; + if depth[idx] == 1 { + covering += 1; + index_sum += idx; + } + } else { + depth[idx] -= 1; + if depth[idx] == 0 { + covering -= 1; + index_sum -= idx; + } + } + i += 1; + } + if i < events.len() && covering > 0 { + let seg = events[i].0 - pos; + union += seg; + if covering == 1 { + exclusive[index_sum] += seg; + } + } + } + + Occupancy { exclusive, union } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ext(start: u64, len: u64) -> Extent { + Extent { start, len } + } + + /// Pristine clones map the same bytes, so none of them holds + /// anything alone and the set costs one copy. + #[test] + fn pristine_clones_hold_nothing_exclusively() { + let shared = vec![ext(0, 1000)]; + let got = occupancy(&[shared.clone(), shared.clone(), shared]); + assert_eq!(got.exclusive, vec![0, 0, 0]); + assert_eq!(got.union, 1000); + } + + /// A clone that rewrote part of itself holds exactly what it + /// rewrote, and the origin keeps the part still shared. + #[test] + fn a_diverged_clone_holds_exactly_what_it_rewrote() { + let origin = vec![ext(0, 1000)]; + // First 400 bytes rewritten elsewhere, the rest still shared. + let clone = vec![ext(5000, 400), ext(400, 600)]; + let got = occupancy(&[origin, clone]); + assert_eq!(got.exclusive, vec![400, 400]); + assert_eq!(got.union, 1400); + } + + /// Sharing is not pairwise. A byte held by three files is exclusive + /// to none of them. + #[test] + fn a_byte_shared_three_ways_is_exclusive_to_none() { + let got = occupancy(&[ + vec![ext(0, 100)], + vec![ext(0, 100)], + vec![ext(0, 100), ext(100, 50)], + ]); + assert_eq!(got.exclusive, vec![0, 0, 50]); + assert_eq!(got.union, 150); + } + + /// The union counts a shared byte once, which is the whole reason + /// the pool figure cannot be a sum of the per-volume numbers. + #[test] + fn union_counts_a_shared_byte_once() { + let got = occupancy(&[vec![ext(0, 800)], vec![ext(0, 800)], vec![ext(0, 800)]]); + let sum_of_referenced: u64 = 800 * 3; + assert_eq!(got.union, 800); + assert!(got.union < sum_of_referenced); + } + + /// Extents that touch but do not overlap stay exclusive, and the + /// sweep must not merge them into a shared region at the seam. + #[test] + fn adjacent_extents_do_not_count_as_shared() { + let got = occupancy(&[vec![ext(0, 100)], vec![ext(100, 100)]]); + assert_eq!(got.exclusive, vec![100, 100]); + assert_eq!(got.union, 200); + } + + /// A file may map the same physical run twice. The union counts it + /// once and it stays exclusive to that file. + #[test] + fn a_file_mapping_a_run_twice_still_holds_it_alone() { + let got = occupancy(&[vec![ext(0, 100), ext(0, 100)]]); + assert_eq!(got.exclusive, vec![100]); + assert_eq!(got.union, 100); + } + + #[test] + fn empty_input_and_empty_files_are_zero() { + assert_eq!(occupancy(&[]).union, 0); + let got = occupancy(&[vec![], vec![ext(0, 0)]]); + assert_eq!(got.exclusive, vec![0, 0]); + assert_eq!(got.union, 0); + } + + /// Partial overlaps split into three regions: mine, ours, theirs. + #[test] + fn partial_overlap_splits_into_three_regions() { + let got = occupancy(&[vec![ext(0, 100)], vec![ext(60, 100)]]); + assert_eq!(got.exclusive, vec![60, 60]); + assert_eq!(got.union, 160); + } +} diff --git a/crates/ember-macos/src/lib.rs b/crates/ember-macos/src/lib.rs index 6e3f0a1..7d613be 100644 --- a/crates/ember-macos/src/lib.rs +++ b/crates/ember-macos/src/lib.rs @@ -1,3 +1,4 @@ +mod extents; pub mod image; pub mod network; pub mod platform; @@ -16,6 +17,13 @@ use ember_core::backend::{InitConfig, StorageBackend}; use ember_core::config::GlobalConfig; use ember_core::error::Result; +/// Construct the active storage backend. macOS has exactly one, so +/// this cannot fail; the signature matches Linux's, where a config can +/// name a backend that has no implementation. +pub fn try_create_storage(config: &GlobalConfig) -> Result> { + Ok(Arc::new(MacosStorage::new(config))) +} + /// Construct the active storage backend. pub fn create_storage(config: &GlobalConfig) -> Arc { Arc::new(MacosStorage::new(config)) diff --git a/crates/ember-macos/src/storage.rs b/crates/ember-macos/src/storage.rs index c9f8090..c72ab3d 100644 --- a/crates/ember-macos/src/storage.rs +++ b/crates/ember-macos/src/storage.rs @@ -11,13 +11,20 @@ //! └── rootfs.img # APFS clone of base image //! ``` +use std::collections::BTreeMap; use std::ffi::CString; use std::fs; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; use std::process::Command; -use ember_core::backend::{InitConfig, StorageBackend, VolumeHandle}; +use nix::sys::statvfs::statvfs; + +use crate::extents::{self, Extent}; + +use ember_core::backend::{ + InitConfig, PoolUsage, StorageBackend, StorageUsage, VolumeHandle, VolumeUsage, +}; use ember_core::config::size::ByteSize; use ember_core::error::{Error, Result}; use ember_core::image::registry::ImageEntry; @@ -67,6 +74,35 @@ impl MacosStorage { fn image_path(&self, name: &str) -> PathBuf { self.images_dir().join(format!("{name}.img")) } + + /// Read the physical extents of every disk image in the + /// installation, in a stable order. + /// + /// Files that vanish between the walk and the scan are dropped + /// rather than reported as empty, since a half-created VM is not + /// the same thing as one occupying nothing. + fn scan_tree(&self) -> Result> { + let mut paths = Vec::new(); + collect_images(&self.vms_dir(), &mut paths); + collect_images(&self.images_dir(), &mut paths); + paths.sort(); + + let mut scanned = Vec::with_capacity(paths.len()); + for path in paths { + let Ok(meta) = fs::metadata(&path) else { + continue; + }; + let Some(extents) = extents::scan(&path)? else { + continue; + }; + scanned.push(ScannedFile { + path, + len: meta.len(), + extents, + }); + } + Ok(scanned) + } } impl StorageBackend for MacosStorage { @@ -354,6 +390,77 @@ impl StorageBackend for MacosStorage { Ok(vec![]) } + /// APFS accounting comes from physical extent maps, not from + /// `st_blocks`. `st_blocks` counts the blocks a file maps rather + /// than the ones it owns, so a fresh clone reports its origin's + /// full figure while costing nothing. + /// + /// We scan every `.img` in the installation even when the caller + /// asks about one VM. The pool figure is installation-wide, and + /// exclusivity is not a property a volume has on its own: a file's + /// blocks are exclusive only relative to everything else that might + /// map them, so a narrower scan would report every volume as fully + /// exclusive. + fn usage(&self, vms: &[VmMetadata], images: &[ImageEntry]) -> Result { + let scanned = self.scan_tree()?; + + let occupancy = extents::occupancy( + &scanned + .iter() + .map(|f| f.extents.clone()) + .collect::>(), + ); + + let usage_of = |path: &Path| -> Option { + let index = scanned.iter().position(|f| f.path == path)?; + let file = &scanned[index]; + Some(VolumeUsage { + provisioned: file.len, + // A subset of this file's own extents, so the + // `exclusive <= referenced` invariant holds by + // construction rather than by arithmetic. + exclusive: occupancy.exclusive[index], + referenced: Some(file.referenced()), + logical: None, + }) + }; + + let vm_usage: BTreeMap = vms + .iter() + .filter_map(|vm| Some((vm.name.clone(), usage_of(&self.vm_rootfs(&vm.name))?))) + .collect(); + let image_usage: BTreeMap = images + .iter() + .filter_map(|img| { + Some(( + img.local_name.clone(), + usage_of(&self.image_path(&img.local_name))?, + )) + }) + .collect(); + + // There is no pool, so "allocated" is what ember occupies and + // capacity is that plus whatever the containing filesystem will + // still give us. Mirrors how the ZFS backend reports its + // dataset tree rather than the raw vdev. + let allocated = occupancy.union; + let available = available_bytes(&self.state_dir)?; + + Ok(StorageUsage { + pool: PoolUsage { + capacity: allocated.saturating_add(available), + allocated, + // Nothing on APFS is charged for space it has not + // written, so there is no reserved-but-empty gap. + reserved: 0, + logical: None, + metadata: None, + }, + vms: vm_usage, + images: image_usage, + }) + } + fn deinit(&self, purge: bool) -> Result<()> { // The state directory layout (`images/`, `vms/`, `kernels/`, // `network/`) is owned by ember; on `--purge` we drop the disk @@ -700,6 +807,55 @@ pub(crate) fn find_e2fsprogs_tool(name: &str) -> String { name.to_string() } +/// One disk image file and the physical bytes it maps. +struct ScannedFile { + path: PathBuf, + /// Logical length, the size the guest sees. + len: u64, + extents: Vec, +} + +impl ScannedFile { + /// Everything this file maps, shared blocks included. + fn referenced(&self) -> u64 { + self.extents.iter().map(|e| e.len).sum() + } +} + +/// Collect every `.img` file under `dir` into `out`, recursively. +/// +/// Missing or unreadable directories contribute nothing. This is a +/// report, and refusing to print a pool line because one VM directory +/// is unreadable would be worse than under-counting it. Symlinks are +/// not followed, so a link pointing back up the tree cannot make the +/// walk recurse forever or double-count a file against itself. +fn collect_images(dir: &Path, out: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let Ok(file_type) = entry.file_type() else { + continue; + }; + let path = entry.path(); + if file_type.is_dir() { + collect_images(&path, out); + } else if file_type.is_file() && path.extension().and_then(|e| e.to_str()) == Some("img") { + out.push(path); + } + } +} + +/// Bytes still available to an unprivileged writer on the filesystem +/// holding `path`. +fn available_bytes(path: &Path) -> Result { + let stat = statvfs(path).map_err(|e| Error::Io { + path: path.to_path_buf(), + source: std::io::Error::from(e), + })?; + Ok(stat.blocks_available() as u64 * stat.fragment_size() as u64) +} + /// Check whether the given path resides on an APFS volume. /// /// Runs `diskutil info ` and looks for `File System Personality: APFS` diff --git a/docs/BTRFS-SPEC.md b/docs/BTRFS-SPEC.md index e725c0a..3fa3187 100644 --- a/docs/BTRFS-SPEC.md +++ b/docs/BTRFS-SPEC.md @@ -506,9 +506,9 @@ The btrfs backend is structurally almost identical to the macOS APFS backend — ## Storage Efficiency Diagnostics -The existing `ember debug storage-efficiency` command (implemented for macOS/APFS) works unchanged for btrfs. It uses `st_blocks * 512` from `stat` to measure actual disk allocation per `.img` file — reflink clones on btrfs report reduced `st_blocks` just like APFS clones do, so the logical-vs-actual comparison and CoW ratio calculation are portable across both file-based backends. +`ember storage usage` works for btrfs the same way it does for macOS/APFS, but `st_blocks` is not the way to measure it. On APFS a reflinked clone reports its origin's full `st_blocks` while costing nothing, because the field counts the blocks a file maps rather than the ones it owns. Assume btrfs behaves the same until measured, and measure it before writing the backend rather than after. -Additionally, btrfs provides `btrfs filesystem du` which can show shared/exclusive/total space per file, giving more granular insight into CoW savings. This could be surfaced as an optional detail in the storage efficiency report but is not required for the initial implementation. +btrfs has the better tool for this anyway: `btrfs filesystem du` reports shared, exclusive and total per file directly, which maps onto `VolumeUsage` without a sweep of our own. That makes it the natural source for `exclusive` and `referenced`, where APFS has to reconstruct the same answer from physical extents. ## External Dependencies diff --git a/docs/DM-THIN-SPEC.md b/docs/DM-THIN-SPEC.md index 8010f28..4d892e7 100644 --- a/docs/DM-THIN-SPEC.md +++ b/docs/DM-THIN-SPEC.md @@ -594,11 +594,11 @@ it offers ZFS-like block-level CoW with no kernel module, at the cost of a more ## Storage efficiency diagnostics -`ember debug storage-efficiency` for dm-thin reports both per-volume and pool-level metrics: +`ember storage usage` for dm-thin reports both per-volume and pool-level metrics. See `STORAGE-USAGE-SPEC.md` for the model and the metadata-snapshot hazards. -* Per-volume virtual size: from the activated device's table. -* Per-volume exclusive blocks: from `thin_ls --metadata-snap=- /dev/loopMETA`. Computing this requires a metadata snapshot — taken under suspend or via `dmsetup message ember-pool 0 "reserve_metadata_snap"` — which has measurable overhead. The command surfaces it on demand only. -* Pool capacity, allocated, and free: from `dmsetup status ember-pool`. Output format: `/ /`. +* Per-volume virtual size: from the record's `disk_size_gib` / `size_mib`. +* Per-volume mapped and exclusive blocks: from `thin_ls -m` against the metadata loop device, under a reservation taken with `dmsetup message 0 "reserve_metadata_snap"`. Reading through metadata rather than `dmsetup status` also covers volumes that are not currently activated. +* Pool capacity, allocated, and free: from `dmsetup status `. Output format: `/ /`. The macOS `st_blocks` approach used by the btrfs and APFS backends does not apply — dm-thin volumes are block devices, not files, and `stat` on `/dev/mapper/...` reports no allocation. diff --git a/docs/MACOS-SPEC.md b/docs/MACOS-SPEC.md index d0db610..51ae698 100644 --- a/docs/MACOS-SPEC.md +++ b/docs/MACOS-SPEC.md @@ -207,30 +207,39 @@ resize2fs vms//rootfs.img ### The Problem -Unlike ZFS (where `zfs list -o used,refer` clearly shows per-dataset space usage and CoW savings), APFS has no per-file way to measure clone savings. Both `du` and Finder report clones as if they occupy full space. This means a user with 10 VMs cloned from a 2GB image would see `du` report 20GB even though actual disk usage is ~2GB. +ZFS answers this with a property lookup: `zfs list -o used,refer` shows per-dataset usage and CoW savings directly. APFS exposes no equivalent per-file field. Both `du` and Finder report clones as if they occupied full space, so a user with 10 VMs cloned from a 2GB image sees `du` report 20GB against an actual 2GB. -### `ember debug storage-efficiency` +The information is there, just not in `stat`. Physical extent maps show which bytes each file maps, so scanning the tree and comparing extents recovers the per-file answer. See the APFS section of `STORAGE-USAGE-SPEC.md`. -A built-in diagnostic command that reports CoW savings: +### `ember storage usage` + +A built-in diagnostic command that reports CoW savings. See `STORAGE-USAGE-SPEC.md` for the cross-platform model. ``` -$ ember debug storage-efficiency - -Storage Efficiency Report -───────────────────────── -Images: 2 (3.2 GB logical) -VMs: 8 (25.6 GB logical) - ────────────────── -Total logical: 28.8 GB -Actual disk used: 4.1 GB (via df) -CoW efficiency: 7.0x space savings +$ ember storage usage + +Pool 460 GiB capacity, 2 GiB used (0%), 458 GiB free + +VMS +NAME PROVISIONED REFERENCED EXCLUSIVE SHARED COMPRESSION +vm0 8 GiB 2 GiB 412 MiB 1.6 GiB - + +IMAGES +NAME PROVISIONED REFERENCED EXCLUSIVE SHARED COMPRESSION +alpine 2 GiB 1.6 GiB 0 B 1.6 GiB - ``` +`vm0` was cloned from `alpine`, so the 1.6 GiB they both map is charged to neither: the image holds nothing exclusively while the clone exists, and the pool counts those bytes once. The pool figure is 2 GiB, not the 3.6 GiB the two `REFERENCED` values add up to. + +APFS does not compress ember's disk images, so `COMPRESSION` renders as `-`. The other columns carry real numbers. + **How it works:** -1. **Logical size**: Sum of all `.img` file sizes via `stat` (apparent file size) -2. **Actual disk usage**: Sum of `st_blocks * 512` for each `.img` file — this reports actually-allocated 512-byte blocks, which reflects CoW sharing (APFS clones share blocks, so `st_blocks` is lower than the logical size) -3. **CoW ratio**: Logical size divided by actual disk usage +1. **Provisioned**: the `.img` file length via `stat`, the size the guest sees. +2. **Referenced**: the sum of the file's physical extent lengths, read with `fcntl(F_LOG2PHYS_EXT)`, which is everything the file maps whether or not it shares it. +3. **Exclusive and shared**: every `.img` file under `vms/` and `images/` is scanned and the extents are swept together. A physical byte mapped by exactly one file is exclusive to it, a byte mapped by several is shared. + +Note that `st_blocks` looks like a shortcut here and is not one. It counts the blocks a file maps, not the blocks it owns, so a fresh clone reports its origin's full figure while costing nothing. See the APFS section of `STORAGE-USAGE-SPEC.md`. ### `cp -c` Failure Detection @@ -263,7 +272,7 @@ As an additional safeguard, `ember vm create` measures the wall-clock time of th ``` Warning: disk clone took 3.2s — this may indicate copy-on-write is not working. -Run `ember debug storage-efficiency` to check. +Run `ember storage usage` to check. ``` ## Networking: vmnet (Shared Mode) diff --git a/docs/MACOS-TODO.md b/docs/MACOS-TODO.md index 295a0e7..6fd4670 100644 --- a/docs/MACOS-TODO.md +++ b/docs/MACOS-TODO.md @@ -44,7 +44,7 @@ ## Phase 2.5: Storage Efficiency Diagnostics -- [x] Implement `ember debug storage-efficiency` command +- [x] Implement the storage usage report (now `ember storage usage`) - [x] Report logical size (sum of all .img file sizes via stat) - [x] Report actual disk usage (df / diskutil apfs list) - [x] Report CoW efficiency ratio diff --git a/docs/STORAGE-USAGE-SPEC.md b/docs/STORAGE-USAGE-SPEC.md new file mode 100644 index 0000000..9c9346d --- /dev/null +++ b/docs/STORAGE-USAGE-SPEC.md @@ -0,0 +1,343 @@ +# Ember — Storage Usage Accounting + +Ember reports provisioned sizes everywhere and actual sizes nowhere. +`ember vm list` prints `VmMetadata.disk_size_gib`, `ember vm inspect` prints the same field, and `ember info` prints no capacity at all. +The only command that tries, `ember debug storage-efficiency`, walks `state_dir/images/data/*.img` and `state_dir/vms//rootfs.img`, which are paths that exist only on macOS. +On Linux it reports zero for everything. + +This spec adds one accounting method to `StorageBackend` and wires four commands to it. + +## What we want to be able to answer + +* How much disk does this VM actually occupy, and how much comes back if I delete it? +* How much is shared with the image or fork origin, and how much has diverged? +* How full is the pool, and how much of the saving is compression? + +The third question is also the measuring instrument for the compression work described in `DM-THIN-SPEC.md`. +Without it we cannot say what a compression layer would buy, nor whether it helped after enabling it. + +## The model + +This is an **occupancy** model. It answers where space has gone, not what a delete would give back. +Those are different questions on ZFS, and trying to make one field answer both is what makes the numbers incoherent. + +Four numbers per volume, all in bytes. + +| Field | Meaning | +|-------|---------| +| `provisioned` | Virtual size the guest sees. What we report today. | +| `exclusive` | Physical bytes this volume holds that are not shared with an origin. | +| `referenced` | Physical bytes reachable from this volume, shared blocks included. | +| `logical` | Uncompressed size of `referenced`. | + +`exclusive` is the number every backend can produce. +`referenced` and `logical` are optional because not every backend can measure them. + +The invariant that keeps the table readable is `exclusive <= referenced` whenever `referenced` is known. +Any definition of `exclusive` that can exceed what the volume references makes the derived shared column meaningless. + +Two quantities are derived rather than stored, so they cannot disagree with their inputs: + +* Shared bytes: `referenced - exclusive`. +* Compression ratio: `logical / referenced`. + +### What `exclusive` is not + +It is not what a destroy frees. On ZFS a volume is additionally charged for its refreservation and for blocks held only by its own snapshots, and neither is in `exclusive`. +Nor does it capture the origin side of a clone relationship: ZFS charges blocks shared between an origin and its clones to the origin, so an image whose clones all still exist reports occupancy for blocks that no single delete can reclaim. +dm-thin refcounts symmetrically and has neither problem. + +We accept the asymmetry rather than paper over it. Reclaim accounting on ZFS depends on the whole clone graph, and the pool line already tells a user how much room is left. + +### Types + +In `ember-core/src/backend.rs`, next to `VolumeHandle`: + +```rust +/// Space accounting for a single volume, in bytes. +pub struct VolumeUsage { + pub provisioned: u64, + /// Physical bytes this volume holds that are not shared with an + /// origin. Always within `referenced` when that is known. + pub exclusive: u64, + /// `None` when the backend cannot separate shared blocks from + /// exclusive ones. + pub referenced: Option, + /// Uncompressed size of `referenced`. `None` when the backend does + /// not compress. + pub logical: Option, +} + +/// Pool-wide capacity, in bytes. +pub struct PoolUsage { + pub capacity: u64, + pub allocated: u64, + /// Part of `allocated` that is reserved but holds no data, so it + /// compresses to nothing and stays out of the ratio. Zero for + /// backends without reservations. + pub reserved: u64, + /// Uncompressed size of the data within `allocated`. `None` when + /// the backend does not compress. + pub logical: Option, + /// Backends with a separate metadata device report it here. + pub metadata: Option, +} + +pub struct MetadataUsage { + pub capacity: u64, + pub used: u64, +} + +/// Accounting for a whole installation, produced in one pass. +pub struct StorageUsage { + pub pool: PoolUsage, + /// Keyed by `VmMetadata::name`. A missing key means the backend + /// could not account for that VM. + pub vms: BTreeMap, + /// Keyed by `ImageEntry::local_name`. + pub images: BTreeMap, +} +``` + +### Trait surface + +One method, not three: + +```rust +/// Measure space usage across the installation. +/// +/// Takes the state records rather than discovering volumes itself, +/// because the name-to-volume mapping lives in state and not in the +/// backend. Returns the whole set in one value so that backends which +/// have to walk pool-wide metadata do that walk once instead of once +/// per volume. +fn usage(&self, vms: &[VmMetadata], images: &[ImageEntry]) -> Result; +``` + +The batching is the point. +A per-volume `vm_usage(&VmMetadata)` would make dm-thin reserve a metadata snapshot and walk the mapping trees once per VM, and it could not express the APFS answer at all, where a volume's exclusive figure is only defined relative to every other volume that might share its blocks. + +No default implementation. +A new backend must decide what it can measure rather than silently inheriting zeros. + +## Backend mappings + +### ZFS + +Everything comes from two commands against `/`. + +Volumes, one call covering both the `images/` and `vms/` subtrees: + +``` +zfs list -Hp -r -t volume -o name,volsize,usedbydataset,usedbyrefreservation,referenced,logicalreferenced +``` + +| Field | ZFS property | +|-------|--------------| +| `provisioned` | `volsize` | +| `exclusive` | `usedbydataset` | +| `referenced` | `referenced` | +| `logical` | `logicalreferenced` | + +Two neighbouring properties look like better fits for `exclusive` and both break the invariant. + +`used` is the obvious choice, and it is what a destroy frees, but a zvol from `zfs create -V` carries a refreservation for its full virtual size and `used` counts that reservation as consumed. +Image volumes on a live pool report a `used` of 8.4 GiB against a `referenced` of 1.9 GiB. + +Adding `usedbysnapshots` fails for a subtler reason: that property is by definition space the live volume no longer references, so folding it in pushes occupancy outside `referenced`. +On a pool whose fork snapshots hold a single kilobyte, that is enough to make both image rows report `exclusive` above `referenced`. + +`usedbydataset` is a subset of `referenced` by ZFS's own definition, so the invariant holds by construction. + +Rows are matched to records by dataset name, which for ZFS is what `VmMetadata::disk_path` and `ImageEntry::disk_path` already hold. + +`-p` gives exact byte counts. We do not read `refcompressratio`, since `logicalreferenced / referenced` reproduces it and cannot drift from the other fields. + +Pool, one call: + +``` +zfs get -Hp -o value used,available,logicalused +``` + +`capacity` is `used + available` and `allocated` is `used`. +This deliberately describes the dataset tree ember owns rather than the raw vdev, so quotas and sibling datasets on a shared pool are accounted for and the free figure means what a user expects. + +`reserved` is the sum of `usedbyrefreservation` over the volume rows. +ZFS exposes no aggregate property for it, which is why it comes from the same listing rather than a third call. +It matters because `used` includes reservations and `logicalused` does not, so dividing one by the other counts empty reservation as perfectly compressed data. +On a live pool that understates compression by about 5%, 2.01x against a true 2.11x, and this report is meant to be the instrument we judge a compression layer by. + +The reservation is also why the volume rows do not sum to the pool line, so the CLI prints it as its own row rather than leaving a silent gap. + +`metadata` is `None`. + +### dm-thin + +Unlike every other method on the backend, `usage` does not activate the pool. +Measuring is a query, and `ember vm list` has no business loading a pool table, attaching loop devices, and running a full `thin_check` as a side effect of listing VMs. +An inactive pool produces an error saying so, which the best-effort callers render as `-`. + +Pool numbers are already parsed. +`pool::status` returns `PoolStatus` in blocks, so this is arithmetic on values we fetch today only to gate on health: + +* `capacity` = `total_data_blocks` × pool block size +* `allocated` = `used_data_blocks` × pool block size +* `metadata` = `total_metadata_blocks` and `used_metadata_blocks`, each × 4096, the fixed thin-pool metadata block size +* `reserved` = 0, since a thin volume is never charged for space it has not written +* `logical` = `None` + +Note the two different multipliers. Data is counted in pool blocks (64 KiB by default) and metadata in the kernel's fixed 4 KiB blocks, so using one scale for both misreports metadata by 16x. + +Per-volume numbers need a metadata snapshot, because the live metadata device is owned by the kernel and cannot be read directly: + +1. `dmsetup message 0 reserve_metadata_snap` +2. `thin_ls -m --no-headers -o DEV,MAPPED_BYTES,EXCLUSIVE_BYTES ` +3. `dmsetup message 0 release_metadata_snap` + +| Field | Source | +|-------|--------| +| `provisioned` | `disk_size_gib` / `size_mib` from the record | +| `exclusive` | `EXCLUSIVE_BYTES` | +| `referenced` | `MAPPED_BYTES` | +| `logical` | `None` | + +Rows are matched to records by thin id. +A record with no `thin_id`, and a `thin_id` the pool no longer knows about, are both omitted from the map rather than reported as zero. +Ids the pool holds that no record claims (a staging volume leaked by a failed image pull, another install) are ignored, so they show up in the pool figure without inventing a row. + +This path works for volumes that are not currently activated, which matters because dm-thin activates lazily and a stopped VM usually has no `/dev/mapper` entry. Reading `dmsetup status` on the thin device instead would only cover active volumes and would give mapped sectors without an exclusive count. + +Four hazards to handle: + +* **The snapshot is a single slot per pool.** `reserve_metadata_snap` fails with `EBUSY` when one is already held. Report that as a distinct error naming `dmsetup message 0 release_metadata_snap` as the remedy, because the usual cause is a stale reservation from a killed process, and a stale reservation also pins metadata blocks that the pool would otherwise reuse. +* **Release must happen on every path.** The release is done by a guard type whose `Drop` fires on early return and on panic, not by a trailing statement. +* **A signal still leaks it.** `Drop` does not run on SIGINT, so Ctrl-C during a `thin_ls` scan strands a reservation and the next reader sees the EBUSY above. We accept that and make the error tell the operator how to clear it, rather than installing a signal handler for one command. +* **We never force-release.** A reservation we did not take may belong to another process. We fail with the message above instead of stealing it. + +When the reservation or the scan fails, the whole call fails, even though the pool figures were already in hand. `ember storage usage` exists to measure, so a partial answer that looks complete is worse than an error. + +### APFS + +Sharing between APFS clones is visible from userspace. `fcntl(F_LOG2PHYS_EXT)` maps a logical offset in a file to the physical byte range backing it, and `SEEK_DATA` skips holes without walking them. Reading a file's extents tells us which physical bytes it maps, and two files that map the same bytes are sharing them. + +`st_blocks` cannot answer this and must not be used for it. It counts the blocks a file maps, not the blocks it owns, so a fresh clone reports its origin's full figure while costing nothing. It separates allocated extents from holes, not shared blocks from unshared ones. Summing it over an install of one image and fifteen clones overstates real occupancy by more than 5x, and feeding that sum into `capacity` makes the reported capacity grow every time a free clone is made. + +So we scan rather than stat: + +1. Walk `vms/` and `images/` for every `.img` file. +2. Read each one's physical extents. +3. Sweep all the extents together. A physical byte mapped by exactly one file is exclusive to that file, a byte mapped by several is shared, and the count of distinct bytes mapped is the tree's real occupancy. + +| Field | Source | +|-------|--------| +| `provisioned` | file length | +| `exclusive` | bytes in extents no other file in the tree maps | +| `referenced` | sum of the file's extent lengths, which equals `st_blocks` × 512 | +| `logical` | `None`, ember does not compress its disk images | + +`exclusive <= referenced` holds by construction, since the exclusive bytes are a subset of the file's own extents. + +The whole tree is scanned even when the caller passes a single record, and two separate things depend on that. The pool figure is installation-wide while a caller such as `vm inspect` hands the backend one VM. And exclusivity is not a property of a volume on its own: a file's blocks are exclusive only relative to everything else that might map them, so a sweep restricted to the requested records would report every volume as fully exclusive. + +That is the second reason the trait batches. The dm-thin argument is about doing one expensive walk instead of many. On APFS, batching is what makes the answer computable at all. + +Pool numbers mirror the ZFS treatment so the two read the same way. `allocated` is the union of every extent in the tree, each shared byte counted once, and `capacity` is that plus the containing filesystem's available space. `reserved` is 0 and `metadata` is `None`. + +Two limits we state rather than chase: + +* Exclusive means "not shared with another volume ember owns". Blocks shared with a file outside the tree, a user's own `cp -c` of a rootfs for instance, are counted as exclusive. This is the same species of gap as the ZFS asymmetry above. +* As on ZFS, `exclusive` is not what a delete frees. An APFS snapshot can hold the blocks after the file is gone. + +The cost is a full extent scan per report. APFS coalesces aggressively, so extent counts track the number of data islands rather than file size: 4 GiB of contiguous data is around 130 extents, while a sparse ext4 rootfs with metadata scattered through it runs a few thousand. A tree of one image, two VMs and three forks scans in about 25 ms. Should that ever become a problem, the cheap path is available without changing the model, since `vm list` and `info` are best-effort and can stat for `referenced` alone and leave the sweep to `ember storage usage`. + +## CLI surface + +### `ember storage usage` + +New subcommand next to `ember storage grow`. + +Captured from a live ZFS pool: + +``` +$ ember storage usage + +Pool 481.4 GiB capacity, 297.5 GiB used (62%), 183.9 GiB free +Compression 599.3 GiB logical -> 284.3 GiB on disk (2.11x) +Reserved 13.2 GiB charged to the pool but holding no data + +VMS +NAME PROVISIONED REFERENCED EXCLUSIVE SHARED COMPRESSION +aj-dev 200 GiB 98.5 GiB 97.2 GiB 1.3 GiB 1.97x +mz-dev 200 GiB 10.1 GiB 8.2 GiB 1.9 GiB 2.08x +mz-dev-auto-scaling 200 GiB 92.5 GiB 89.7 GiB 2.8 GiB 2.13x +mz-dev-bugs 200 GiB 88.7 GiB 85.3 GiB 3.4 GiB 2.22x + +IMAGES +NAME PROVISIONED REFERENCED EXCLUSIVE SHARED COMPRESSION +ubuntu-dev 6.3 GiB 1.9 GiB 1.9 GiB 0 B 2.24x +ubuntu-dev-new 6.7 GiB 2 GiB 2 GiB 0 B 2.26x +``` + +Columns whose backing field is `None` render `-`. +On dm-thin that means the `COMPRESSION` column is `-` throughout and the `Compression` line is omitted, and a `Metadata` line appears instead showing metadata device usage. +The `Reserved` line appears only when a backend has reservations, so only on ZFS. + +`--format json` emits `StorageUsage` directly, matching the `OutputFormat` enum the other commands use. + +This command reports backend errors as errors. It is the one place where being unable to measure is a failure rather than a blank. + +### `ember vm list` + +Gains a `USED` column showing `exclusive`, next to the existing provisioned `DISK`. + +Usage here is best-effort. If `usage()` fails, every row renders `-` and the listing still succeeds. Listing VMs must not start depending on a healthy pool, since one common reason to list them is that storage is broken. + +### `ember vm inspect` + +Gains `Used`, `Referenced`, `Shared`, and `Compression` rows, omitting the ones whose field is `None`. Best-effort, same rule as `vm list`. + +### `ember info` + +Gains a `Capacity` line, and a compression line when the backend compresses. Best-effort. +Not labelled `Pool`, because `info_extra` already prints `ZFS pool ` and two unrelated rows called pool read as a contradiction. + +### Removed + +`ember debug storage-efficiency` is deleted, and with it the `debug` subcommand tree, which has no other members. Its useful half is `ember storage usage` and its APFS-specific half is now `MacosStorage::usage`. + +References in `README.md`, `MACOS-SPEC.md`, `MACOS-TODO.md`, `BTRFS-SPEC.md`, `DM-THIN-SPEC.md`, and `TEST-SPEC.md` are updated to the new command, and `tests/macos_storage.rs` is retargeted at it. + +### Where the best-effort helper lives + +`try_usage` sits in `src/backend.rs`, alongside `create_storage`, not in the `storage` subcommand module. +It is not a `storage` subcommand concern, it is how non-storage commands ask for usage, and putting it under `cli::storage` would make `cli::vm` and `cli::storage` import each other. + +It builds the backend through `try_create_storage`, the fallible sibling of `create_storage`. The infallible form panics on a config naming an unimplemented backend, and a panic is not best-effort: `vm list` and `info` are exactly the commands someone runs to diagnose a bad config. + +## Testing + +Unit tests, no root required. Note that the workspace sets `default-members = ["."]`, so a bare `cargo test` runs only the root package. The backend crates need naming: `cargo test -p ember-core -p ember-macos` on a Mac, and `-p ember-linux` on Linux, where the ext4 helpers those tests shell out to actually exist. + +* `thin_ls` output parsing: the empty pool, a header row that `--no-headers` failed to suppress, and short rows. +* `zfs list -Hp` row parsing, including the `-` ZFS prints for inapplicable properties. +* Occupancy never exceeding `referenced`, checked against the four rows a live pool actually produces rather than a hand-written fixture. An earlier cut passed its own guard test because the fixture zeroed the one field that broke the invariant. +* Metadata block accounting from a `PoolStatus`, pinning that data and metadata use different multipliers. +* The thin-id join: a record matched to its row, a record with no id, a record with a stale id, and a row no record claims. +* Derived shared bytes and compression ratio, including the divide-by-zero guards and the saturating subtraction. +* Rendering of `-` for every `None` field. +* The APFS extent sweep, as pure interval logic over synthetic extents so it needs no APFS to run: pristine clones share everything and hold nothing exclusively, a partly rewritten clone holds exactly what it rewrote, a byte shared three ways is exclusive to none of them, and the union counts each shared byte once. + +Integration tests in `tests/storage_usage.rs`, using the existing `TestEnv`. All of them are `#[ignore]`d, matching every other file in `tests/`, because `TestEnv` builds a real backend and needs root on Linux. `run-integration-tests.sh` passes `--ignored`, so a test left un-ignored would be skipped by the project runner and would break a bare `cargo test`. + +* After `ember vm create`, `ember storage usage` lists the VM with `exclusive > 0` and `exclusive <= referenced`. +* The image row satisfies the same invariant, which is where the refreservation trap bites. +* After a fork, the fork's `referenced` exceeds its `exclusive`, which is the sharing every CoW backend is supposed to deliver, APFS included. Forks are created with `--no-start`, since `TestEnv` installs a kernel that cannot boot. +* After a fork, the pool's `allocated` is below the sum of the volumes' `referenced`. This is the property that fails when a backend counts a shared block once per clone, and it is checked against a real fork rather than a fixture, because the assumption that broke here was one no fixture would have questioned. macOS-only for now: the same should hold on ZFS, but `allocated` there also carries refreservation and snapshot charges, so the honest assertion is a different one and belongs with someone who can run it against a pool. +* `ember vm list` still succeeds and its row ends in `-` when the backend cannot report. Linux-only: the break is a `config.json` naming a nonexistent pool, and the APFS backend reads neither `pool` nor `storage_path`. +* The dm-thin variant creates an image and a VM so the thin-id join is actually exercised, then calls the command a third time to prove the metadata snapshot was released. + +## Out of scope + +* Per-snapshot accounting. Snapshots are visible in the ZFS numbers via `used` but get no rows of their own. +* Historical tracking or deltas over time. +* Any accounting for a compression layer that does not exist yet. When one is added, its physical-versus-logical figures land in `PoolUsage::logical` and `VolumeUsage::logical`, which are already shaped for it. diff --git a/docs/TEST-SPEC.md b/docs/TEST-SPEC.md index 94c4409..2162b2a 100644 --- a/docs/TEST-SPEC.md +++ b/docs/TEST-SPEC.md @@ -65,7 +65,7 @@ tests/ resize.rs # UNIFIED — grow, shrink-fails, multiple-grows, metadata check fork.rs # UNIFIED — basic, overrides, delete-cleanup, error cases ssh.rs # UNIFIED — exec, cp, exec-on-stopped-fails - macos_storage.rs # macOS-only: APFS clone efficiency, HFS+ fallback, storage-efficiency cmd + macos_storage.rs # macOS-only: APFS clone efficiency, HFS+ fallback, storage usage macos_ember_vz.rs # macOS-only: low-level ember-vz component tests (optional, for debugging) ``` @@ -213,7 +213,7 @@ Remove `#![cfg(target_os = "linux")]`. After steps 4-5 extracted snapshot/resize tests, `macos_storage.rs` retains only APFS-specific tests: - `apfs_clone_does_not_reduce_free_space` -- `storage_efficiency_shows_savings` +- `storage_usage_reports_images_and_vms` - `vm_delete_removes_storage` - `cp_c_fails_gracefully_on_non_apfs` diff --git a/src/backend.rs b/src/backend.rs index 9e2d353..7b9b166 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -34,9 +34,28 @@ pub type Network = ember_macos::MacosNetwork; pub type Storage = Arc; #[cfg(target_os = "linux")] -pub use ember_linux::{create_storage, init_storage}; +pub use ember_linux::{create_storage, init_storage, try_create_storage}; #[cfg(target_os = "macos")] -pub use ember_macos::{create_storage, init_storage}; +pub use ember_macos::{create_storage, init_storage, try_create_storage}; + +/// Best-effort space accounting, for commands where storage is not the +/// subject: `vm list`, `vm inspect`, `info`. +/// +/// Those must keep working when storage cannot be measured, since one +/// common reason to run them is that storage is broken. Callers render +/// a dash for whatever comes back missing. `ember storage usage` is the +/// strict counterpart and reports the error instead. +/// +/// Note that the maps are restricted to the records passed in while the +/// pool figures are always installation-wide, so a caller asking about +/// one VM still gets whole-pool capacity. +pub fn try_usage( + config: &ember_core::config::GlobalConfig, + vms: &[ember_core::state::vm::VmMetadata], + images: &[ember_core::image::registry::ImageEntry], +) -> Option { + try_create_storage(config).ok()?.usage(vms, images).ok() +} #[cfg(target_os = "linux")] pub type CurrentPlatform = ember_linux::LinuxPlatform; diff --git a/src/cli.rs b/src/cli.rs index 3d37c05..315fc7e 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,5 +1,4 @@ pub mod cp; -pub mod debug; pub mod deinit; pub mod exec; pub(crate) mod fmt; @@ -70,10 +69,6 @@ pub enum Command { /// Show ember configuration and status overview Info, - /// Debugging and diagnostics - #[command(subcommand)] - Debug(debug::DebugCommand), - /// Reconcile internal state with actual VM process state Reconcile, diff --git a/src/cli/debug.rs b/src/cli/debug.rs deleted file mode 100644 index b3d6a2d..0000000 --- a/src/cli/debug.rs +++ /dev/null @@ -1,152 +0,0 @@ -//! Debug commands for inspecting ember internals. - -use std::path::Path; - -#[cfg(unix)] -use std::os::unix::fs::MetadataExt; - -use clap::Subcommand; - -#[derive(Subcommand)] -pub enum DebugCommand { - /// Report CoW storage efficiency (logical vs actual disk usage) - StorageEfficiency, -} - -pub fn run(cmd: &DebugCommand, state_dir: &Path) -> anyhow::Result<()> { - match cmd { - DebugCommand::StorageEfficiency => storage_efficiency(state_dir), - } -} - -/// Report storage efficiency by comparing logical file sizes against -/// actual disk usage. -/// -/// Logical size: sum of all `.img` file sizes via `stat` (what `du` reports). -/// Actual disk usage: free space delta on the volume, approximated by -/// subtracting current free space from volume capacity and comparing -/// against logical totals. -fn storage_efficiency(state_dir: &Path) -> anyhow::Result<()> { - let images_dir = state_dir.join("images").join("data"); - let vms_dir = state_dir.join("vms"); - - // Count images and their logical sizes. - let (image_count, image_bytes) = count_img_files(&images_dir); - - // Count VM rootfs files and their logical sizes. - let mut vm_count: u64 = 0; - let mut vm_bytes: u64 = 0; - - if vms_dir.exists() { - if let Ok(entries) = std::fs::read_dir(&vms_dir) { - for entry in entries.flatten() { - let vm_dir = entry.path(); - if !vm_dir.is_dir() { - continue; - } - - // Count rootfs.img for this VM. - let rootfs = vm_dir.join("rootfs.img"); - if let Ok(meta) = std::fs::metadata(&rootfs) { - vm_count += 1; - vm_bytes += meta.len(); - } - } - } - } - - let total_logical = image_bytes + vm_bytes; - - // Get actual disk usage by summing allocated blocks for all .img files. - // On APFS, cloned files only report their unique (non-shared) blocks, - // so this correctly reflects CoW savings. - let actual_used = get_actual_disk_bytes(state_dir); - - println!(); - println!("Storage Efficiency Report"); - println!("{}", "─".repeat(40)); - println!( - "Images: {:>3} ({} logical)", - image_count, - format_bytes(image_bytes) - ); - println!( - "VMs: {:>3} ({} logical)", - vm_count, - format_bytes(vm_bytes) - ); - println!(" {}", "─".repeat(22)); - println!("Total logical: {}", format_bytes(total_logical)); - - if let Some(used) = actual_used { - println!("Actual disk used: {}", format_bytes(used)); - if used > 0 && total_logical > used { - let ratio = total_logical as f64 / used as f64; - println!("CoW efficiency: {:.1}x space savings", ratio); - } - } else { - println!("Actual disk used: (could not determine)"); - } - println!(); - - Ok(()) -} - -/// Count `.img` files in a directory and sum their logical sizes. -fn count_img_files(dir: &Path) -> (u64, u64) { - let mut count: u64 = 0; - let mut bytes: u64 = 0; - - if let Ok(entries) = std::fs::read_dir(dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().and_then(|e| e.to_str()) == Some("img") { - if let Ok(meta) = std::fs::metadata(&path) { - count += 1; - bytes += meta.len(); - } - } - } - } - - (count, bytes) -} - -/// Get actual disk bytes used by all `.img` files under the state directory. -/// -/// Uses `st_blocks` from file metadata, which reports 512-byte blocks -/// actually allocated on disk. On APFS, cloned files only count their -/// unique (non-shared) blocks, so this correctly reflects CoW savings. -#[cfg(unix)] -fn get_actual_disk_bytes(state_dir: &Path) -> Option { - let mut total_blocks: u64 = 0; - sum_img_blocks(state_dir, &mut total_blocks); - // st_blocks counts 512-byte blocks. - Some(total_blocks * 512) -} - -#[cfg(not(unix))] -fn get_actual_disk_bytes(_state_dir: &Path) -> Option { - None -} - -/// Recursively walk a directory and sum `st_blocks` for all `.img` files. -#[cfg(unix)] -fn sum_img_blocks(dir: &Path, total: &mut u64) { - let entries = match std::fs::read_dir(dir) { - Ok(e) => e, - Err(_) => return, - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - sum_img_blocks(&path, total); - } else if path.extension().and_then(|e| e.to_str()) == Some("img") { - if let Ok(meta) = std::fs::metadata(&path) { - *total += meta.blocks(); - } - } - } -} - -use super::fmt::format_bytes_binary as format_bytes; diff --git a/src/cli/fmt.rs b/src/cli/fmt.rs index 017b1b0..57bf0a9 100644 --- a/src/cli/fmt.rs +++ b/src/cli/fmt.rs @@ -70,10 +70,55 @@ pub fn format_bytes_binary(bytes: u64) -> String { } } +/// Placeholder for a figure the storage backend could not measure. +/// +/// Distinct from a measured zero, which renders as `0 B`. +pub const UNKNOWN: &str = "-"; + +/// [`format_bytes_binary`] for a value the backend may not know. +pub fn format_bytes_opt(bytes: Option) -> String { + bytes.map_or_else(|| UNKNOWN.to_string(), format_bytes_binary) +} + +/// Format a compression ratio as `2.01x`. +pub fn format_ratio(ratio: Option) -> String { + ratio.map_or_else(|| UNKNOWN.to_string(), |r| format!("{r:.2}x")) +} + +/// Format a fill level as a whole percentage. A zero-capacity pool +/// reads as 0% rather than dividing by zero. +pub fn format_percent(used: u64, capacity: u64) -> String { + if capacity == 0 { + return "0%".to_string(); + } + format!("{:.0}%", (used as f64 / capacity as f64) * 100.0) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn unknown_values_render_as_dash() { + assert_eq!(format_bytes_opt(None), "-"); + assert_eq!(format_ratio(None), "-"); + // A measured zero must stay distinguishable from unknown. + assert_eq!(format_bytes_opt(Some(0)), "0 B"); + } + + #[test] + fn ratios_keep_two_decimals() { + assert_eq!(format_ratio(Some(2.0)), "2.00x"); + assert_eq!(format_ratio(Some(1.9666)), "1.97x"); + } + + #[test] + fn percentages_round_and_guard_zero_capacity() { + assert_eq!(format_percent(0, 0), "0%"); + assert_eq!(format_percent(1, 4), "25%"); + assert_eq!(format_percent(2, 3), "67%"); + } + #[test] fn whole_values_have_no_decimal() { assert_eq!(format_bytes_binary(512 * MIB), "512 MiB"); diff --git a/src/cli/info.rs b/src/cli/info.rs index 4232864..06bc95e 100644 --- a/src/cli/info.rs +++ b/src/cli/info.rs @@ -1,6 +1,7 @@ use std::path::Path; use crate::backend::{CurrentPlatform, Platform}; +use crate::cli::fmt::{format_bytes_binary, format_percent, format_ratio}; use ember_core::config::GlobalConfig; use ember_core::image::registry::ImageRegistry; use ember_core::state::store::StateStore; @@ -42,5 +43,26 @@ pub fn run(state_dir: &Path) -> anyhow::Result<()> { println!("Images: {}", images.len()); println!("VMs: {} ({} running)", vms.len(), running); + // Best-effort: `ember info` is the command you reach for when + // something is wrong, so it must not fail just because the pool + // cannot be measured. `ember storage usage` is the strict version. + if let Some(usage) = crate::backend::try_usage(&config, &vms, &images.images) { + let pool = &usage.pool; + println!( + "Capacity: {} of {} used ({}), {} free", + format_bytes_binary(pool.allocated), + format_bytes_binary(pool.capacity), + format_percent(pool.allocated, pool.capacity), + format_bytes_binary(pool.free()), + ); + if let (Some(logical), Some(ratio)) = (pool.logical, pool.compression_ratio()) { + println!( + "Compression: {} logical ({})", + format_bytes_binary(logical), + format_ratio(Some(ratio)), + ); + } + } + Ok(()) } diff --git a/src/cli/storage.rs b/src/cli/storage.rs index 785a052..812470d 100644 --- a/src/cli/storage.rs +++ b/src/cli/storage.rs @@ -1,18 +1,28 @@ //! `ember storage` subcommands: pool-level administration. +use std::collections::BTreeMap; use std::path::Path; use clap::{Args, Subcommand}; -use crate::backend::create_storage; +use crate::backend::{create_storage, StorageUsage, VolumeUsage}; +use crate::cli::fmt::{ + format_bytes_binary, format_bytes_opt, format_percent, format_ratio, print_table, Align, +}; +use crate::cli::vm::OutputFormat; use ember_core::config::size::ByteSize; use ember_core::config::GlobalConfig; +use ember_core::image::registry::ImageRegistry; use ember_core::state::store::StateStore; +use ember_core::state::vm; #[derive(Subcommand)] pub enum StorageCommand { /// Grow the underlying pool capacity (dm-thin only). Grow(GrowArgs), + + /// Report actual disk usage per VM and image. + Usage(UsageArgs), } #[derive(Args)] @@ -23,9 +33,17 @@ pub struct GrowArgs { pub size: ByteSize, } +#[derive(Args)] +pub struct UsageArgs { + /// Output format + #[arg(long, value_enum, default_value_t = OutputFormat::Table)] + pub format: OutputFormat, +} + pub fn run(cmd: &StorageCommand, state_dir: &Path) -> anyhow::Result<()> { match cmd { StorageCommand::Grow(args) => grow(args, state_dir), + StorageCommand::Usage(args) => usage(args, state_dir), } } @@ -36,3 +54,103 @@ fn grow(args: &GrowArgs, state_dir: &Path) -> anyhow::Result<()> { storage.grow(args.size)?; Ok(()) } + +/// Unlike the best-effort usage shown by `vm list` and `info`, this +/// command reports a backend failure as a failure. Being unable to +/// measure is the one thing it exists to do. +fn usage(args: &UsageArgs, state_dir: &Path) -> anyhow::Result<()> { + let store = StateStore::new(state_dir.to_path_buf()); + let config: GlobalConfig = store.read(&store.config_path())?; + let storage = create_storage(&config); + + let vms = vm::list(&store)?; + let images = ImageRegistry::load(&store)?; + let usage = storage.usage(&vms, &images.images)?; + + match args.format { + OutputFormat::Json => println!("{}", serde_json::to_string_pretty(&usage)?), + OutputFormat::Table => print_usage(&usage), + } + Ok(()) +} + +fn print_usage(usage: &StorageUsage) { + let pool = &usage.pool; + println!(); + println!( + "Pool {} capacity, {} used ({}), {} free", + format_bytes_binary(pool.capacity), + format_bytes_binary(pool.allocated), + format_percent(pool.allocated, pool.capacity), + format_bytes_binary(pool.free()), + ); + // Against `occupied`, not `allocated`: the ratio divides by it, and + // printing the larger figure here would make the line contradict + // its own arithmetic. + if let (Some(logical), Some(ratio)) = (pool.logical, pool.compression_ratio()) { + println!( + "Compression {} logical -> {} on disk ({})", + format_bytes_binary(logical), + format_bytes_binary(pool.occupied()), + format_ratio(Some(ratio)), + ); + } + if pool.reserved > 0 { + println!( + "Reserved {} charged to the pool but holding no data", + format_bytes_binary(pool.reserved), + ); + } + if let Some(meta) = pool.metadata { + println!( + "Metadata {} of {} used ({})", + format_bytes_binary(meta.used), + format_bytes_binary(meta.capacity), + format_percent(meta.used, meta.capacity), + ); + } + + print_section("VMS", &usage.vms); + print_section("IMAGES", &usage.images); + println!(); +} + +fn print_section(heading: &str, volumes: &BTreeMap) { + if volumes.is_empty() { + return; + } + println!(); + println!("{heading}"); + let rows: Vec> = volumes + .iter() + .map(|(name, u)| { + vec![ + name.clone(), + format_bytes_binary(u.provisioned), + format_bytes_opt(u.referenced), + format_bytes_binary(u.exclusive), + format_bytes_opt(u.shared()), + format_ratio(u.compression_ratio()), + ] + }) + .collect(); + print_table( + &[ + "NAME", + "PROVISIONED", + "REFERENCED", + "EXCLUSIVE", + "SHARED", + "COMPRESSION", + ], + &[ + Align::Left, + Align::Right, + Align::Right, + Align::Right, + Align::Right, + Align::Right, + ], + &rows, + ); +} diff --git a/src/cli/vm.rs b/src/cli/vm.rs index 93c8dc6..34bf6c7 100644 --- a/src/cli/vm.rs +++ b/src/cli/vm.rs @@ -3,7 +3,9 @@ use std::path::{Path, PathBuf}; use clap::{Args, Subcommand}; use uuid::Uuid; -use super::fmt::{format_bytes_binary, print_table, Align, GIB, MIB}; +use super::fmt::{ + format_bytes_binary, format_bytes_opt, format_ratio, print_table, Align, GIB, MIB, +}; use crate::backend::{ create_storage, CurrentPlatform, Network, NetworkBackend, Platform, Storage, Vm, VmBackend, VolumeHandle, @@ -1441,9 +1443,21 @@ fn list(args: &ListArgs, state_dir: &Path) -> anyhow::Result<()> { return Ok(()); } + // Usage is a nicety here, not the point of the command, so + // a backend that cannot answer leaves the column blank + // rather than failing the listing. + let usage = store + .read::(&store.config_path()) + .ok() + .and_then(|config| crate::backend::try_usage(&config, &vms, &[])); + let rows: Vec> = vms .iter() .map(|vm| { + let used = usage + .as_ref() + .and_then(|u| u.vms.get(&vm.name)) + .map(|u| u.exclusive); vec![ vm.name.clone(), vm.status.to_string(), @@ -1451,11 +1465,12 @@ fn list(args: &ListArgs, state_dir: &Path) -> anyhow::Result<()> { vm.cpus.to_string(), format_bytes_binary(vm.memory_mib as u64 * MIB), format_bytes_binary(vm.disk_size_gib as u64 * GIB), + format_bytes_opt(used), ] }) .collect(); print_table( - &["NAME", "STATUS", "IMAGE", "CPUS", "MEM", "DISK"], + &["NAME", "STATUS", "IMAGE", "CPUS", "MEM", "DISK", "USED"], &[ Align::Left, Align::Left, @@ -1463,6 +1478,7 @@ fn list(args: &ListArgs, state_dir: &Path) -> anyhow::Result<()> { Align::Right, Align::Right, Align::Right, + Align::Right, ], &rows, ); @@ -1495,6 +1511,26 @@ fn inspect(args: &InspectArgs, state_dir: &Path) -> anyhow::Result<()> { "Disk: {}", format_bytes_binary(metadata.disk_size_gib as u64 * GIB) ); + // Best-effort, same rule as `vm list`. + let usage = store + .read::(&store.config_path()) + .ok() + .and_then(|config| { + crate::backend::try_usage(&config, std::slice::from_ref(&metadata), &[]) + }) + .and_then(|u| u.vms.get(&metadata.name).copied()); + if let Some(usage) = usage { + println!("Used: {}", format_bytes_binary(usage.exclusive)); + if let Some(referenced) = usage.referenced { + println!("Referenced: {}", format_bytes_binary(referenced)); + } + if let Some(shared) = usage.shared() { + println!("Shared: {}", format_bytes_binary(shared)); + } + if let Some(ratio) = usage.compression_ratio() { + println!("Compression: {}", format_ratio(Some(ratio))); + } + } println!("Kernel: {}", metadata.kernel_path.display()); for (label, value) in CurrentPlatform::inspect_vm_extra(&metadata) { println!("{:<13}{}", label, value); diff --git a/src/main.rs b/src/main.rs index 42c6811..7c5020e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ pub mod image; use clap::Parser; use cli::kernel::KernelCommand; +use cli::storage::StorageCommand; use cli::vm::VmCommand; use cli::{Cli, Command}; @@ -31,11 +32,11 @@ fn needs_root(command: &Command) -> bool { command, Command::Version | Command::Info - | Command::Debug(_) | Command::Ssh(_) | Command::Exec(_) | Command::Cp(_) | Command::Vm(VmCommand::List(_) | VmCommand::Inspect(_)) + | Command::Storage(StorageCommand::Usage(_)) | Command::Kernel(KernelCommand::List) ) } @@ -50,12 +51,12 @@ fn needs_reconcile(command: &Command) -> bool { Command::Version | Command::Info | Command::Init(_) - | Command::Debug(_) | Command::Reconcile | Command::Ssh(_) | Command::Exec(_) | Command::Cp(_) | Command::Vm(VmCommand::List(_) | VmCommand::Inspect(_)) + | Command::Storage(StorageCommand::Usage(_)) | Command::Kernel(_) ) } @@ -82,7 +83,6 @@ fn main() -> anyhow::Result<()> { Command::Exec(args) => cli::exec::run(args, &cli.state_dir), Command::Cp(args) => cli::cp::run(args, &cli.state_dir), Command::Info => cli::info::run(&cli.state_dir), - Command::Debug(cmd) => cli::debug::run(cmd, &cli.state_dir), Command::Reconcile => { CurrentPlatform::reconcile(&cli.state_dir); Ok(()) diff --git a/tests/macos_storage.rs b/tests/macos_storage.rs index 29ee870..027d6f4 100644 --- a/tests/macos_storage.rs +++ b/tests/macos_storage.rs @@ -2,7 +2,7 @@ //! //! These tests verify macOS-specific storage behaviors: //! - APFS CoW clone space efficiency (clones don't consume extra space) -//! - Storage efficiency debug command +//! - Space accounting via `ember storage usage` //! - VM delete removes storage //! - Non-APFS (HFS+) detection and warnings //! @@ -59,10 +59,10 @@ fn apfs_clone_does_not_reduce_free_space() { ); } -/// `ember debug storage-efficiency` should report images and VMs. +/// `ember storage usage` should report images and VMs. #[test] #[ignore] -fn storage_efficiency_shows_savings() { +fn storage_usage_reports_images_and_vms() { let tmp = tempfile::tempdir().unwrap(); let state_dir = common::macos::setup_init(tmp.path()); let state = state_dir.to_str().unwrap(); @@ -91,23 +91,27 @@ fn storage_efficiency_shows_savings() { ); } - let output = common::ember(&["--state-dir", state, "debug", "storage-efficiency"]); + let output = common::ember(&["--state-dir", state, "storage", "usage"]); let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); assert!( output.status.success(), - "storage-efficiency failed.\nstdout: {stdout}\nstderr: {stderr}" + "storage usage failed.\nstdout: {stdout}\nstderr: {stderr}" ); - assert!( - stdout.contains("Images:"), - "expected 'Images:' in: {stdout}" - ); - assert!(stdout.contains("VMs:"), "expected 'VMs:' in: {stdout}"); - assert!( - stdout.contains("Total logical:"), - "expected 'Total logical:' in: {stdout}" - ); + assert!(stdout.contains("Pool"), "expected a pool line in: {stdout}"); + assert!(stdout.contains("IMAGES"), "expected images in: {stdout}"); + // The three VMs and their three forks. + for i in 0..3 { + assert!( + stdout.contains(&format!("effvm{i}")), + "expected effvm{i} in: {stdout}" + ); + assert!( + stdout.contains(&format!("efffork{i}")), + "expected efffork{i} in: {stdout}" + ); + } } /// VM delete should remove all storage (rootfs + VM directory). diff --git a/tests/storage_usage.rs b/tests/storage_usage.rs new file mode 100644 index 0000000..949e8df --- /dev/null +++ b/tests/storage_usage.rs @@ -0,0 +1,344 @@ +//! Integration tests for `ember storage usage` and the best-effort +//! usage columns on `vm list`. +//! +//! Every test here is `#[ignore]`d, matching the rest of `tests/`: +//! `TestEnv` builds a real backend (a loopback ZFS pool on Linux, an +//! APFS temp dir on macOS) and needs root on Linux. `run-integration-tests.sh` +//! passes `--ignored`, so these run there and stay out of `cargo test`. +//! +//! ```text +//! sudo cargo test --test storage_usage -- --ignored --test-threads=1 +//! ``` + +#[allow(dead_code)] +mod common; + +use common::{ember, TestEnv}; + +/// Parse the JSON output of `ember storage usage`. +fn usage_json(state_dir: &str) -> serde_json::Value { + let output = ember(&[ + "--state-dir", + state_dir, + "storage", + "usage", + "--format", + "json", + ]); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "storage usage failed.\nstdout: {stdout}\nstderr: {stderr}" + ); + serde_json::from_str(&stdout).unwrap_or_else(|e| panic!("invalid JSON: {e}\n{stdout}")) +} + +fn as_u64(value: &serde_json::Value) -> u64 { + value + .as_u64() + .unwrap_or_else(|| panic!("not a u64: {value}")) +} + +/// A freshly created VM occupies real space, and never reports holding +/// more than it references. +#[test] +#[ignore = "requires root + a real storage backend"] +fn usage_reports_vm_occupancy() { + let env = TestEnv::with_vm("usage_vm", "usage-vm"); + let usage = usage_json(env.state()); + + let vm = &usage["vms"]["usage-vm"]; + assert!(!vm.is_null(), "VM missing from usage report: {usage:#}"); + + let exclusive = as_u64(&vm["exclusive"]); + assert!(exclusive > 0, "a created VM occupies blocks: {vm:#}"); + assert!(as_u64(&vm["provisioned"]) > 0, "no virtual size: {vm:#}"); + + // `referenced` is optional: dm-thin and ZFS report it, APFS does not. + if let Some(referenced) = vm["referenced"].as_u64() { + assert!( + exclusive <= referenced, + "exclusive ({exclusive}) must stay within referenced ({referenced})" + ); + } + + // The pool has to account for at least what this VM occupies. + let allocated = as_u64(&usage["pool"]["allocated"]); + assert!( + allocated >= exclusive, + "pool allocated ({allocated}) is smaller than one VM's exclusive ({exclusive})" + ); +} + +/// Images are reported alongside VMs, and hold no more than they +/// reference despite carrying a refreservation on ZFS. +#[test] +#[ignore = "requires root + a real storage backend"] +fn usage_reports_images() { + let env = TestEnv::with_vm("usage_images", "usage-img-vm"); + let usage = usage_json(env.state()); + + let images = usage["images"] + .as_object() + .unwrap_or_else(|| panic!("images is not an object: {usage:#}")); + assert_eq!(images.len(), 1, "expected the one pulled image: {usage:#}"); + + let image = images.values().next().unwrap(); + assert!(as_u64(&image["provisioned"]) > 0); + if let Some(referenced) = image["referenced"].as_u64() { + assert!( + as_u64(&image["exclusive"]) <= referenced, + "image exclusive exceeds referenced: {image:#}" + ); + } +} + +/// A fork shares blocks with its origin, which is the whole point of +/// the CoW backends. Every backend measures this, so nothing skips +/// here. It used to: APFS reported `referenced` as null and this test +/// returned early on macOS, which left the one assertion aimed at +/// sharing unrun on the backend whose sharing was broken. +#[test] +#[ignore = "requires root + a real storage backend"] +fn fork_shares_blocks_with_origin() { + let env = TestEnv::with_vm("usage_fork", "usage-src"); + + // `--no-start` because `TestEnv::with_vm` installs a dummy kernel + // that cannot boot, and fork starts the copy by default. + let output = ember(&[ + "--state-dir", + env.state(), + "vm", + "fork", + "usage-src", + "usage-fork", + "--no-start", + ]); + assert!( + output.status.success(), + "fork failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let usage = usage_json(env.state()); + let fork = &usage["vms"]["usage-fork"]; + assert!(!fork.is_null(), "fork missing from usage report: {usage:#}"); + + let Some(referenced) = fork["referenced"].as_u64() else { + return; // Backend cannot separate shared from exclusive. + }; + let exclusive = as_u64(&fork["exclusive"]); + assert!( + referenced > exclusive, + "a fresh fork should share blocks with its origin, \ + but referenced ({referenced}) <= exclusive ({exclusive})" + ); +} + +/// The pool figure counts a shared block once, no matter how many +/// volumes map it. +/// +/// This is the regression test for the APFS backend reporting +/// `st_blocks` as occupancy. `st_blocks` counts the blocks a file maps +/// rather than the ones it owns, so summing it charged every shared +/// block once per clone and the pool figure grew each time a free clone +/// was made. With one image, two VMs and three forks it overstated real +/// occupancy by 2.6x. +/// +/// macOS-only, deliberately. The same property should hold on ZFS and +/// dm-thin, but `PoolUsage::allocated` on ZFS also carries +/// refreservation and snapshot charges, so the honest form of this +/// assertion there is a different one. Writing it here without a pool +/// to run it against is how the bug it guards got in. +#[cfg(target_os = "macos")] +#[test] +#[ignore = "requires root + a real storage backend"] +fn pool_counts_shared_blocks_once() { + let env = TestEnv::with_vm("usage_shared", "usage-shared"); + + let output = ember(&[ + "--state-dir", + env.state(), + "vm", + "fork", + "usage-shared", + "usage-shared-fork", + "--no-start", + ]); + assert!( + output.status.success(), + "fork failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let usage = usage_json(env.state()); + let volumes: Vec<&serde_json::Value> = usage["vms"] + .as_object() + .into_iter() + .chain(usage["images"].as_object()) + .flat_map(|m| m.values()) + .collect(); + + let referenced: u64 = volumes.iter().map(|v| as_u64(&v["referenced"])).sum(); + let allocated = as_u64(&usage["pool"]["allocated"]); + + // The fork shares nearly all of its origin, so counting each block + // once has to come out strictly below counting them per volume. + assert!( + allocated < referenced, + "pool allocated ({allocated}) must count shared blocks once, \ + but it is not below the sum of referenced ({referenced}): {usage:#}" + ); +} + +/// `vm list` gains a USED column and keeps working regardless. +#[test] +#[ignore = "requires root + a real storage backend"] +fn vm_list_shows_used_column() { + let env = TestEnv::with_vm("usage_list", "usage-list-vm"); + + let output = ember(&["--state-dir", env.state(), "vm", "list"]); + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("USED"), "no USED column:\n{stdout}"); + assert!(stdout.contains("usage-list-vm"), "VM missing:\n{stdout}"); +} + +/// Listing VMs must survive a backend that cannot answer, because one +/// common reason to list them is that storage is broken. +/// +/// Linux-only: the break is a `config.json` pointing at a pool that +/// does not exist, and `MacosStorage` reads neither `pool` nor +/// `storage_path` (it derives every path from `state_dir`), so the same +/// edit leaves the APFS backend perfectly able to measure. +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires root + a real storage backend"] +fn vm_list_survives_unmeasurable_storage() { + let env = TestEnv::with_vm("usage_broken", "usage-broken-vm"); + + let config_path = env.state_dir.join("config.json"); + let mut config: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&config_path).unwrap()).unwrap(); + config["pool"] = serde_json::Value::String("ember-no-such-pool".to_string()); + config["storage_path"] = serde_json::Value::String("/nonexistent/ember-usage".to_string()); + std::fs::write(&config_path, serde_json::to_string_pretty(&config).unwrap()).unwrap(); + + let output = ember(&["--state-dir", env.state(), "vm", "list"]); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "vm list must not fail when usage is unavailable: {}", + String::from_utf8_lossy(&output.stderr) + ); + + // USED is the last column, so the VM's row ends in the dash. + let row = stdout + .lines() + .find(|l| l.starts_with("usage-broken-vm")) + .unwrap_or_else(|| panic!("VM missing from listing:\n{stdout}")); + assert!( + row.ends_with(" -"), + "expected an unmeasurable USED column, got: {row:?}" + ); +} + +// --------------------------------------------------------------------------- +// dm-thin +// --------------------------------------------------------------------------- + +/// The dm-thin per-volume path: thin ids on the records have to join +/// against `thin_ls` rows read through a metadata snapshot, and that +/// snapshot has to be released again. A leaked one would make the +/// second call fail with EBUSY and would pin metadata blocks. +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires root + dm-thin kernel module"] +fn dm_thin_usage_measures_volumes_and_releases_snapshot() { + let tmp = tempfile::tempdir().unwrap(); + let storage_path = tmp.path().join("dm-thin"); + let state_dir = tmp.path().join("state"); + let state = state_dir.to_str().unwrap().to_string(); + + let _cleanup = common::linux::DmThinCleanup { + state_dir: state_dir.clone(), + }; + + let output = ember(&[ + "--state-dir", + &state, + "init", + "--storage", + "dm-thin", + "--storage-path", + storage_path.to_str().unwrap(), + "--size", + "2G", + "--instance-id", + "beef", + ]); + assert!( + output.status.success(), + "dm-thin init failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + // An empty pool still reports capacity and metadata. + let usage = usage_json(&state); + assert!(as_u64(&usage["pool"]["capacity"]) > 0); + assert!(as_u64(&usage["pool"]["metadata"]["capacity"]) > 0); + assert!( + usage["pool"]["logical"].is_null(), + "dm-thin does not compress" + ); + + // Now give it something to measure. Without a VM the maps stay + // empty and the thin-id join is never exercised. + common::require_docker(); + let output = ember(&["--state-dir", &state, "image", "pull", "alpine:latest"]); + assert!( + output.status.success(), + "image pull failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let kernel = tmp.path().join("vmlinux-dummy"); + std::fs::write(&kernel, b"not a real kernel").unwrap(); + let output = ember(&[ + "--state-dir", + &state, + "vm", + "create", + "thinvm", + "--image", + "alpine:latest", + "--kernel", + kernel.to_str().unwrap(), + "--no-start", + ]); + assert!( + output.status.success(), + "vm create failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let usage = usage_json(&state); + let vm = &usage["vms"]["thinvm"]; + assert!(!vm.is_null(), "thin id join produced nothing: {usage:#}"); + let exclusive = as_u64(&vm["exclusive"]); + let referenced = as_u64(&vm["referenced"]); + assert!( + exclusive <= referenced, + "exclusive ({exclusive}) exceeds referenced ({referenced})" + ); + // A fresh clone shares nearly everything with the image base. + assert!(referenced > 0, "clone references no blocks: {vm:#}"); + assert!( + !usage["images"].as_object().unwrap().is_empty(), + "image missing from usage report: {usage:#}" + ); + + // A third call can only succeed if the previous two released their + // metadata snapshots. + let _ = usage_json(&state); +}