From dad46daa6a910268bb53c3b30bd8126971727171 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 00:09:34 +0000 Subject: [PATCH 01/20] fix(atomic): retry an interrupted ACL read and free its scratch fgetxattr was not retried on EINTR, so a signal refused the replacement. atomicWrite also leaked the temporary path, the step names and the spare ACL read. Tests cover both under std.testing.allocator. --- CHANGELOG.md | 14 +-- changelog.d/+atomic-eintr.bugfix.md | 1 + changelog.d/+atomic-leak.bugfix.md | 1 + changelog.d/+rootlock-api.bugfix.md | 1 + changelog.d/+rootlock-truncate.bugfix.md | 1 + src/atomic.zig | 108 +++++++++++++++++++++-- towncrier.toml | 37 ++++++++ 7 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 changelog.d/+atomic-eintr.bugfix.md create mode 100644 changelog.d/+atomic-leak.bugfix.md create mode 100644 changelog.d/+rootlock-api.bugfix.md create mode 100644 changelog.d/+rootlock-truncate.bugfix.md create mode 100644 towncrier.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b28c1..9055909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,6 @@ # Changelog -## Unreleased - -* `state.RootLock.acquire` no longer truncates the file behind a hard link - planted at `.lock`. It opens the state root `O_DIRECTORY|O_NOFOLLOW` and - both lock files relative to it with `O_NOFOLLOW`, creating them mode - 0600; a lock file name that is not a regular file with one link is - refused before it is opened, so a FIFO or symbolic link there is never - followed; and once both locks are held, the gate must still be a regular - file whose one link is `.lock` before it is truncated. -* A refused gate is the new `RootLock.AcquireError.GateNotPrivate` - (`NotPrivate` stays the sidecar's), a symbolic link as the state root is - `NotDir`, and `LockOps.open_sidecar` takes the state root's descriptor - and a name. + ## 0.8.1 diff --git a/changelog.d/+atomic-eintr.bugfix.md b/changelog.d/+atomic-eintr.bugfix.md new file mode 100644 index 0000000..075b0b9 --- /dev/null +++ b/changelog.d/+atomic-eintr.bugfix.md @@ -0,0 +1 @@ +`atomic` retries an ACL read interrupted by a signal, instead of refusing the replacement. diff --git a/changelog.d/+atomic-leak.bugfix.md b/changelog.d/+atomic-leak.bugfix.md new file mode 100644 index 0000000..2640931 --- /dev/null +++ b/changelog.d/+atomic-leak.bugfix.md @@ -0,0 +1 @@ +`atomicWrite` frees the path, the step names and the spare ACL read it allocates. diff --git a/changelog.d/+rootlock-api.bugfix.md b/changelog.d/+rootlock-api.bugfix.md new file mode 100644 index 0000000..59e0f64 --- /dev/null +++ b/changelog.d/+rootlock-api.bugfix.md @@ -0,0 +1 @@ +A refused gate is the new `RootLock.AcquireError.GateNotPrivate` (`NotPrivate` stays the sidecar's), a symbolic link as the state root is `NotDir`, and `LockOps.open_sidecar` takes the state root's descriptor and a name. diff --git a/changelog.d/+rootlock-truncate.bugfix.md b/changelog.d/+rootlock-truncate.bugfix.md new file mode 100644 index 0000000..72d1dd9 --- /dev/null +++ b/changelog.d/+rootlock-truncate.bugfix.md @@ -0,0 +1 @@ +`state.RootLock.acquire` no longer truncates the file behind a hard link planted at `.lock`. It opens the state root `O_DIRECTORY|O_NOFOLLOW` and both lock files relative to it with `O_NOFOLLOW`, creating them mode 0600; a lock file name that is not a regular file with one link is refused before it is opened, so a FIFO or symbolic link there is never followed; and once both locks are held, the gate must still be a regular file whose one link is `.lock` before it is truncated. diff --git a/src/atomic.zig b/src/atomic.zig index 49dc641..1d1f15e 100644 --- a/src/atomic.zig +++ b/src/atomic.zig @@ -94,11 +94,19 @@ pub const Why = struct { pub const Error = error{ Refused, OutOfMemory } || sys.Error; +const static_refused: []const u8 = "refused"; + fn refuse(a: Allocator, why: *Why, comptime f: []const u8, args: anytype) Error { - why.msg = a.print(f, args) catch "refused"; + why.msg = a.print(f, args) catch static_refused; return error.Refused; } +/// Free a message `refuse` stored, including its out-of-memory fallback. +fn freeMsg(a: Allocator, msg: []const u8) void { + if (msg.len == 0 or msg.ptr == static_refused.ptr) return; + a.free(msg); +} + /// The value of each ACL attribute present, keyed by its name. pub const Acls = struct { present: [acl_attributes.len]?[]const u8 = .{ null, null, null }, @@ -159,12 +167,27 @@ fn errnoText(e: linux.E) []const u8 { }; } +fn freeAcls(a: Allocator, acls: Acls) void { + for (acls.present) |p| if (p) |s| a.free(s); +} + +/// One `fgetxattr`, retrying when a signal interrupts it. `op` cannot be +/// used here: the errno itself decides whether a missing attribute is an +/// error, and `op` collapses errnos into `sys.Error`. +fn xattrGet(fd: sys.fd_t, name: [:0]const u8, buf: []u8) usize { + while (true) { + const rc = ops.fgetxattr(fd, name, buf); + if (linux.errno(rc) != .INTR) return rc; + } +} + fn readAllAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { const listed = try listedAcls(a, fd, why); var acls: Acls = .{}; + errdefer freeAcls(a, acls); var buf: [64 * 1024]u8 = undefined; for (acl_attributes, 0..) |name, i| { - const rc = ops.fgetxattr(fd, name, &buf); + const rc = xattrGet(fd, name, &buf); const e = linux.errno(rc); if (e == .SUCCESS) { acls.present[i] = try a.dupe(u8, buf[0..rc]); @@ -189,7 +212,9 @@ fn readAllAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { pub fn readAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { const before = try fsx.fstat(fd); const acls = try readAllAcls(a, fd, why); + errdefer freeAcls(a, acls); const again = try readAllAcls(a, fd, why); + defer freeAcls(a, again); if (!acls.eql(again) or !sameIdentity(try fsx.fstat(fd), before)) return refuse(a, why, "access metadata changed during inspection", .{}); return acls; @@ -225,13 +250,16 @@ fn restoreMetadata(a: Allocator, fd: sys.fd_t, metadata: ?Metadata, mode: u32, w break; }; if (primary) |i| { - if (!(try readAcls(a, fd, why)).eql(m.acls)) { + const now = try readAcls(a, fd, why); + defer freeAcls(a, now); + if (!now.eql(m.acls)) { const v = m.acls.present[i].?; _ = try op(ops.fsetxattr, .{ fd, acl_attributes[i], v }); } } for (acl_attributes) |_| { const present = try readAcls(a, fd, why); + defer freeAcls(a, present); var extra: ?usize = null; for (present.present, m.acls.present, 0..) |p, want, i| if (p != null and want == null) { extra = i; @@ -243,7 +271,9 @@ fn restoreMetadata(a: Allocator, fd: sys.fd_t, metadata: ?Metadata, mode: u32, w const restored = try fsx.fstat(fd); if (restored.uid != m.info.uid or restored.gid != m.info.gid or restored.perm() != mode) return refuse(a, why, "ownership or mode did not round-trip: expected {d}:{d} {o}, found {d}:{d} {o}", .{ m.info.uid, m.info.gid, mode, restored.uid, restored.gid, restored.perm() }); - if (!(try readAcls(a, fd, why)).eql(m.acls)) return refuse(a, why, "access-control attributes did not round-trip", .{}); + const round = try readAcls(a, fd, why); + defer freeAcls(a, round); + if (!round.eql(m.acls)) return refuse(a, why, "access-control attributes did not round-trip", .{}); } /// Called before each named step; a test hook can stop the program there. @@ -359,12 +389,17 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write } else |_| {} const metadata_path: ?[]const u8 = opts.metadata_from orelse (if (exists(path)) path else null); var metadata: ?Metadata = null; + defer if (metadata) |m| freeAcls(a, m.acls); if (metadata_path) |mp| { var inner: Why = .{}; - metadata = readMetadata(a, mp, &inner) catch |err| return refuse(a, why, "cannot read access metadata for {s}: {s}", .{ mp, if (err == error.Refused) inner.msg else describe(err) }); + metadata = readMetadata(a, mp, &inner) catch |err| { + defer freeMsg(a, inner.msg); + return refuse(a, why, "cannot read access metadata for {s}: {s}", .{ mp, if (err == error.Refused) inner.msg else describe(err) }); + }; } const mode: u32 = opts.mode orelse (if (metadata) |m| m.info.perm() else fileMode(path)); const tmp = try a.print("{s}/.{s}.new", .{ parent, std.fs.path.basename(path) }); + defer a.free(tmp); if (fsx.statAt(fsx.cwd, tmp, false)) |stale| { // A privileged writer hands its temporary file to the metadata owner // before a crash can interrupt it. @@ -377,6 +412,7 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write { const step = try a.print("{s}.write", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } const fd = try sys.openat(fsx.cwd, tmp, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, .NOFOLLOW = true }, 0o600); @@ -392,20 +428,29 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write defer sys.close(fd); try sys.writeAll(fd, data); var inner: Why = .{}; - restoreMetadata(a, fd, metadata, mode, &inner) catch |err| return refuse(a, why, "cannot preserve access metadata for {s}: {s}", .{ path, if (err == error.Refused) inner.msg else describe(err) }); + restoreMetadata(a, fd, metadata, mode, &inner) catch |err| { + defer freeMsg(a, inner.msg); + return refuse(a, why, "cannot preserve access metadata for {s}: {s}", .{ path, if (err == error.Refused) inner.msg else describe(err) }); + }; { const step = try a.print("{s}.fsync", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } _ = try op(ops.fsync, .{fd}); } if (metadata) |m| { var inner: Why = .{}; - const current = readMetadata(a, metadata_path.?, &inner) catch |err| return refuse(a, why, "cannot recheck access metadata for {s}: {s}", .{ metadata_path.?, if (err == error.Refused) inner.msg else describe(err) }); + const current = readMetadata(a, metadata_path.?, &inner) catch |err| { + defer freeMsg(a, inner.msg); + return refuse(a, why, "cannot recheck access metadata for {s}: {s}", .{ metadata_path.?, if (err == error.Refused) inner.msg else describe(err) }); + }; + defer freeAcls(a, current.acls); if (!sameMetadata(current, m)) return refuse(a, why, "access metadata of {s} changed before {s} was replaced", .{ metadata_path.?, path }); } { const step = try a.print("{s}.rename", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } try sys.renameat(fsx.cwd, tmp, fsx.cwd, path); @@ -474,6 +519,55 @@ test "a step a test makes fail leaves the destination and no temporary file" { } } +test "atomicWrite frees what it allocates" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const dir = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/atomic-noleak-{d}", .{ cwd, linux.getpid() }); + defer gpa.free(dir); + try makedirs(dir); + defer fsx.deleteTree(gpa, fsx.cwd, dir) catch {}; + const path = try std.fmt.allocPrint(gpa, "{s}/j.json", .{dir}); + defer gpa.free(path); + var why: Why = .{}; + try atomicWrite(gpa, path, "one\n", .{}, &why); + try atomicWrite(gpa, path, "two\n", .{}, &why); + try std.testing.expectEqual(@as(usize, 0), why.msg.len); +} + +test "an interrupted ACL read is retried" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const dir = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/atomic-eintr-{d}", .{ cwd, linux.getpid() }); + defer gpa.free(dir); + try makedirs(dir); + defer fsx.deleteTree(gpa, fsx.cwd, dir) catch {}; + const path = try std.fmt.allocPrint(gpa, "{s}/f", .{dir}); + defer gpa.free(path); + const created = try sys.openat(fsx.cwd, path, .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o600); + sys.close(created); + const fd = try sys.openat(fsx.cwd, path, .{}, 0); + defer sys.close(fd); + const old = ops.*; + defer ops.* = old; + const S = struct { + var left: u8 = 0; + fn get(fd_: sys.fd_t, name: [:0]const u8, buf: []u8) usize { + if (left > 0) { + left -= 1; + return @bitCast(-@as(isize, @backingInt(linux.E.INTR))); + } + return linux.fgetxattr(fd_, name.ptr, buf.ptr, buf.len); + } + }; + S.left = 2; + ops.fgetxattr = S.get; + var why: Why = .{}; + const acls = try readAcls(gpa, fd, &why); + defer freeAcls(gpa, acls); +} + test "the umask is read without changing it" { const old: u32 = @intCast(linux.syscall1(.umask, 0o027)); defer _ = linux.syscall1(.umask, old); diff --git a/towncrier.toml b/towncrier.toml new file mode 100644 index 0000000..19a34e1 --- /dev/null +++ b/towncrier.toml @@ -0,0 +1,37 @@ +[tool.towncrier] +name = "zig-common" +directory = "changelog.d" +filename = "CHANGELOG.md" +start_string = "\n" +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/stackhands/zig-common/pull/{issue})" +underlines = ["", "", ""] +# Setting ignore, even to nothing, is what makes `towncrier build` fail a +# fragment whose name has no configured type. Unset, that file is skipped +# and the draft still passes. .gitkeep stays ignored either way. +ignore = [] + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Fixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Miscellaneous" +showcontent = true From 6901bf9c49b3de83346ca1e686cdfd587682d487 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 00:09:34 +0000 Subject: [PATCH 02/20] fix(fsx): a short write does not destroy the destination copyTo renames the new bytes into place, so a full disk or a file-size limit leaves an existing file unchanged. appendLine rolls a short write back to the size before it, so the line list does not gain a partial line. --- changelog.d/+append-short-write.bugfix.md | 1 + changelog.d/+copy-short-write.bugfix.md | 1 + src/fsx.zig | 160 ++++++++++++++++++++-- 3 files changed, 149 insertions(+), 13 deletions(-) create mode 100644 changelog.d/+append-short-write.bugfix.md create mode 100644 changelog.d/+copy-short-write.bugfix.md diff --git a/changelog.d/+append-short-write.bugfix.md b/changelog.d/+append-short-write.bugfix.md new file mode 100644 index 0000000..d50a541 --- /dev/null +++ b/changelog.d/+append-short-write.bugfix.md @@ -0,0 +1 @@ +`fsx.appendLine` rolls a short write back, so a full disk does not leave a partial line. diff --git a/changelog.d/+copy-short-write.bugfix.md b/changelog.d/+copy-short-write.bugfix.md new file mode 100644 index 0000000..da6671e --- /dev/null +++ b/changelog.d/+copy-short-write.bugfix.md @@ -0,0 +1 @@ +`fsx.copyTo` renames the new bytes into place, so a short write leaves the existing file unchanged. diff --git a/src/fsx.zig b/src/fsx.zig index e473fc0..de2aab6 100644 --- a/src/fsx.zig +++ b/src/fsx.zig @@ -128,22 +128,55 @@ fn copyFd(src: fd_t, dst: fd_t) Error!void { } } -/// Copy an open regular file to `dir/name`, as cp(1) without options does: -/// an existing destination is truncated and keeps its mode, a new one gets -/// the source's permission bits less the umask. A destination that is a -/// symbolic link or not a regular file is refused. Returns the open -/// destination, which the caller closes. +/// Copy an open regular file to `dir/name`. The bytes are written to a +/// temporary file in `dir`, synced, and renamed over `name`, so a short +/// write (a full disk, a file-size limit) leaves an existing destination +/// as it was and does not publish a partial file under `name`. An existing +/// regular file keeps its owner, when this process may set it, and its +/// mode; a new file gets the source's permission bits less the umask. A +/// destination that is a symbolic link or not a regular file is refused. +/// A destination with other hard links keeps those links on the previous +/// inode; `name` is the new inode. Returns the open destination, which the +/// caller closes. pub fn copyTo(src: fd_t, src_stat: Stat, dir: fd_t, name: []const u8) Error!fd_t { - const fd = try sys.openat(dir, name, .{ + const existing: ?Stat = statAt(dir, name, false) catch |err| switch (err) { + error.NotFound => null, + else => |e| return e, + }; + if (existing) |st| { + if (st.isLink()) return error.SymLink; + if (st.isDir()) return error.IsDir; + if (!st.isReg()) return error.InvalidArgument; + } + var rnd: [4]u8 = undefined; + _ = try sys.call(linux.getrandom, .{ &rnd, rnd.len, 0 }); + var tmp_buf: [sys.path_max]u8 = undefined; + const tmp = std.fmt.bufPrint(&tmp_buf, ".{s}.{d}.{x:0>8}.tmp", .{ std.fs.path.basename(name), linux.getpid(), std.mem.readInt(u32, &rnd, .little) }) catch return error.NameTooLong; + const fd = try sys.openat(dir, tmp, .{ .ACCMODE = .WRONLY, .CREAT = true, + .EXCL = true, .NOFOLLOW = true, - .NONBLOCK = true, - }, @intCast(src_stat.mode & 0o777)); - errdefer sys.close(fd); - if (!(try fstat(fd)).isReg()) return error.InvalidArgument; - _ = try sys.call(linux.ftruncate, .{ fd, 0 }); + }, if (existing != null) 0o600 else @intCast(src_stat.mode & 0o777)); + var done = false; + errdefer { + sys.close(fd); + if (!done) sys.unlinkat(dir, tmp, 0) catch {}; + } try copyFd(src, fd); + if (existing) |st| { + if ((try fstat(fd)).uid != st.uid or (try fstat(fd)).gid != st.gid) { + sys.fchown(fd, st.uid, st.gid) catch |err| switch (err) { + error.AccessDenied => {}, + else => |e| return e, + }; + } + try sys.fchmod(fd, @intCast(st.perm())); + } + try sys.fsync(fd); + try sys.renameat(dir, tmp, dir, name); + done = true; + try sys.fsync(dir); return fd; } @@ -159,10 +192,22 @@ pub fn appendLine(gpa: Allocator, dir: fd_t, name: []const u8, line: []const u8) .NONBLOCK = true, }, 0o666); defer sys.close(fd); - if (!(try fstat(fd)).isReg()) return error.InvalidArgument; + const st = try fstat(fd); + if (!st.isReg()) return error.InvalidArgument; const bytes = try std.mem.concat(gpa, u8, &.{ line, "\n" }); defer gpa.free(bytes); - try sys.writeAll(fd, bytes); + // One write is atomic on Linux only when it completes. A short write + // (ENOSPC part way through) is rolled back to the size before it, so + // the list does not gain a partial line. Safe while this process is + // the only writer; a concurrent append in that window can be shortened. + sys.writeAll(fd, bytes) catch |err| { + if (fstat(fd)) |now| { + if (now.size > st.size and now.size - st.size < bytes.len) { + _ = linux.ftruncate(fd, @intCast(st.size)); + } + } else |_| {} + return err; + }; } /// Read a whole regular file opened by the caller; more than `limit` bytes @@ -355,3 +400,92 @@ test "replaceContents keeps the mode" { defer gpa.free(got); try std.testing.expectEqualStrings("b\n", got); } + +fn withFileLimit(bytes: u64, body: *const fn () anyerror!void) anyerror!void { + var old: linux.rlimit = undefined; + if (linux.errno(linux.prlimit(0, .FSIZE, null, &old)) != .SUCCESS) return error.Unexpected; + const lim: linux.rlimit = .{ .cur = @intCast(bytes), .max = old.max }; + if (linux.errno(linux.prlimit(0, .FSIZE, &lim, null)) != .SUCCESS) return error.Unexpected; + const ign: linux.Sigaction = .{ .handler = .{ .handler = linux.SIG.IGN }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.XFSZ, &ign, &prev); + defer { + _ = linux.sigaction(.XFSZ, &prev, null); + _ = linux.prlimit(0, .FSIZE, &old, null); + } + try body(); +} + +test "copyTo leaves an existing file unchanged when the write cannot finish" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try realpath(cwd, ".", &cwd_buf); + var pb: [sys.path_max]u8 = undefined; + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-copy-{d}", .{ here, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try openDir(cwd, base, false); + defer { + deleteTree(gpa, cwd, base) catch {}; + sys.close(d); + } + const dst = try sys.openat(d, "dest", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o640); + try sys.writeAll(dst, "OLDDATA\n"); + sys.close(dst); + const srcfd = try sys.openat(d, "src", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + try sys.writeAll(srcfd, "NEWDATA-IS-LONGER-THAN-THE-LIMIT\n"); + sys.close(srcfd); + const opened = try openRegular(d, "src", false); + defer sys.close(opened.fd); + + const Hold = struct { + var dir: fd_t = -1; + var src: fd_t = -1; + var st: Stat = undefined; + fn go() Error!void { + if (copyTo(src, st, dir, "dest")) |fd| sys.close(fd) else |_| {} + } + }; + Hold.dir = d; + Hold.src = opened.fd; + Hold.st = opened.stat; + try withFileLimit(8, Hold.go); + + const got = try openRegular(d, "dest", false); + defer sys.close(got.fd); + const text = try readAll(gpa, got.fd, 100); + defer gpa.free(text); + try std.testing.expectEqualStrings("OLDDATA\n", text); + try std.testing.expectEqual(@as(u32, 0o640), got.stat.perm()); +} + +test "appendLine does not leave a partial line when the write cannot finish" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try realpath(cwd, ".", &cwd_buf); + var pb: [sys.path_max]u8 = undefined; + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-append-{d}", .{ here, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try openDir(cwd, base, false); + defer { + deleteTree(gpa, cwd, base) catch {}; + sys.close(d); + } + const f = try sys.openat(d, "list", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + try sys.writeAll(f, "hi\n"); + sys.close(f); + const Hold = struct { + var dir: fd_t = -1; + fn go() Error!void { + appendLine(std.testing.allocator, dir, "list", "this line is far longer than the file size limit") catch {}; + } + }; + Hold.dir = d; + try withFileLimit(8, Hold.go); + const got = try openRegular(d, "list", false); + defer sys.close(got.fd); + const text = try readAll(gpa, got.fd, 200); + defer gpa.free(text); + try std.testing.expectEqualStrings("hi\n", text); +} From 7fc702590e72776862296cdbd74200717df4bc61 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 00:09:34 +0000 Subject: [PATCH 03/20] fix(state): a lock waits out signals and refuses a foreign gate lockExclusive reads its deadline from the monotonic clock, so a signal that cuts a sleep short does not expire the wait. RootLock refuses a gate or sidecar that is not a regular file of this user with one link. --- changelog.d/+lock-gate.bugfix.md | 1 + changelog.d/+lock-timeout.bugfix.md | 1 + src/state.zig | 104 ++++++++++++++++++++++------ 3 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 changelog.d/+lock-gate.bugfix.md create mode 100644 changelog.d/+lock-timeout.bugfix.md diff --git a/changelog.d/+lock-gate.bugfix.md b/changelog.d/+lock-gate.bugfix.md new file mode 100644 index 0000000..f387f65 --- /dev/null +++ b/changelog.d/+lock-gate.bugfix.md @@ -0,0 +1 @@ +`state.RootLock` also refuses a gate or sidecar that another user owns, as `GateNotPrivate` and `NotPrivate`: each must be a regular file of this user with one link. diff --git a/changelog.d/+lock-timeout.bugfix.md b/changelog.d/+lock-timeout.bugfix.md new file mode 100644 index 0000000..8c5eb11 --- /dev/null +++ b/changelog.d/+lock-timeout.bugfix.md @@ -0,0 +1 @@ +`state.lockExclusive` measures its deadline on the monotonic clock, so a signal that cuts a sleep short does not expire the wait. diff --git a/src/state.zig b/src/state.zig index d175b1e..b095941 100644 --- a/src/state.zig +++ b/src/state.zig @@ -43,12 +43,15 @@ pub fn openPrivate(dir: fsx.fd_t, name: []const u8, create: bool) Error!?fsx.fd_ pub const LockError = Error || error{TimedOut}; /// flock(LOCK_EX) on `fd`, giving up after `timeout_ms` when one is given. +/// The deadline is read from the monotonic clock. A signal that cuts a +/// sleep short does not consume the whole wait: the next check looks at +/// the clock again. pub fn lockExclusive(fd: fsx.fd_t, timeout_ms: ?u32) LockError!void { - const ms = timeout_ms orelse { + const deadline: ?i64 = if (timeout_ms) |ms| nowMs() + ms else null; + if (deadline == null) { _ = try sys.call(linux.flock, .{ fd, std.posix.LOCK.EX }); return; - }; - var left: i64 = ms; + } while (true) { const rc = linux.flock(fd, std.posix.LOCK.EX | std.posix.LOCK.NB); switch (linux.errno(rc)) { @@ -56,11 +59,10 @@ pub fn lockExclusive(fd: fsx.fd_t, timeout_ms: ?u32) LockError!void { .AGAIN, .INTR => {}, else => return error.Unexpected, } + const left = deadline.? - nowMs(); if (left <= 0) return error.TimedOut; - const step: i64 = @min(left, 20); - var ts: linux.timespec = .{ .sec = 0, .nsec = @intCast(step * std.time.ns_per_ms) }; + var ts: linux.timespec = .{ .sec = 0, .nsec = @intCast(@min(left, 20) * std.time.ns_per_ms) }; _ = linux.nanosleep(&ts, null); - left -= step; } } @@ -124,6 +126,60 @@ test "a file that is not private is not read" { try lockExclusive(lk, 100); } +fn onUsr1(_: linux.SIG) callconv(.c) void {} + +test "lockExclusive waits out the timeout while signals arrive" { + const gpa = std.testing.allocator; + var pb: [sys.path_max]u8 = undefined; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-locktime-{d}", .{ cwd, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(fsx.cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try fsx.openDir(fsx.cwd, base, false); + defer { + fsx.deleteTree(gpa, fsx.cwd, base) catch {}; + sys.close(d); + } + const held = try sys.openat(d, "L", .{ .ACCMODE = .RDWR, .CREAT = true }, 0o600); + defer sys.close(held); + const waiter = try sys.openat(d, "L", .{ .ACCMODE = .RDWR }, 0); + defer sys.close(waiter); + _ = try sys.call(linux.flock, .{ held, std.posix.LOCK.EX }); + defer _ = linux.flock(held, std.posix.LOCK.UN); + + const handler: linux.Sigaction = .{ .handler = .{ .handler = &onUsr1 }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.USR1, &handler, &prev); + defer _ = linux.sigaction(.USR1, &prev, null); + const Stop = struct { + var go = std.atomic.Value(bool).init(true); + fn buzz() void { + // This thread must not catch SIGUSR1, or the waiter's sleep is + // never the one the signal interrupts. + var blocked = linux.sigemptyset(); + linux.sigaddset(&blocked, .USR1); + _ = linux.sigprocmask(linux.SIG.BLOCK, &blocked, null); + while (go.load(.acquire)) { + _ = linux.kill(linux.getpid(), .USR1); + var ts: linux.timespec = .{ .sec = 0, .nsec = std.time.ns_per_ms }; + _ = linux.nanosleep(&ts, null); + } + } + }; + Stop.go.store(true, .release); + const buzz = try std.Thread.spawn(.{}, Stop.buzz, .{}); + var t0: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &t0); + try std.testing.expectError(error.TimedOut, lockExclusive(waiter, 400)); + var t1: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &t1); + Stop.go.store(false, .release); + buzz.join(); + const ms = (t1.sec - t0.sec) * 1000 + @divTrunc(t1.nsec - t0.nsec, 1_000_000); + try std.testing.expect(ms >= 300 and ms < 5000); +} + // ---- the state-root lock: a flock gate and a POSIX record lock ------------- /// A state root's lock. The flock gate excludes processes on one kernel; the @@ -136,8 +192,9 @@ test "a file that is not private is not read" { /// Others may be able to write the state root. The root is opened without /// following a symbolic link as its last component, both files through its /// descriptor without following one at their names, and each is trusted -/// only as a regular file with one link: a FIFO planted at a name is never -/// opened, and the file behind a planted hard link is never truncated. +/// only as a regular file of this user with one link: a FIFO planted at a +/// name is never opened, and the file behind a planted hard link, or a file +/// another user owns, is never truncated. pub const RootLock = struct { gate: fsx.fd_t, posix: fsx.fd_t, @@ -148,9 +205,10 @@ pub const RootLock = struct { pub const AcquireError = Error || error{ Busy, - /// STATE_ROOT/.lock.posix is not a regular file with one link. + /// STATE_ROOT/.lock.posix is not a regular file of this user with + /// one link. NotPrivate, - /// STATE_ROOT/.lock is not a regular file with one link. + /// STATE_ROOT/.lock is not a regular file of this user with one link. GateNotPrivate, OutOfMemory, }; @@ -207,10 +265,10 @@ pub const RootLock = struct { /// STATE_ROOT is created when absent and opened O_DIRECTORY|O_NOFOLLOW; /// a symbolic link there is NotDir. Each lock file is opened relative to /// that descriptor with O_NOFOLLOW, created mode 0600, only when its name - /// is absent or a regular file with one link, and its descriptor must be - /// that file. The gate is checked again once both locks are held, right - /// before it is truncated. Allocates nothing; `gpa` stays for source - /// compatibility. + /// is absent or a regular file of this user with one link, and its + /// descriptor must be that file. The gate is checked again once both + /// locks are held, right before it is truncated. Allocates nothing; + /// `gpa` stays for source compatibility. pub fn acquire(gpa: Allocator, state_root: []const u8, timeout_ms: ?u32) AcquireError!RootLock { _ = gpa; const atomic = @import("atomic.zig"); @@ -234,12 +292,12 @@ pub const RootLock = struct { } /// `dir/name` opened read-write through `open`; null unless the name is - /// absent or a regular file with one link before the open, so a FIFO, - /// device or symbolic link there is never opened, and the descriptor is - /// that file after it. + /// absent or a regular file of this user with one link before the open, + /// so a FIFO, device or symbolic link there is never opened, and the + /// descriptor is that file after it. fn openLockFile(dir: fsx.fd_t, name: [:0]const u8, open: *const fn (dir: fsx.fd_t, name: [*:0]const u8) usize) Error!?fsx.fd_t { if (fsx.statAt(dir, name, false)) |st| { - if (!st.isReg() or st.nlink != 1) return null; + if (!st.isReg() or st.nlink != 1 or st.uid != linux.geteuid()) return null; } else |err| switch (err) { error.NotFound => {}, else => |e| return e, @@ -254,13 +312,13 @@ pub const RootLock = struct { return null; } - /// Whether `fd` is a regular file whose one link is `dir/name`. The name - /// is looked up as well as the descriptor: a planted hard link unlinked - /// while the lock is awaited leaves the descriptor on the other name's - /// file, with one link. + /// Whether `fd` is a regular file of this user whose one link is + /// `dir/name`. The name is looked up as well as the descriptor: a + /// planted hard link unlinked while the lock is awaited leaves the + /// descriptor on the other name's file, with one link. fn isSoleLink(dir: fsx.fd_t, name: []const u8, fd: fsx.fd_t) Error!bool { const st = try fsx.fstat(fd); - if (!st.isReg() or st.nlink != 1) return false; + if (!st.isReg() or st.nlink != 1 or st.uid != linux.geteuid()) return false; const at = fsx.statAt(dir, name, false) catch |err| switch (err) { error.NotFound => return false, else => |e| return e, From abe21fa4dc23a49be0e1f9522978c8fb2b0a3fd9 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 00:09:34 +0000 Subject: [PATCH 04/20] fix(json): a truncated escape is an encoding error decodeCache and key comparison read past a string that ended on a backslash. They now return an encoding error, and the bytes allocated before that error are freed. --- changelog.d/+json-escape.bugfix.md | 1 + src/json.zig | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 changelog.d/+json-escape.bugfix.md diff --git a/changelog.d/+json-escape.bugfix.md b/changelog.d/+json-escape.bugfix.md new file mode 100644 index 0000000..006d50b --- /dev/null +++ b/changelog.d/+json-escape.bugfix.md @@ -0,0 +1 @@ +`json` returns an encoding error for a truncated or unknown escape, instead of reading past the string. diff --git a/src/json.zig b/src/json.zig index 8c732f9..da7dcfc 100644 --- a/src/json.zig +++ b/src/json.zig @@ -41,8 +41,10 @@ fn keyEql(raw: []const u8, want: []const u8) bool { while (i < raw.len) { var c: u21 = raw[i]; if (c == '\\') { + if (i + 1 >= raw.len) return false; const e = raw[i + 1]; if (e == 'u') { + if (i + 6 > raw.len) return false; c = std.fmt.parseInt(u16, raw[i + 2 .. i + 6], 16) catch return false; i += 6; } else { @@ -225,6 +227,7 @@ pub const DecodeError = Allocator.Error || error{Encoding}; /// surrogate is returned as itself. fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { var out: std.ArrayList(u21) = .empty; + errdefer out.deinit(arena); var i: usize = 0; while (i < raw.len) { const c = raw[i]; @@ -235,6 +238,7 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { i += ch.len; continue; } + if (i + 1 >= raw.len) return error.Encoding; const e = raw[i + 1]; i += 2; const simple: ?u21 = switch (e) { @@ -246,12 +250,14 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { 'n' => '\n', 'r' => '\r', 't' => '\t', - else => null, + 'u' => null, + else => return error.Encoding, }; if (simple) |s| { try out.append(arena, s); continue; } + if (i + 4 > raw.len) return error.Encoding; var cp: u21 = std.fmt.parseInt(u16, raw[i .. i + 4], 16) catch return error.Encoding; i += 4; if (cp >= 0xD800 and cp <= 0xDBFF and i + 6 <= raw.len and raw[i] == '\\' and raw[i + 1] == 'u') { @@ -272,6 +278,7 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { pub fn decodeCache(arena: Allocator, raw: []const u8) DecodeError![]u8 { const cps = try codePoints(arena, raw); var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(arena); for (cps) |cp| { if (cp >= 0xDC80 and cp <= 0xDCFF) { try out.append(arena, @intCast(cp - 0xDC00)); @@ -350,6 +357,13 @@ test "cache strings round-trip bytes that are not UTF-8" { const v = try parse(a, out.items); try std.testing.expectEqualStrings("a\"\xff\u{e9}\u{1F600}\n\x7f", try decodeCache(a, v.string)); try std.testing.expectError(error.Encoding, decodeCache(a, "\\ud800")); + // A truncated escape is an encoding error. It must not panic, and the + // bytes allocated before the error are freed when the caller did not + // pass an arena. + try std.testing.expectError(error.Encoding, decodeCache(std.testing.allocator, "a\\")); + try std.testing.expectError(error.Encoding, decodeCache(std.testing.allocator, "\\u12")); + const broken: Value = .{ .object = &.{.{ .key = "\\", .value = .null }} }; + try std.testing.expect(broken.get("x") == null); } test "wire strings are one byte per code point" { From 9dd22ccf8a358b69a00145470c9102be2ffb8a46 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 00:09:34 +0000 Subject: [PATCH 05/20] fix(pytext): free pySplit's list when allocation fails The list was abandoned on the error path, and the slice it returned was not the whole allocation, so a caller could not free it. --- changelog.d/+pysplit-oom.bugfix.md | 1 + src/pytext.zig | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 changelog.d/+pysplit-oom.bugfix.md diff --git a/changelog.d/+pysplit-oom.bugfix.md b/changelog.d/+pysplit-oom.bugfix.md new file mode 100644 index 0000000..8a550c8 --- /dev/null +++ b/changelog.d/+pysplit-oom.bugfix.md @@ -0,0 +1 @@ +`pytext.pySplit` frees its list when an allocation fails, and the slice it returns is the whole allocation. diff --git a/src/pytext.zig b/src/pytext.zig index 6e3bc35..c3c28d9 100644 --- a/src/pytext.zig +++ b/src/pytext.zig @@ -118,6 +118,7 @@ pub fn pyStrip(s: []const u8) []const u8 { /// str.split() with no separator: the runs between Unicode whitespace. pub fn pySplit(arena: Allocator, s: []const u8) Allocator.Error![]const []const u8 { var out: std.ArrayList([]const u8) = .empty; + errdefer out.deinit(arena); var i: usize = 0; var start: ?usize = null; while (i < s.len) { @@ -130,7 +131,7 @@ pub fn pySplit(arena: Allocator, s: []const u8) Allocator.Error![]const []const i += ch.len; } if (start) |st| try out.append(arena, s[st..]); - return out.items; + return try out.toOwnedSlice(arena); } /// shlex.quote: unchanged when every byte is in [A-Za-z0-9_@%+=:,./-] and @@ -167,6 +168,16 @@ test "pyStrip and pySplit use Unicode whitespace" { try std.testing.expectEqualStrings("waitpkg", parts[1]); } +fn splitOwned(a: std.mem.Allocator) !void { + const parts = try pySplit(a, "a b c"); + defer a.free(parts); + try std.testing.expectEqual(@as(usize, 3), parts.len); +} + +test "pySplit frees its list when an allocation fails" { + try std.testing.checkAllAllocationFailures(std.testing.allocator, splitOwned, .{}); +} + test "shellQuote" { const gpa = std.testing.allocator; const cases = [_][2][]const u8{ From 0f81f20db4edd1359301d1571d10787b457c7d9d Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 00:09:34 +0000 Subject: [PATCH 06/20] fix(logs): do not publish a partial gzip under the final name The archive is written under a temporary name, synced and renamed. A crash or a short write no longer leaves a partial name.gz that a later run skips. --- changelog.d/+gzip-partial.bugfix.md | 1 + src/logs.zig | 96 ++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 changelog.d/+gzip-partial.bugfix.md diff --git a/changelog.d/+gzip-partial.bugfix.md b/changelog.d/+gzip-partial.bugfix.md new file mode 100644 index 0000000..4833319 --- /dev/null +++ b/changelog.d/+gzip-partial.bugfix.md @@ -0,0 +1 @@ +`logs.gzipInPlace` writes the archive under a temporary name and renames it into place, so a crash does not leave a partial `.gz` at the final name. diff --git a/src/logs.zig b/src/logs.zig index 6cdd7a8..e388aed 100644 --- a/src/logs.zig +++ b/src/logs.zig @@ -26,7 +26,18 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip const gz_name = try gpa.print("{s}.gz", .{name}); defer gpa.free(gz_name); - const dst = try sys.openat(dir, gz_name, .{ + // Refuse an existing archive before creating anything, as gzip does. + // The bytes then go to a temporary name, so a crash or a short write + // cannot leave a partial `name.gz` that a later run skips. + if (fsx.statAt(dir, gz_name, false)) |_| return error.Exists else |err| switch (err) { + error.NotFound => {}, + else => |e| return e, + } + var rnd: [4]u8 = undefined; + _ = try sys.call(linux.getrandom, .{ &rnd, rnd.len, 0 }); + const tmp_name = try gpa.print(".{s}.{d}.{x:0>8}.tmp", .{ gz_name, linux.getpid(), std.mem.readInt(u32, &rnd, .little) }); + defer gpa.free(tmp_name); + const dst = try sys.openat(dir, tmp_name, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, @@ -35,7 +46,7 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip var done = false; defer { sys.close(dst); - if (!done) sys.unlinkat(dir, gz_name, 0) catch {}; + if (!done) sys.unlinkat(dir, tmp_name, 0) catch {}; } const file: Io.File = .{ .handle = dst, .flags = .{ .nonblocking = false } }; @@ -81,8 +92,17 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip try sys.fchmod(dst, @intCast(src.stat.perm())); const times = [2]linux.timespec{ src.stat.atime, src.stat.mtime }; _ = try sys.call(linux.futimens, .{ dst, × }); - try sys.unlinkat(dir, name, 0); + try sys.fsync(dst); + try sys.renameat(dir, tmp_name, dir, gz_name); + sys.unlinkat(dir, name, 0) catch |err| { + // The archive is complete, but the original is still there. Drop + // the archive so a retry compresses the original instead of + // skipping an existing .gz and leaving both. + sys.unlinkat(dir, gz_name, 0) catch {}; + return err; + }; done = true; + try sys.fsync(dir); } /// The lines of a `.licensed-logs` list with every line found in `renames` @@ -110,6 +130,76 @@ pub fn renameInList(gpa: Allocator, content: []const u8, renames: *const std.Str return try out.toOwnedSlice(gpa); } +fn archiveIntact(path: [*:0]const u8) bool { + const pid: isize = @bitCast(linux.fork()); + if (pid == 0) { + const argv = [_:null]?[*:0]const u8{ "gzip", "-t", path, null }; + const envp = [_:null]?[*:0]const u8{}; + _ = linux.execve("/usr/bin/gzip", &argv, &envp); + linux.exit_group(127); + } + var status: u32 = 0; + _ = linux.wait4(@intCast(pid), @ptrCast(&status), 0, null); + return linux.W.IFEXITED(status) and linux.W.EXITSTATUS(status) == 0; +} + +test "gzip does not publish a partial archive under the final name" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const base = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/zig-common-gzip-{d}", .{ here, linux.getpid() }); + defer gpa.free(base); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(fsx.cwd, (try sys.z(&zb, base)).ptr, 0o700); + const dir = try fsx.openDir(fsx.cwd, base, false); + defer { + fsx.deleteTree(gpa, fsx.cwd, base) catch {}; + sys.close(dir); + } + const src = try sys.openat(dir, "a.log", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + const ur = try sys.openat(fsx.cwd, "/dev/urandom", .{}, 0); + defer sys.close(ur); + var buf: [64 * 1024]u8 = undefined; + var left: usize = 6 << 20; + while (left > 0) { + const n = try sys.read(ur, buf[0..@min(buf.len, left)]); + if (n == 0) return error.Unexpected; + try sys.writeAll(src, buf[0..n]); + left -= n; + } + sys.close(src); + const gz = try std.fmt.allocPrint(gpa, "{s}/a.log.gz", .{base}); + defer gpa.free(gz); + var gz_z: [sys.path_max:0]u8 = undefined; + const gz_sent = try sys.z(&gz_z, gz); + + const Watch = struct { + var stop = std.atomic.Value(bool).init(false); + var partial = std.atomic.Value(bool).init(false); + var path: [*:0]const u8 = undefined; + fn run() void { + while (!stop.load(.acquire)) { + if (fsx.statAt(fsx.cwd, std.mem.span(path), false)) |_| { + if (!archiveIntact(path)) partial.store(true, .release); + } else |_| {} + var ts: linux.timespec = .{ .sec = 0, .nsec = 2 * std.time.ns_per_ms }; + _ = linux.nanosleep(&ts, null); + } + } + }; + Watch.path = gz_sent.ptr; + Watch.stop.store(false, .release); + Watch.partial.store(false, .release); + const watch = try std.Thread.spawn(.{}, Watch.run, .{}); + const io = std.Io.Threaded.global_single_threaded.io(); + try gzipInPlace(gpa, io, dir, "a.log"); + Watch.stop.store(true, .release); + watch.join(); + try std.testing.expect(!Watch.partial.load(.acquire)); + try std.testing.expect(archiveIntact(gz_sent.ptr)); + try std.testing.expectError(error.NotFound, fsx.statAt(dir, "a.log", false)); +} + test "renameInList" { const gpa = std.testing.allocator; var m: std.StringHashMapUnmanaged([]const u8) = .empty; From 170141d995cee8976c8040074d037211e930b6cb Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 00:09:34 +0000 Subject: [PATCH 07/20] fix(proc): a signal during a /proc scan is not an empty group groupAlive treated EINTR from getdents as "nobody is alive", so waitGroupGone could return while the group was still running. --- changelog.d/+group-alive-eintr.bugfix.md | 1 + src/proc.zig | 29 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 changelog.d/+group-alive-eintr.bugfix.md diff --git a/changelog.d/+group-alive-eintr.bugfix.md b/changelog.d/+group-alive-eintr.bugfix.md new file mode 100644 index 0000000..5503de3 --- /dev/null +++ b/changelog.d/+group-alive-eintr.bugfix.md @@ -0,0 +1 @@ +`proc.groupAlive` retries a `/proc` read interrupted by a signal, instead of reporting that the process group is gone. diff --git a/src/proc.zig b/src/proc.zig index 8fb12e5..5ad554d 100644 --- a/src/proc.zig +++ b/src/proc.zig @@ -189,6 +189,9 @@ pub fn groupAlive(pgid: linux.pid_t) bool { var dents: [16 * 1024]u8 align(8) = undefined; while (true) { const n = linux.getdents64(dir, &dents, dents.len); + // EINTR is not a failed scan. Treating it as "nobody is alive" + // would let waitGroupGone return while the group is still running. + if (linux.errno(n) == .INTR) continue; if (linux.errno(n) != .SUCCESS or n == 0) return false; var pos: usize = 0; while (pos < n) { @@ -532,6 +535,32 @@ test "a set cancel flag kills the child" { try std.testing.expectError(error.Canceled, run(gpa, "/bin/sh", &.{ "sh", "-c", "sleep 30" }, no_env, .{ .cancel = &flag })); } +fn onUsr1Proc(_: linux.SIG) callconv(.c) void {} + +test "groupAlive stays true for a live group while signals arrive" { + const handler: linux.Sigaction = .{ .handler = .{ .handler = &onUsr1Proc }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.USR1, &handler, &prev); + defer _ = linux.sigaction(.USR1, &prev, null); + const Stop = struct { + var go = std.atomic.Value(bool).init(true); + fn buzz() void { + var blocked = linux.sigemptyset(); + linux.sigaddset(&blocked, .USR1); + _ = linux.sigprocmask(linux.SIG.BLOCK, &blocked, null); + while (go.load(.acquire)) _ = linux.kill(linux.getpid(), .USR1); + } + }; + Stop.go.store(true, .release); + const buzz = try std.Thread.spawn(.{}, Stop.buzz, .{}); + defer { + Stop.go.store(false, .release); + buzz.join(); + } + const pgid: linux.pid_t = @intCast(linux.getpgid(0)); + for (0..50) |_| try std.testing.expect(groupAlive(pgid)); +} + test "run refuses a relative program" { const gpa = std.testing.allocator; try std.testing.expectError(error.NotAbsolute, run(gpa, "sh", &.{"sh"}, no_env, .{})); From 9bb981327ff0ce61e847cdd327442d6e131e0e5c Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 00:09:34 +0000 Subject: [PATCH 08/20] docs: errno 133 is the hardware-error text COMPAT.md said errno 134 was the unnamed hardware-error text. On x86_64 that text is errno 133, HWPOISON, which Zig names; 134 is unnamed, as glibc prints it. The module summary matches the copy and the lock gate. --- COMPAT.md | 6 ++++-- README.md | 4 ++-- changelog.d/+errno-133.doc.md | 1 + src/sys.zig | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 changelog.d/+errno-133.doc.md diff --git a/COMPAT.md b/COMPAT.md index d3d1875..682c753 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -21,6 +21,8 @@ breaks a tool shows before the tool moves to it. `sys.describe` prints glibc's `strerror` text, as Python's `os.strerror` does, for every errno the Zig standard library's `linux.E` names. An errno it does -not name prints `[Errno N] Unknown error N`; on x86_64 that is errno 134, -which glibc calls "Memory page has hardware error". +not name prints `[Errno N] Unknown error N`. On x86_64, errno 133 is +`HWPOISON`, `[Errno 133] Memory page has hardware error`, and errno 134 is +not a named errno: both glibc and this library print `[Errno 134] Unknown +error 134`. diff --git a/README.md b/README.md index 4d0bc54..63ffeb4 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,11 @@ the entry and its hash. | module | what it does | |---|---| | `sys` | `std.os.linux` calls with EINTR retried and errno mapped to a Zig error set | -| `fsx` | open without following a final symbolic link, checks on the descriptor (`fstat`), cp-style copy, line append, atomic replace keeping owner, mode and access ACLs, directory iteration, `rm -rf` that never follows links, `realpath` through `/proc/self/fd` | +| `fsx` | open without following a final symbolic link, checks on the descriptor (`fstat`), a copy that renames into place so a short write does not destroy the destination, line append, atomic replace keeping owner, mode and access ACLs, directory iteration, `rm -rf` that never follows links, `realpath` through `/proc/self/fd` | | `proc` | run a program by absolute path with an empty signal mask, default handlers and no descriptors above 2, by default in its own session; stdin fed and stdout and stderr read at once, each with a cap; an optional deadline, SIGTERM, a grace period, SIGKILL to the process group; a cancel flag a signal handler can set; group members left after the child exits are killed before it is reaped; `which` looks a name up in an explicit PATH | | `logs` | `gzip -9` in place with gzip's header (name, mtime, OS byte), owner, mode and times kept; rewrite of `.licensed-logs` entries | | `env` | the environment as raw bytes with getenv's first-entry rule, duplicate names found, a child environment built with each name once, shell name check | -| `state` | private files in a shared directory: trusted only when the open file is a regular file of this user with one link and no group or other bits, a lock trusted the same way, atomic 0600 writes; `RootLock`, a flock gate plus a POSIX record lock on a sidecar, with a deadline | +| `state` | private files in a shared directory: trusted only when the open file is a regular file of this user with one link and no group or other bits, a lock trusted the same way, atomic 0600 writes; `RootLock`, a flock gate plus a POSIX record lock on a sidecar, with a deadline read from the clock. The gate and the sidecar must each be a regular file of this user with one link | | `json` | a JSON reader with Python's two string encodings (surrogateescape text, one byte per code point), Python values, and `json.dumps(indent=N, sort_keys=True)` byte for byte, float repr included | | `pytext` | byte strings as Python text: the surrogateescape character, the 'replace' decoder, str.isspace, strip and split, shlex.quote | | `atomic` | `atomic_write`: replace a file only after its bytes, owner, mode and every ACL form (GPFS NFSv4, NFSv4, POSIX access) are durable and read back; fault points before write, fsync and rename | diff --git a/changelog.d/+errno-133.doc.md b/changelog.d/+errno-133.doc.md new file mode 100644 index 0000000..5ca82eb --- /dev/null +++ b/changelog.d/+errno-133.doc.md @@ -0,0 +1 @@ +`COMPAT.md` names errno 133 as the hardware-error text. Errno 134 is unnamed, which is what glibc prints too. diff --git a/src/sys.zig b/src/sys.zig index fea0e2a..ec130c7 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -363,4 +363,18 @@ test "describe prints the errno the kernel gave, as Python's OSError does" { // An error that the last errno does not map to gets its usual errno. try std.testing.expectEqualStrings("[Errno 2] No such file or directory", describe(error.NotFound)); try std.testing.expectEqual(@as(?Error, null), asError(error.OutOfMemory)); + try std.testing.expectEqualStrings("[Errno 133] Memory page has hardware error", describeErrno(.HWPOISON)); + try std.testing.expectEqualStrings("[Errno 134] Unknown error 134", describeErrno(@fromBackingInt(@intCast(134)))); + const compat_fd = try openat(linux.AT.FDCWD, "COMPAT.md", .{}, 0); + defer close(compat_fd); + var compat: std.ArrayList(u8) = .empty; + defer compat.deinit(std.testing.allocator); + var chunk: [512]u8 = undefined; + while (true) { + const n = try read(compat_fd, &chunk); + if (n == 0) break; + try compat.appendSlice(std.testing.allocator, chunk[0..n]); + } + try std.testing.expect(std.mem.indexOf(u8, compat.items, "errno 133") != null); + try std.testing.expect(std.mem.indexOf(u8, compat.items, "errno 134,") == null); } From 7b717fdd70c4d0ba4a97d625ec6aa22708f9dcf7 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:42:13 +0000 Subject: [PATCH 09/20] test(proc): a standard descriptor closed at start A caller started with fd 0, 1 or 2 closed gets /dev/null at that number. With fd 1 closed the child's stderr ends up in the captured stdout; with fd 0 or 2 closed the child loses that stream at exec. Each test forks, closes the descriptor and checks what the child sees. They fail until childExec is fixed. --- src/proc.zig | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/proc.zig b/src/proc.zig index 5ad554d..12b5046 100644 --- a/src/proc.zig +++ b/src/proc.zig @@ -565,3 +565,52 @@ test "run refuses a relative program" { const gpa = std.testing.allocator; try std.testing.expectError(error.NotAbsolute, run(gpa, "sh", &.{"sh"}, no_env, .{})); } + +/// Run `body` in a forked child, which may close the descriptors the test +/// runner talks over, and expect it to exit 0. +fn expectChildPasses(comptime body: fn () u8) !void { + const pid: isize = @bitCast(linux.fork()); + if (pid < 0) return error.Unexpected; + if (pid == 0) linux.exit_group(body()); + var status: i32 = 0; + while (linux.errno(linux.wait4(@intCast(pid), &status, 0, null)) == .INTR) {} + try std.testing.expectEqual(Term{ .exited = 0 }, decode(@bitCast(status))); +} + +/// 0 when `script` under /bin/sh prints `want`, 1 when it prints something +/// else, 2 when it cannot be run. +fn shPrints(script: []const u8, stdin: ?[]const u8, want: []const u8) u8 { + const r = run(std.heap.page_allocator, "/bin/sh", &.{ "sh", "-c", script }, no_env, .{ .stdin = stdin }) catch return 2; + if (std.mem.eql(u8, r.stdout, want)) return 0; + std.debug.print("stdout \"{s}\", want \"{s}\"\n", .{ r.stdout, want }); + return 1; +} + +test "a closed stdout at start does not send the child's stderr to its stdout" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(1); + return shPrints("echo out; echo err >&2", null, "out\n"); + } + }.body); +} + +test "a closed stdin at start leaves the child's stdin and stderr on their files" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(0); + const r = shPrints("readlink /proc/self/fd/0 || echo closed", null, "/dev/null\n"); + if (r != 0) return r; + return shPrints("cat; readlink /proc/self/fd/2", "in\n", "in\n/dev/null\n"); + } + }.body); +} + +test "a closed stderr at start leaves the child's stderr on /dev/null" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(2); + return shPrints("readlink /proc/self/fd/2 || echo closed", null, "/dev/null\n"); + } + }.body); +} From 39a292aea7f98d78e80728bb2c3484d6ced8f2e3 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:43:12 +0000 Subject: [PATCH 10/20] fix(proc): a standard descriptor closed at start keeps the child's streams When the caller started with fd 0, 1 or 2 closed, /dev/null or a pipe took that number, and childExec's dup2 sequence either clobbered it (fd 1: the child's stderr went to the captured stdout) or made a no-op dup2 onto itself that kept close-on-exec (fd 0 or 2: the child started with that stream closed). Sources below 3 now move above 2 before the dup2s, as Python's _posixsubprocess does. --- changelog.d/+proc-std-fds.bugfix.md | 1 + src/proc.zig | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 changelog.d/+proc-std-fds.bugfix.md diff --git a/changelog.d/+proc-std-fds.bugfix.md b/changelog.d/+proc-std-fds.bugfix.md new file mode 100644 index 0000000..c586839 --- /dev/null +++ b/changelog.d/+proc-std-fds.bugfix.md @@ -0,0 +1 @@ +`proc.run` gives the child /dev/null or its pipes on fds 0 to 2 when the caller started with one of them closed: the child's stderr no longer lands in the captured stdout, and its stdin or stderr is no longer closed at exec. diff --git a/src/proc.zig b/src/proc.zig index 12b5046..83df0cc 100644 --- a/src/proc.zig +++ b/src/proc.zig @@ -130,9 +130,18 @@ fn childExec( _ = linux.sigaction(@fromBackingInt(@intCast(s)), &act, null); } _ = linux.sigprocmask(linux.SIG.SETMASK, &empty, null); - if (linux.errno(linux.dup2(in_fd, 0)) != .SUCCESS) linux.exit_group(127); - if (linux.errno(linux.dup2(out_fd, 1)) != .SUCCESS) linux.exit_group(127); - if (linux.errno(linux.dup2(err_fd, 2)) != .SUCCESS) linux.exit_group(127); + // A source at 0, 1 or 2 (the caller started with that descriptor + // closed) moves above 2 first: a dup2 onto it would clobber it, and a + // dup2 onto itself keeps its close-on-exec flag. + var src = [3]sys.fd_t{ in_fd, out_fd, err_fd }; + for (&src) |*fd| if (fd.* < 3) { + const moved = linux.fcntl(fd.*, linux.F.DUPFD, 3); + if (linux.errno(moved) != .SUCCESS) linux.exit_group(127); + fd.* = @intCast(moved); + }; + for (src, 0..) |fd, target| { + if (linux.errno(linux.dup2(fd, @intCast(target))) != .SUCCESS) linux.exit_group(127); + } const keep_fd = exec_err_fd; if (keep_fd > 3) _ = linux.close_range(3, @intCast(keep_fd - 1), .{ .UNSHARE = false, .CLOEXEC = false }); _ = linux.close_range(if (keep_fd >= 3) @intCast(keep_fd + 1) else 3, std.math.maxInt(linux.fd_t), .{ .UNSHARE = false, .CLOEXEC = false }); From 6d209c76f060b66faab07d5328ea082eda8eaf4e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:44:31 +0000 Subject: [PATCH 11/20] test(proc): a status lost to an ignored SIGCHLD With SIGCHLD set to SIG_IGN the kernel reaps the child itself and wait4 fails with ECHILD. reap reports that as a kill by signal 9, so a child that exited 3 reads as SIGKILLed. The test expects error.NoChild and fails until reap returns the wait4 error. --- src/proc.zig | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/proc.zig b/src/proc.zig index 83df0cc..0248309 100644 --- a/src/proc.zig +++ b/src/proc.zig @@ -623,3 +623,12 @@ test "a closed stderr at start leaves the child's stderr on /dev/null" { } }.body); } + +test "a status lost to an ignored SIGCHLD is an error, not a kill by signal 9" { + const ignore: linux.Sigaction = .{ .handler = .{ .handler = linux.SIG.IGN }, .mask = linux.sigemptyset(), .flags = 0 }; + var old: linux.Sigaction = undefined; + _ = linux.sigaction(.CHLD, &ignore, &old); + defer _ = linux.sigaction(.CHLD, &old, null); + try std.testing.expectError(error.NoChild, run(std.testing.allocator, "/bin/sh", &.{ "sh", "-c", "exit 3" }, no_env, .{})); + try std.testing.expectEqual(linux.E.CHILD, sys.last_errno); +} From de97e8f4bc8b24a12792c6acbf57e8497bcda0e7 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:45:25 +0000 Subject: [PATCH 12/20] fix(proc): a lost exit status is an error, not signal 9 reap turned any wait4 failure into a kill by signal 9. With SIGCHLD ignored the kernel reaps the child and wait4 fails with ECHILD, so run reported a SIGKILL for a child that exited normally. reap now returns the wait4 error through sys.call, run returns error.NoChild with sys.last_errno set, and run's doc says not to ignore SIGCHLD. The error paths still reap and keep their own error. --- changelog.d/+proc-wait-status.bugfix.md | 1 + src/proc.zig | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 changelog.d/+proc-wait-status.bugfix.md diff --git a/changelog.d/+proc-wait-status.bugfix.md b/changelog.d/+proc-wait-status.bugfix.md new file mode 100644 index 0000000..ed26eda --- /dev/null +++ b/changelog.d/+proc-wait-status.bugfix.md @@ -0,0 +1 @@ +`proc.run` returns `error.NoChild` when the child's status is lost because the caller ignores SIGCHLD, instead of reporting a kill by signal 9. diff --git a/src/proc.zig b/src/proc.zig index 0248309..a9b40e0 100644 --- a/src/proc.zig +++ b/src/proc.zig @@ -177,16 +177,12 @@ fn exitedWithin(pid: linux.pid_t, ms: ?i64) bool { } } -fn reap(pid: linux.pid_t) Term { - while (true) { - var status: i32 = 0; - const rc = linux.wait4(pid, &status, 0, null); - switch (linux.errno(rc)) { - .SUCCESS => return decode(@bitCast(status)), - .INTR => continue, - else => return .{ .signaled = 9 }, - } - } +/// Reap the child. With SIGCHLD ignored (SIG_IGN or SA_NOCLDWAIT) the +/// kernel has reaped it and its status is gone: `error.NoChild`. +fn reap(pid: linux.pid_t) sys.Error!Term { + var status: i32 = 0; + _ = try sys.call(linux.wait4, .{ pid, &status, 0, null }); + return decode(@bitCast(status)); } /// True while a live (not zombie) process is in process group `pgid`. A @@ -279,6 +275,8 @@ fn keep(i: usize, bytes: []const u8, out: *std.ArrayList(u8), err_buf: *std.Arra } /// Run `exe` (an absolute path) with `argv` (argv[0] included) and `envp`. +/// The caller must not ignore SIGCHLD (SIG_IGN or SA_NOCLDWAIT): the kernel +/// then reaps the child and its status is lost, which is `error.NoChild`. pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp, opts: Options) Error!Result { if (exe.len == 0 or exe[0] != '/') return error.NotAbsolute; @@ -370,7 +368,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp }; if (got == 4) { if (opts.pgid_out) |o| o.store(0, .seq_cst); - _ = reap(pid); + _ = reap(pid) catch null; _ = try sys.result(@bitCast(-@as(isize, @intCast(e)))); return error.Unexpected; } @@ -493,7 +491,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp if (opts.new_session) kill(pid, true, .KILL); if (opts.new_session and opts.wait_for_group) waitGroupGone(pid); if (opts.pgid_out) |o| o.store(0, .seq_cst); - _ = reap(pid); + _ = reap(pid) catch null; return err; } // Members of the child's group outliving it are not ours to keep; the @@ -501,7 +499,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp if (opts.new_session) kill(pid, true, .KILL); if (opts.new_session and opts.wait_for_group) waitGroupGone(pid); if (opts.pgid_out) |o| o.store(0, .seq_cst); - const term = reap(pid); + const term = try reap(pid); const o = try out.toOwnedSlice(gpa); errdefer gpa.free(o); return .{ .term = term, .stdout = o, .stderr = try err_buf.toOwnedSlice(gpa) }; From 2487373c847738f0da496351ba67b4d278de7ada Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:47:10 +0000 Subject: [PATCH 13/20] test(json): a truncated or malformed escape decodeCache("abc\\") and decodeWire("\\u12") index past the string and panic, as does a key lookup on a key that ends in a backslash. The decoders also take "\\u+123" (parseInt allows a sign) and read "\\x0041" as a \u escape. The test expects error.Encoding under std.testing.allocator, so nothing allocated before the error may leak. --- src/json.zig | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/json.zig b/src/json.zig index da7dcfc..512ae5f 100644 --- a/src/json.zig +++ b/src/json.zig @@ -378,6 +378,16 @@ test "wire strings are one byte per code point" { try std.testing.expectError(error.Encoding, decodeWire(a, "\\u0100")); } +test "a truncated or malformed escape is an encoding error" { + const gpa = std.testing.allocator; + for ([_][]const u8{ "abc\\", "\\u12", "\\ud83d\\ude0", "\\u+123", "\\x0041" }) |raw| { + try std.testing.expectError(error.Encoding, decodeCache(gpa, raw)); + try std.testing.expectError(error.Encoding, decodeWire(gpa, raw)); + } + const obj: Value = .{ .object = &.{.{ .key = "a\\", .value = .null }} }; + try std.testing.expect(obj.get("a") == null); +} + test "parse objects, duplicates and nesting" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); From fe15bec781a2445aca6bec90027cae271172470e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:48:14 +0000 Subject: [PATCH 14/20] fix(json): a truncated or malformed escape is an encoding error codePoints read raw[i + 1] and the four \u digits without a bounds check, so a string ending in a backslash or a short \u escape panicked. It also handed any unknown escape to the \u branch and parsed the digits with parseInt, which accepts a sign. Escapes are now checked: a cut-off, unknown or non-hex escape is error.Encoding. keyEql gets the same bounds check. The decoders free their scratch and partial output on error, and return exact-length slices. --- changelog.d/+json-escapes.bugfix.md | 1 + src/json.zig | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 changelog.d/+json-escapes.bugfix.md diff --git a/changelog.d/+json-escapes.bugfix.md b/changelog.d/+json-escapes.bugfix.md new file mode 100644 index 0000000..52a810b --- /dev/null +++ b/changelog.d/+json-escapes.bugfix.md @@ -0,0 +1 @@ +`json.decodeCache` and `json.decodeWire` return `error.Encoding` for an escape that is cut off, unknown or a `\u` without four hex digits, where they read past the string or took `\u+123` and `\x0041`; what they allocated before the error is freed. `Value.get` does not read past a key that ends in a backslash. diff --git a/src/json.zig b/src/json.zig index 512ae5f..5d50ac2 100644 --- a/src/json.zig +++ b/src/json.zig @@ -44,8 +44,7 @@ fn keyEql(raw: []const u8, want: []const u8) bool { if (i + 1 >= raw.len) return false; const e = raw[i + 1]; if (e == 'u') { - if (i + 6 > raw.len) return false; - c = std.fmt.parseInt(u16, raw[i + 2 .. i + 6], 16) catch return false; + c = hex4(raw, i + 2) catch return false; i += 6; } else { c = switch (e) { @@ -71,6 +70,13 @@ fn isHex(s: []const u8) bool { return true; } +/// The four hex digits of a \u escape at `raw[i..]`; parseInt alone would +/// also take a sign. +fn hex4(raw: []const u8, i: usize) error{Encoding}!u16 { + if (i + 4 > raw.len or !isHex(raw[i .. i + 4])) return error.Encoding; + return std.fmt.parseInt(u16, raw[i .. i + 4], 16) catch unreachable; +} + pub const ParseError = Allocator.Error || error{Syntax}; const max_depth = 32; @@ -224,7 +230,8 @@ pub fn parse(arena: Allocator, s: []const u8) ParseError!Value { pub const DecodeError = Allocator.Error || error{Encoding}; /// Code points of a raw JSON string, surrogate pairs joined; a lone -/// surrogate is returned as itself. +/// surrogate is returned as itself. An escape json.loads refuses (cut off, +/// unknown, or \u without four hex digits) is an error. fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { var out: std.ArrayList(u21) = .empty; errdefer out.deinit(arena); @@ -257,11 +264,10 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { try out.append(arena, s); continue; } - if (i + 4 > raw.len) return error.Encoding; - var cp: u21 = std.fmt.parseInt(u16, raw[i .. i + 4], 16) catch return error.Encoding; + var cp: u21 = try hex4(raw, i); i += 4; if (cp >= 0xD800 and cp <= 0xDBFF and i + 6 <= raw.len and raw[i] == '\\' and raw[i + 1] == 'u') { - const lo = std.fmt.parseInt(u16, raw[i + 2 .. i + 6], 16) catch return error.Encoding; + const lo = try hex4(raw, i + 2); if (lo >= 0xDC00 and lo <= 0xDFFF) { cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); i += 6; @@ -269,7 +275,7 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { } try out.append(arena, cp); } - return out.items; + return out.toOwnedSlice(arena); } /// Cache text: code points as UTF-8, a lone surrogate U+DC80..U+DCFF as the @@ -277,6 +283,7 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { /// environment and is an error. pub fn decodeCache(arena: Allocator, raw: []const u8) DecodeError![]u8 { const cps = try codePoints(arena, raw); + defer arena.free(cps); var out: std.ArrayList(u8) = .empty; errdefer out.deinit(arena); for (cps) |cp| { @@ -290,13 +297,15 @@ pub fn decodeCache(arena: Allocator, raw: []const u8) DecodeError![]u8 { try out.appendSlice(arena, b[0..n]); } } - return out.items; + return out.toOwnedSlice(arena); } /// Wire text: every code point must be at most U+00FF and is one byte. pub fn decodeWire(arena: Allocator, raw: []const u8) DecodeError![]u8 { const cps = try codePoints(arena, raw); + defer arena.free(cps); const out = try arena.alloc(u8, cps.len); + errdefer arena.free(out); for (cps, 0..) |cp, i| { if (cp > 0xFF) return error.Encoding; out[i] = @intCast(cp); From 303616d8566e8a752b0b67843d86259e5504a1b0 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:49:59 +0000 Subject: [PATCH 15/20] test(json): an integer past i64 json.loads keeps 18446744073709551616 as an exact int. toPy falls back to parseFloat when parseInt overflows, so it becomes the float 18446744073709552000 and dumps writes it back as 1.8446744073709552e+19. The test expects an error, and checks that the i64 bounds and float-shaped numbers still read as before. --- src/json.zig | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/json.zig b/src/json.zig index 5d50ac2..27cbe0d 100644 --- a/src/json.zig +++ b/src/json.zig @@ -621,3 +621,14 @@ test "dumps matches json.dumps(indent=2, sort_keys=True)" { const parsed = try toPy(a, try parse(a, s)); try std.testing.expectEqualStrings(s, try dumps(a, parsed, 2, true)); } + +test "an integer past i64 is an error, not a rounded float" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const a = arena_state.allocator(); + try std.testing.expectError(error.Encoding, toPy(a, try parse(a, "18446744073709551616"))); + try std.testing.expectError(error.Encoding, toPy(a, try parse(a, "[-9223372036854775809]"))); + try std.testing.expectEqual(Py{ .int = std.math.minInt(i64) }, try toPy(a, try parse(a, "-9223372036854775808"))); + try std.testing.expectEqual(Py{ .float = 1e19 }, try toPy(a, try parse(a, "1e19"))); + try std.testing.expectEqual(Py{ .float = 3.0 }, try toPy(a, try parse(a, "3.0"))); +} From 8687fa647c58b295e877016998d0af08b4c625a4 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:50:44 +0000 Subject: [PATCH 16/20] fix(json): an integer past i64 is an error, not a rounded float toPy tried parseInt and fell back to parseFloat on any failure, so an integer that overflows i64 came back as a float and changed type and digits. The JSON text's shape now decides: no fraction or exponent is an int, and one that does not fit is error.Encoding, which toPy already returns for a value it cannot represent. The error set is unchanged. --- changelog.d/+json-int-range.bugfix.md | 1 + src/json.zig | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 changelog.d/+json-int-range.bugfix.md diff --git a/changelog.d/+json-int-range.bugfix.md b/changelog.d/+json-int-range.bugfix.md new file mode 100644 index 0000000..57b8b18 --- /dev/null +++ b/changelog.d/+json-int-range.bugfix.md @@ -0,0 +1 @@ +`json.toPy` returns `error.Encoding` for an integer that does not fit an `i64`, which `json.loads` keeps exact, instead of a rounded float; a number with a fraction or an exponent is still a float. diff --git a/src/json.zig b/src/json.zig index 27cbe0d..a93e642 100644 --- a/src/json.zig +++ b/src/json.zig @@ -436,12 +436,24 @@ pub const Py = union(enum) { } }; -/// The parsed document as Python values, strings decoded as cache text. +/// A number as json.loads reads it: an int when the text has no fraction or +/// exponent, a float otherwise. An int that does not fit an i64, which +/// Python keeps exact, cannot be represented and is an error, not a +/// rounded float. +fn pyNumber(n: []const u8) error{Encoding}!Py { + if (std.mem.findAny(u8, n, ".eE") != null) { + return .{ .float = std.fmt.parseFloat(f64, n) catch return error.Encoding }; + } + return .{ .int = std.fmt.parseInt(i64, n, 10) catch return error.Encoding }; +} + +/// The parsed document as Python values, strings decoded as cache text; an +/// integer past i64 is `error.Encoding`. pub fn toPy(arena: Allocator, v: Value) DecodeError!Py { return switch (v) { .null => .null, .bool => |b| .{ .bool = b }, - .number => |n| if (std.fmt.parseInt(i64, n, 10)) |i| .{ .int = i } else |_| .{ .float = std.fmt.parseFloat(f64, n) catch return error.Encoding }, + .number => |n| try pyNumber(n), .string => |raw| .{ .str = try decodeCache(arena, raw) }, .array => |items| blk: { const out = try arena.alloc(Py, items.len); From 5b254d3dd097754193a6d93680ee1453706edf92 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:52:18 +0000 Subject: [PATCH 17/20] test(json): arrays and objects nest at most 32 deep The depth check runs on every value at depth > 32. An empty container at depth 32 has no value inside it to check, so 33 nested empty arrays parse while 33 arrays around a scalar do not. The test pins one limit for both: 32 nested arrays or objects. It fails until the check runs on entering a container. --- src/json.zig | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/json.zig b/src/json.zig index a93e642..559f9b4 100644 --- a/src/json.zig +++ b/src/json.zig @@ -412,6 +412,27 @@ test "parse objects, duplicates and nesting" { try std.testing.expectError(error.Syntax, parse(a, "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]")); } +/// `inner` inside `n` arrays, or objects when `open` opens one. +fn nested(a: Allocator, n: usize, open: []const u8, inner: []const u8, close: []const u8) Allocator.Error![]u8 { + var out: std.ArrayList(u8) = .empty; + for (0..n) |_| try out.appendSlice(a, open); + try out.appendSlice(a, inner); + for (0..n) |_| try out.appendSlice(a, close); + return out.items; +} + +test "arrays and objects nest at most 32 deep" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const a = arena_state.allocator(); + _ = try parse(a, try nested(a, 32, "[", "1", "]")); + _ = try parse(a, try nested(a, 31, "[", "[]", "]")); + _ = try parse(a, try nested(a, 31, "{\"k\":", "{}", "}")); + try std.testing.expectError(error.Syntax, parse(a, try nested(a, 33, "[", "1", "]"))); + try std.testing.expectError(error.Syntax, parse(a, try nested(a, 32, "[", "[]", "]"))); + try std.testing.expectError(error.Syntax, parse(a, try nested(a, 32, "{\"k\":", "{}", "}"))); +} + // ---- Python values and json.dumps ----------------------------------------- /// A value as a Python program built it for json.dumps. Strings are byte From 6b67296512751eba31174ccd47803d25b3038474 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:53:25 +0000 Subject: [PATCH 18/20] fix(json): one nesting limit of 32, documented with the module The depth check ran at the top of every value, so an empty container at depth 32 passed and the limit was 33 for empty containers but 32 around a scalar. It now runs on entering an array or object: at most 32 nested containers, now the public json.max_depth, documented at parse. The module doc still described one guard's JSON. It now names what the module holds: the parser, the cache and wire codecs, toPy, and dumps. --- changelog.d/+json-docs.bugfix.md | 1 + src/json.zig | 29 ++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 changelog.d/+json-docs.bugfix.md diff --git a/changelog.d/+json-docs.bugfix.md b/changelog.d/+json-docs.bugfix.md new file mode 100644 index 0000000..cbb5b65 --- /dev/null +++ b/changelog.d/+json-docs.bugfix.md @@ -0,0 +1 @@ +`json.parse` refuses arrays and objects nested more than 32 deep, now `json.max_depth`, whether or not the innermost one is empty; 33 nested empty containers used to parse. The limit is documented, and the module summary covers the parser, both codecs, `toPy` and `dumps`. diff --git a/src/json.zig b/src/json.zig index 559f9b4..8ed13ec 100644 --- a/src/json.zig +++ b/src/json.zig @@ -1,10 +1,15 @@ -//! The JSON the guard reads and writes, with the two string encodings the -//! Python guard used: -//! - cache: json.dumps(ensure_ascii=True) of text decoded with -//! surrogateescape, so a byte that is not UTF-8 is a lone \udcXX; -//! - wire: the restore helper's request and reply, one character per byte +//! JSON as the Python tools these programs replace read and wrote it. +//! `parse` reads one document into `Value`s, keeping strings and numbers +//! as their JSON text until a codec or `toPy` decodes them. The two string +//! codecs are the encodings those tools used: +//! - cache (`decodeCache`, `writeCacheString`): json.dumps(ensure_ascii=True) +//! of text decoded with surrogateescape, so a byte that is not UTF-8 is a +//! lone \udcXX; +//! - wire (`decodeWire`, `writeWireString`): one character per byte //! (latin-1), so every byte is \u00XX or itself. -//! Strings are kept as their raw JSON text until a codec decodes them. +//! `toPy` turns a document into `Py` values, strings as cache text, and +//! `dumps` writes `Py` values as json.dumps(indent=N, sort_keys=...) does, +//! byte for byte, float repr included. const std = @import("std"); const text = @import("pytext.zig"); @@ -79,7 +84,9 @@ fn hex4(raw: []const u8, i: usize) error{Encoding}!u16 { pub const ParseError = Allocator.Error || error{Syntax}; -const max_depth = 32; +/// Arrays and objects nest at most this deep. The parser recurses once per +/// level, so the bound keeps its stack use fixed. +pub const max_depth = 32; const Parser = struct { s: []const u8, @@ -98,8 +105,8 @@ const Parser = struct { return false; } + /// A value inside `depth` arrays and objects. fn value(p: *Parser, depth: usize) ParseError!Value { - if (depth > max_depth) return error.Syntax; p.ws(); if (p.i >= p.s.len) return error.Syntax; switch (p.s[p.i]) { @@ -108,6 +115,7 @@ const Parser = struct { 'f' => return if (p.lit("false")) .{ .bool = false } else error.Syntax, '"' => return .{ .string = try p.string() }, '[' => { + if (depth >= max_depth) return error.Syntax; p.i += 1; var items: std.ArrayList(Value) = .empty; p.ws(); @@ -131,6 +139,7 @@ const Parser = struct { } }, '{' => { + if (depth >= max_depth) return error.Syntax; p.i += 1; var members: std.ArrayList(Member) = .empty; p.ws(); @@ -218,7 +227,9 @@ const Parser = struct { } }; -/// Parse one JSON document; all memory comes from `arena`. +/// Parse one JSON document; all memory comes from `arena`. Arrays and +/// objects nested more than `max_depth` (32) deep are `error.Syntax`, +/// though json.loads reads them. pub fn parse(arena: Allocator, s: []const u8) ParseError!Value { var p: Parser = .{ .s = s, .arena = arena }; const v = try p.value(0); From 7ef8ba94b58d1f66854257915696eaaa6f902a25 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:55:31 +0000 Subject: [PATCH 19/20] test(fsx): copyTo after the source was read copyTo says it copies an open file as cp(1) does, but copyFd read from the descriptor's offset and left it at the end. A caller that read the first line got a copy without it. The test reads a header first, then expects the whole file in the copy and the offset where the read left it. --- src/fsx.zig | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/fsx.zig b/src/fsx.zig index de2aab6..51a77f6 100644 --- a/src/fsx.zig +++ b/src/fsx.zig @@ -489,3 +489,32 @@ test "appendLine does not leave a partial line when the write cannot finish" { defer gpa.free(text); try std.testing.expectEqualStrings("hi\n", text); } + +test "copyTo copies the whole source and leaves its offset" { + const gpa = std.testing.allocator; + var tmp_path_buf: [sys.path_max]u8 = undefined; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try realpath(cwd, ".", &cwd_buf); + const base = try std.mem.print(&tmp_path_buf, "{s}/.zig-cache/zig-common-copy-{d}", .{ here, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try openDir(cwd, base, false); + defer { + deleteTree(gpa, cwd, base) catch {}; + sys.close(d); + } + const f = try sys.openat(d, "src", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o600); + try sys.writeAll(f, "header\nbody\n"); + sys.close(f); + const src = try openRegular(d, "src", false); + defer sys.close(src.fd); + var head: [7]u8 = undefined; + try std.testing.expectEqual(@as(usize, 7), try sys.read(src.fd, &head)); + sys.close(try copyTo(src.fd, src.stat, d, "dst")); + const dst = try openRegular(d, "dst", false); + defer sys.close(dst.fd); + const got = try readAll(gpa, dst.fd, 100); + defer gpa.free(got); + try std.testing.expectEqualStrings("header\nbody\n", got); + try std.testing.expectEqual(@as(u64, 7), linux.lseek(src.fd, 0, linux.SEEK.CUR)); +} From 294bb8c332f4cb57084fd183c09c0d80e84971cb Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 01:56:36 +0000 Subject: [PATCH 20/20] fix(fsx): copyTo copies the whole file and leaves the offset copyTo's doc promises a cp(1) copy of an open file, but copyFd read from the current offset and left it at the end of the file. copyFd now reads with pread from byte 0, so the copy is whole and the caller's offset is left alone. eb-keep-logs, the one caller, rewinds the source after the copy, which now does nothing; its tests pass against this change. --- changelog.d/+copy-offset.bugfix.md | 1 + src/fsx.zig | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 changelog.d/+copy-offset.bugfix.md diff --git a/changelog.d/+copy-offset.bugfix.md b/changelog.d/+copy-offset.bugfix.md new file mode 100644 index 0000000..1231fdb --- /dev/null +++ b/changelog.d/+copy-offset.bugfix.md @@ -0,0 +1 @@ +`fsx.copyTo` copies the whole source, whatever the descriptor's offset, and leaves the offset where it was, as its doc says; it copied from the offset to the end and left the offset there. diff --git a/src/fsx.zig b/src/fsx.zig index 51a77f6..c29a6fb 100644 --- a/src/fsx.zig +++ b/src/fsx.zig @@ -121,23 +121,26 @@ pub fn openDir(dir: fd_t, path: []const u8, follow: bool) Error!fd_t { fn copyFd(src: fd_t, dst: fd_t) Error!void { var buf: [64 * 1024]u8 = undefined; + var off: linux.off_t = 0; while (true) { - const n = try sys.read(src, &buf); + const n = try sys.call(linux.pread, .{ src, &buf, buf.len, off }); if (n == 0) return; try sys.writeAll(dst, buf[0..n]); + off += @intCast(n); } } -/// Copy an open regular file to `dir/name`. The bytes are written to a -/// temporary file in `dir`, synced, and renamed over `name`, so a short -/// write (a full disk, a file-size limit) leaves an existing destination -/// as it was and does not publish a partial file under `name`. An existing -/// regular file keeps its owner, when this process may set it, and its -/// mode; a new file gets the source's permission bits less the umask. A -/// destination that is a symbolic link or not a regular file is refused. -/// A destination with other hard links keeps those links on the previous -/// inode; `name` is the new inode. Returns the open destination, which the -/// caller closes. +/// Copy an open regular file to `dir/name`: all of it, whatever the +/// descriptor's offset, which is left where it was. The bytes are written +/// to a temporary file in `dir`, synced, and renamed over `name`, so a +/// short write (a full disk, a file-size limit) leaves an existing +/// destination as it was and does not publish a partial file under `name`. +/// An existing regular file keeps its owner, when this process may set it, +/// and its mode; a new file gets the source's permission bits less the +/// umask. A destination that is a symbolic link or not a regular file is +/// refused. A destination with other hard links keeps those links on the +/// previous inode; `name` is the new inode. Returns the open destination, +/// which the caller closes. pub fn copyTo(src: fd_t, src_stat: Stat, dir: fd_t, name: []const u8) Error!fd_t { const existing: ?Stat = statAt(dir, name, false) catch |err| switch (err) { error.NotFound => null,