diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b28c1..9055909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,6 @@ # Changelog -## Unreleased - -* `state.RootLock.acquire` no longer truncates the file behind a hard link - planted at `.lock`. It opens the state root `O_DIRECTORY|O_NOFOLLOW` and - both lock files relative to it with `O_NOFOLLOW`, creating them mode - 0600; a lock file name that is not a regular file with one link is - refused before it is opened, so a FIFO or symbolic link there is never - followed; and once both locks are held, the gate must still be a regular - file whose one link is `.lock` before it is truncated. -* A refused gate is the new `RootLock.AcquireError.GateNotPrivate` - (`NotPrivate` stays the sidecar's), a symbolic link as the state root is - `NotDir`, and `LockOps.open_sidecar` takes the state root's descriptor - and a name. + ## 0.8.1 diff --git a/COMPAT.md b/COMPAT.md index d3d1875..682c753 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -21,6 +21,8 @@ breaks a tool shows before the tool moves to it. `sys.describe` prints glibc's `strerror` text, as Python's `os.strerror` does, for every errno the Zig standard library's `linux.E` names. An errno it does -not name prints `[Errno N] Unknown error N`; on x86_64 that is errno 134, -which glibc calls "Memory page has hardware error". +not name prints `[Errno N] Unknown error N`. On x86_64, errno 133 is +`HWPOISON`, `[Errno 133] Memory page has hardware error`, and errno 134 is +not a named errno: both glibc and this library print `[Errno 134] Unknown +error 134`. diff --git a/README.md b/README.md index 4d0bc54..63ffeb4 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,11 @@ the entry and its hash. | module | what it does | |---|---| | `sys` | `std.os.linux` calls with EINTR retried and errno mapped to a Zig error set | -| `fsx` | open without following a final symbolic link, checks on the descriptor (`fstat`), cp-style copy, line append, atomic replace keeping owner, mode and access ACLs, directory iteration, `rm -rf` that never follows links, `realpath` through `/proc/self/fd` | +| `fsx` | open without following a final symbolic link, checks on the descriptor (`fstat`), a copy that renames into place so a short write does not destroy the destination, line append, atomic replace keeping owner, mode and access ACLs, directory iteration, `rm -rf` that never follows links, `realpath` through `/proc/self/fd` | | `proc` | run a program by absolute path with an empty signal mask, default handlers and no descriptors above 2, by default in its own session; stdin fed and stdout and stderr read at once, each with a cap; an optional deadline, SIGTERM, a grace period, SIGKILL to the process group; a cancel flag a signal handler can set; group members left after the child exits are killed before it is reaped; `which` looks a name up in an explicit PATH | | `logs` | `gzip -9` in place with gzip's header (name, mtime, OS byte), owner, mode and times kept; rewrite of `.licensed-logs` entries | | `env` | the environment as raw bytes with getenv's first-entry rule, duplicate names found, a child environment built with each name once, shell name check | -| `state` | private files in a shared directory: trusted only when the open file is a regular file of this user with one link and no group or other bits, a lock trusted the same way, atomic 0600 writes; `RootLock`, a flock gate plus a POSIX record lock on a sidecar, with a deadline | +| `state` | private files in a shared directory: trusted only when the open file is a regular file of this user with one link and no group or other bits, a lock trusted the same way, atomic 0600 writes; `RootLock`, a flock gate plus a POSIX record lock on a sidecar, with a deadline read from the clock. The gate and the sidecar must each be a regular file of this user with one link | | `json` | a JSON reader with Python's two string encodings (surrogateescape text, one byte per code point), Python values, and `json.dumps(indent=N, sort_keys=True)` byte for byte, float repr included | | `pytext` | byte strings as Python text: the surrogateescape character, the 'replace' decoder, str.isspace, strip and split, shlex.quote | | `atomic` | `atomic_write`: replace a file only after its bytes, owner, mode and every ACL form (GPFS NFSv4, NFSv4, POSIX access) are durable and read back; fault points before write, fsync and rename | diff --git a/changelog.d/+append-short-write.bugfix.md b/changelog.d/+append-short-write.bugfix.md new file mode 100644 index 0000000..d50a541 --- /dev/null +++ b/changelog.d/+append-short-write.bugfix.md @@ -0,0 +1 @@ +`fsx.appendLine` rolls a short write back, so a full disk does not leave a partial line. diff --git a/changelog.d/+atomic-eintr.bugfix.md b/changelog.d/+atomic-eintr.bugfix.md new file mode 100644 index 0000000..075b0b9 --- /dev/null +++ b/changelog.d/+atomic-eintr.bugfix.md @@ -0,0 +1 @@ +`atomic` retries an ACL read interrupted by a signal, instead of refusing the replacement. diff --git a/changelog.d/+atomic-leak.bugfix.md b/changelog.d/+atomic-leak.bugfix.md new file mode 100644 index 0000000..2640931 --- /dev/null +++ b/changelog.d/+atomic-leak.bugfix.md @@ -0,0 +1 @@ +`atomicWrite` frees the path, the step names and the spare ACL read it allocates. diff --git a/changelog.d/+copy-short-write.bugfix.md b/changelog.d/+copy-short-write.bugfix.md new file mode 100644 index 0000000..da6671e --- /dev/null +++ b/changelog.d/+copy-short-write.bugfix.md @@ -0,0 +1 @@ +`fsx.copyTo` renames the new bytes into place, so a short write leaves the existing file unchanged. diff --git a/changelog.d/+errno-133.doc.md b/changelog.d/+errno-133.doc.md new file mode 100644 index 0000000..5ca82eb --- /dev/null +++ b/changelog.d/+errno-133.doc.md @@ -0,0 +1 @@ +`COMPAT.md` names errno 133 as the hardware-error text. Errno 134 is unnamed, which is what glibc prints too. diff --git a/changelog.d/+group-alive-eintr.bugfix.md b/changelog.d/+group-alive-eintr.bugfix.md new file mode 100644 index 0000000..5503de3 --- /dev/null +++ b/changelog.d/+group-alive-eintr.bugfix.md @@ -0,0 +1 @@ +`proc.groupAlive` retries a `/proc` read interrupted by a signal, instead of reporting that the process group is gone. diff --git a/changelog.d/+gzip-partial.bugfix.md b/changelog.d/+gzip-partial.bugfix.md new file mode 100644 index 0000000..4833319 --- /dev/null +++ b/changelog.d/+gzip-partial.bugfix.md @@ -0,0 +1 @@ +`logs.gzipInPlace` writes the archive under a temporary name and renames it into place, so a crash does not leave a partial `.gz` at the final name. diff --git a/changelog.d/+json-escape.bugfix.md b/changelog.d/+json-escape.bugfix.md new file mode 100644 index 0000000..006d50b --- /dev/null +++ b/changelog.d/+json-escape.bugfix.md @@ -0,0 +1 @@ +`json` returns an encoding error for a truncated or unknown escape, instead of reading past the string. diff --git a/changelog.d/+json-escapes.bugfix.md b/changelog.d/+json-escapes.bugfix.md new file mode 100644 index 0000000..52a810b --- /dev/null +++ b/changelog.d/+json-escapes.bugfix.md @@ -0,0 +1 @@ +`json.decodeCache` and `json.decodeWire` return `error.Encoding` for an escape that is cut off, unknown or a `\u` without four hex digits, where they read past the string or took `\u+123` and `\x0041`; what they allocated before the error is freed. `Value.get` does not read past a key that ends in a backslash. diff --git a/changelog.d/+lock-gate.bugfix.md b/changelog.d/+lock-gate.bugfix.md new file mode 100644 index 0000000..f387f65 --- /dev/null +++ b/changelog.d/+lock-gate.bugfix.md @@ -0,0 +1 @@ +`state.RootLock` also refuses a gate or sidecar that another user owns, as `GateNotPrivate` and `NotPrivate`: each must be a regular file of this user with one link. diff --git a/changelog.d/+lock-timeout.bugfix.md b/changelog.d/+lock-timeout.bugfix.md new file mode 100644 index 0000000..8c5eb11 --- /dev/null +++ b/changelog.d/+lock-timeout.bugfix.md @@ -0,0 +1 @@ +`state.lockExclusive` measures its deadline on the monotonic clock, so a signal that cuts a sleep short does not expire the wait. diff --git a/changelog.d/+proc-std-fds.bugfix.md b/changelog.d/+proc-std-fds.bugfix.md new file mode 100644 index 0000000..c586839 --- /dev/null +++ b/changelog.d/+proc-std-fds.bugfix.md @@ -0,0 +1 @@ +`proc.run` gives the child /dev/null or its pipes on fds 0 to 2 when the caller started with one of them closed: the child's stderr no longer lands in the captured stdout, and its stdin or stderr is no longer closed at exec. diff --git a/changelog.d/+proc-wait-status.bugfix.md b/changelog.d/+proc-wait-status.bugfix.md new file mode 100644 index 0000000..ed26eda --- /dev/null +++ b/changelog.d/+proc-wait-status.bugfix.md @@ -0,0 +1 @@ +`proc.run` returns `error.NoChild` when the child's status is lost because the caller ignores SIGCHLD, instead of reporting a kill by signal 9. diff --git a/changelog.d/+pysplit-oom.bugfix.md b/changelog.d/+pysplit-oom.bugfix.md new file mode 100644 index 0000000..8a550c8 --- /dev/null +++ b/changelog.d/+pysplit-oom.bugfix.md @@ -0,0 +1 @@ +`pytext.pySplit` frees its list when an allocation fails, and the slice it returns is the whole allocation. diff --git a/changelog.d/+rootlock-api.bugfix.md b/changelog.d/+rootlock-api.bugfix.md new file mode 100644 index 0000000..59e0f64 --- /dev/null +++ b/changelog.d/+rootlock-api.bugfix.md @@ -0,0 +1 @@ +A refused gate is the new `RootLock.AcquireError.GateNotPrivate` (`NotPrivate` stays the sidecar's), a symbolic link as the state root is `NotDir`, and `LockOps.open_sidecar` takes the state root's descriptor and a name. diff --git a/changelog.d/+rootlock-truncate.bugfix.md b/changelog.d/+rootlock-truncate.bugfix.md new file mode 100644 index 0000000..72d1dd9 --- /dev/null +++ b/changelog.d/+rootlock-truncate.bugfix.md @@ -0,0 +1 @@ +`state.RootLock.acquire` no longer truncates the file behind a hard link planted at `.lock`. It opens the state root `O_DIRECTORY|O_NOFOLLOW` and both lock files relative to it with `O_NOFOLLOW`, creating them mode 0600; a lock file name that is not a regular file with one link is refused before it is opened, so a FIFO or symbolic link there is never followed; and once both locks are held, the gate must still be a regular file whose one link is `.lock` before it is truncated. diff --git a/src/atomic.zig b/src/atomic.zig index 49dc641..1d1f15e 100644 --- a/src/atomic.zig +++ b/src/atomic.zig @@ -94,11 +94,19 @@ pub const Why = struct { pub const Error = error{ Refused, OutOfMemory } || sys.Error; +const static_refused: []const u8 = "refused"; + fn refuse(a: Allocator, why: *Why, comptime f: []const u8, args: anytype) Error { - why.msg = a.print(f, args) catch "refused"; + why.msg = a.print(f, args) catch static_refused; return error.Refused; } +/// Free a message `refuse` stored, including its out-of-memory fallback. +fn freeMsg(a: Allocator, msg: []const u8) void { + if (msg.len == 0 or msg.ptr == static_refused.ptr) return; + a.free(msg); +} + /// The value of each ACL attribute present, keyed by its name. pub const Acls = struct { present: [acl_attributes.len]?[]const u8 = .{ null, null, null }, @@ -159,12 +167,27 @@ fn errnoText(e: linux.E) []const u8 { }; } +fn freeAcls(a: Allocator, acls: Acls) void { + for (acls.present) |p| if (p) |s| a.free(s); +} + +/// One `fgetxattr`, retrying when a signal interrupts it. `op` cannot be +/// used here: the errno itself decides whether a missing attribute is an +/// error, and `op` collapses errnos into `sys.Error`. +fn xattrGet(fd: sys.fd_t, name: [:0]const u8, buf: []u8) usize { + while (true) { + const rc = ops.fgetxattr(fd, name, buf); + if (linux.errno(rc) != .INTR) return rc; + } +} + fn readAllAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { const listed = try listedAcls(a, fd, why); var acls: Acls = .{}; + errdefer freeAcls(a, acls); var buf: [64 * 1024]u8 = undefined; for (acl_attributes, 0..) |name, i| { - const rc = ops.fgetxattr(fd, name, &buf); + const rc = xattrGet(fd, name, &buf); const e = linux.errno(rc); if (e == .SUCCESS) { acls.present[i] = try a.dupe(u8, buf[0..rc]); @@ -189,7 +212,9 @@ fn readAllAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { pub fn readAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { const before = try fsx.fstat(fd); const acls = try readAllAcls(a, fd, why); + errdefer freeAcls(a, acls); const again = try readAllAcls(a, fd, why); + defer freeAcls(a, again); if (!acls.eql(again) or !sameIdentity(try fsx.fstat(fd), before)) return refuse(a, why, "access metadata changed during inspection", .{}); return acls; @@ -225,13 +250,16 @@ fn restoreMetadata(a: Allocator, fd: sys.fd_t, metadata: ?Metadata, mode: u32, w break; }; if (primary) |i| { - if (!(try readAcls(a, fd, why)).eql(m.acls)) { + const now = try readAcls(a, fd, why); + defer freeAcls(a, now); + if (!now.eql(m.acls)) { const v = m.acls.present[i].?; _ = try op(ops.fsetxattr, .{ fd, acl_attributes[i], v }); } } for (acl_attributes) |_| { const present = try readAcls(a, fd, why); + defer freeAcls(a, present); var extra: ?usize = null; for (present.present, m.acls.present, 0..) |p, want, i| if (p != null and want == null) { extra = i; @@ -243,7 +271,9 @@ fn restoreMetadata(a: Allocator, fd: sys.fd_t, metadata: ?Metadata, mode: u32, w const restored = try fsx.fstat(fd); if (restored.uid != m.info.uid or restored.gid != m.info.gid or restored.perm() != mode) return refuse(a, why, "ownership or mode did not round-trip: expected {d}:{d} {o}, found {d}:{d} {o}", .{ m.info.uid, m.info.gid, mode, restored.uid, restored.gid, restored.perm() }); - if (!(try readAcls(a, fd, why)).eql(m.acls)) return refuse(a, why, "access-control attributes did not round-trip", .{}); + const round = try readAcls(a, fd, why); + defer freeAcls(a, round); + if (!round.eql(m.acls)) return refuse(a, why, "access-control attributes did not round-trip", .{}); } /// Called before each named step; a test hook can stop the program there. @@ -359,12 +389,17 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write } else |_| {} const metadata_path: ?[]const u8 = opts.metadata_from orelse (if (exists(path)) path else null); var metadata: ?Metadata = null; + defer if (metadata) |m| freeAcls(a, m.acls); if (metadata_path) |mp| { var inner: Why = .{}; - metadata = readMetadata(a, mp, &inner) catch |err| return refuse(a, why, "cannot read access metadata for {s}: {s}", .{ mp, if (err == error.Refused) inner.msg else describe(err) }); + metadata = readMetadata(a, mp, &inner) catch |err| { + defer freeMsg(a, inner.msg); + return refuse(a, why, "cannot read access metadata for {s}: {s}", .{ mp, if (err == error.Refused) inner.msg else describe(err) }); + }; } const mode: u32 = opts.mode orelse (if (metadata) |m| m.info.perm() else fileMode(path)); const tmp = try a.print("{s}/.{s}.new", .{ parent, std.fs.path.basename(path) }); + defer a.free(tmp); if (fsx.statAt(fsx.cwd, tmp, false)) |stale| { // A privileged writer hands its temporary file to the metadata owner // before a crash can interrupt it. @@ -377,6 +412,7 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write { const step = try a.print("{s}.write", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } const fd = try sys.openat(fsx.cwd, tmp, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, .NOFOLLOW = true }, 0o600); @@ -392,20 +428,29 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write defer sys.close(fd); try sys.writeAll(fd, data); var inner: Why = .{}; - restoreMetadata(a, fd, metadata, mode, &inner) catch |err| return refuse(a, why, "cannot preserve access metadata for {s}: {s}", .{ path, if (err == error.Refused) inner.msg else describe(err) }); + restoreMetadata(a, fd, metadata, mode, &inner) catch |err| { + defer freeMsg(a, inner.msg); + return refuse(a, why, "cannot preserve access metadata for {s}: {s}", .{ path, if (err == error.Refused) inner.msg else describe(err) }); + }; { const step = try a.print("{s}.fsync", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } _ = try op(ops.fsync, .{fd}); } if (metadata) |m| { var inner: Why = .{}; - const current = readMetadata(a, metadata_path.?, &inner) catch |err| return refuse(a, why, "cannot recheck access metadata for {s}: {s}", .{ metadata_path.?, if (err == error.Refused) inner.msg else describe(err) }); + const current = readMetadata(a, metadata_path.?, &inner) catch |err| { + defer freeMsg(a, inner.msg); + return refuse(a, why, "cannot recheck access metadata for {s}: {s}", .{ metadata_path.?, if (err == error.Refused) inner.msg else describe(err) }); + }; + defer freeAcls(a, current.acls); if (!sameMetadata(current, m)) return refuse(a, why, "access metadata of {s} changed before {s} was replaced", .{ metadata_path.?, path }); } { const step = try a.print("{s}.rename", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } try sys.renameat(fsx.cwd, tmp, fsx.cwd, path); @@ -474,6 +519,55 @@ test "a step a test makes fail leaves the destination and no temporary file" { } } +test "atomicWrite frees what it allocates" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const dir = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/atomic-noleak-{d}", .{ cwd, linux.getpid() }); + defer gpa.free(dir); + try makedirs(dir); + defer fsx.deleteTree(gpa, fsx.cwd, dir) catch {}; + const path = try std.fmt.allocPrint(gpa, "{s}/j.json", .{dir}); + defer gpa.free(path); + var why: Why = .{}; + try atomicWrite(gpa, path, "one\n", .{}, &why); + try atomicWrite(gpa, path, "two\n", .{}, &why); + try std.testing.expectEqual(@as(usize, 0), why.msg.len); +} + +test "an interrupted ACL read is retried" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const dir = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/atomic-eintr-{d}", .{ cwd, linux.getpid() }); + defer gpa.free(dir); + try makedirs(dir); + defer fsx.deleteTree(gpa, fsx.cwd, dir) catch {}; + const path = try std.fmt.allocPrint(gpa, "{s}/f", .{dir}); + defer gpa.free(path); + const created = try sys.openat(fsx.cwd, path, .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o600); + sys.close(created); + const fd = try sys.openat(fsx.cwd, path, .{}, 0); + defer sys.close(fd); + const old = ops.*; + defer ops.* = old; + const S = struct { + var left: u8 = 0; + fn get(fd_: sys.fd_t, name: [:0]const u8, buf: []u8) usize { + if (left > 0) { + left -= 1; + return @bitCast(-@as(isize, @backingInt(linux.E.INTR))); + } + return linux.fgetxattr(fd_, name.ptr, buf.ptr, buf.len); + } + }; + S.left = 2; + ops.fgetxattr = S.get; + var why: Why = .{}; + const acls = try readAcls(gpa, fd, &why); + defer freeAcls(gpa, acls); +} + test "the umask is read without changing it" { const old: u32 = @intCast(linux.syscall1(.umask, 0o027)); defer _ = linux.syscall1(.umask, old); diff --git a/src/fsx.zig b/src/fsx.zig index e473fc0..de2aab6 100644 --- a/src/fsx.zig +++ b/src/fsx.zig @@ -128,22 +128,55 @@ fn copyFd(src: fd_t, dst: fd_t) Error!void { } } -/// Copy an open regular file to `dir/name`, as cp(1) without options does: -/// an existing destination is truncated and keeps its mode, a new one gets -/// the source's permission bits less the umask. A destination that is a -/// symbolic link or not a regular file is refused. Returns the open -/// destination, which the caller closes. +/// Copy an open regular file to `dir/name`. The bytes are written to a +/// temporary file in `dir`, synced, and renamed over `name`, so a short +/// write (a full disk, a file-size limit) leaves an existing destination +/// as it was and does not publish a partial file under `name`. An existing +/// regular file keeps its owner, when this process may set it, and its +/// mode; a new file gets the source's permission bits less the umask. A +/// destination that is a symbolic link or not a regular file is refused. +/// A destination with other hard links keeps those links on the previous +/// inode; `name` is the new inode. Returns the open destination, which the +/// caller closes. pub fn copyTo(src: fd_t, src_stat: Stat, dir: fd_t, name: []const u8) Error!fd_t { - const fd = try sys.openat(dir, name, .{ + const existing: ?Stat = statAt(dir, name, false) catch |err| switch (err) { + error.NotFound => null, + else => |e| return e, + }; + if (existing) |st| { + if (st.isLink()) return error.SymLink; + if (st.isDir()) return error.IsDir; + if (!st.isReg()) return error.InvalidArgument; + } + var rnd: [4]u8 = undefined; + _ = try sys.call(linux.getrandom, .{ &rnd, rnd.len, 0 }); + var tmp_buf: [sys.path_max]u8 = undefined; + const tmp = std.fmt.bufPrint(&tmp_buf, ".{s}.{d}.{x:0>8}.tmp", .{ std.fs.path.basename(name), linux.getpid(), std.mem.readInt(u32, &rnd, .little) }) catch return error.NameTooLong; + const fd = try sys.openat(dir, tmp, .{ .ACCMODE = .WRONLY, .CREAT = true, + .EXCL = true, .NOFOLLOW = true, - .NONBLOCK = true, - }, @intCast(src_stat.mode & 0o777)); - errdefer sys.close(fd); - if (!(try fstat(fd)).isReg()) return error.InvalidArgument; - _ = try sys.call(linux.ftruncate, .{ fd, 0 }); + }, if (existing != null) 0o600 else @intCast(src_stat.mode & 0o777)); + var done = false; + errdefer { + sys.close(fd); + if (!done) sys.unlinkat(dir, tmp, 0) catch {}; + } try copyFd(src, fd); + if (existing) |st| { + if ((try fstat(fd)).uid != st.uid or (try fstat(fd)).gid != st.gid) { + sys.fchown(fd, st.uid, st.gid) catch |err| switch (err) { + error.AccessDenied => {}, + else => |e| return e, + }; + } + try sys.fchmod(fd, @intCast(st.perm())); + } + try sys.fsync(fd); + try sys.renameat(dir, tmp, dir, name); + done = true; + try sys.fsync(dir); return fd; } @@ -159,10 +192,22 @@ pub fn appendLine(gpa: Allocator, dir: fd_t, name: []const u8, line: []const u8) .NONBLOCK = true, }, 0o666); defer sys.close(fd); - if (!(try fstat(fd)).isReg()) return error.InvalidArgument; + const st = try fstat(fd); + if (!st.isReg()) return error.InvalidArgument; const bytes = try std.mem.concat(gpa, u8, &.{ line, "\n" }); defer gpa.free(bytes); - try sys.writeAll(fd, bytes); + // One write is atomic on Linux only when it completes. A short write + // (ENOSPC part way through) is rolled back to the size before it, so + // the list does not gain a partial line. Safe while this process is + // the only writer; a concurrent append in that window can be shortened. + sys.writeAll(fd, bytes) catch |err| { + if (fstat(fd)) |now| { + if (now.size > st.size and now.size - st.size < bytes.len) { + _ = linux.ftruncate(fd, @intCast(st.size)); + } + } else |_| {} + return err; + }; } /// Read a whole regular file opened by the caller; more than `limit` bytes @@ -355,3 +400,92 @@ test "replaceContents keeps the mode" { defer gpa.free(got); try std.testing.expectEqualStrings("b\n", got); } + +fn withFileLimit(bytes: u64, body: *const fn () anyerror!void) anyerror!void { + var old: linux.rlimit = undefined; + if (linux.errno(linux.prlimit(0, .FSIZE, null, &old)) != .SUCCESS) return error.Unexpected; + const lim: linux.rlimit = .{ .cur = @intCast(bytes), .max = old.max }; + if (linux.errno(linux.prlimit(0, .FSIZE, &lim, null)) != .SUCCESS) return error.Unexpected; + const ign: linux.Sigaction = .{ .handler = .{ .handler = linux.SIG.IGN }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.XFSZ, &ign, &prev); + defer { + _ = linux.sigaction(.XFSZ, &prev, null); + _ = linux.prlimit(0, .FSIZE, &old, null); + } + try body(); +} + +test "copyTo leaves an existing file unchanged when the write cannot finish" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try realpath(cwd, ".", &cwd_buf); + var pb: [sys.path_max]u8 = undefined; + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-copy-{d}", .{ here, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try openDir(cwd, base, false); + defer { + deleteTree(gpa, cwd, base) catch {}; + sys.close(d); + } + const dst = try sys.openat(d, "dest", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o640); + try sys.writeAll(dst, "OLDDATA\n"); + sys.close(dst); + const srcfd = try sys.openat(d, "src", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + try sys.writeAll(srcfd, "NEWDATA-IS-LONGER-THAN-THE-LIMIT\n"); + sys.close(srcfd); + const opened = try openRegular(d, "src", false); + defer sys.close(opened.fd); + + const Hold = struct { + var dir: fd_t = -1; + var src: fd_t = -1; + var st: Stat = undefined; + fn go() Error!void { + if (copyTo(src, st, dir, "dest")) |fd| sys.close(fd) else |_| {} + } + }; + Hold.dir = d; + Hold.src = opened.fd; + Hold.st = opened.stat; + try withFileLimit(8, Hold.go); + + const got = try openRegular(d, "dest", false); + defer sys.close(got.fd); + const text = try readAll(gpa, got.fd, 100); + defer gpa.free(text); + try std.testing.expectEqualStrings("OLDDATA\n", text); + try std.testing.expectEqual(@as(u32, 0o640), got.stat.perm()); +} + +test "appendLine does not leave a partial line when the write cannot finish" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try realpath(cwd, ".", &cwd_buf); + var pb: [sys.path_max]u8 = undefined; + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-append-{d}", .{ here, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try openDir(cwd, base, false); + defer { + deleteTree(gpa, cwd, base) catch {}; + sys.close(d); + } + const f = try sys.openat(d, "list", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + try sys.writeAll(f, "hi\n"); + sys.close(f); + const Hold = struct { + var dir: fd_t = -1; + fn go() Error!void { + appendLine(std.testing.allocator, dir, "list", "this line is far longer than the file size limit") catch {}; + } + }; + Hold.dir = d; + try withFileLimit(8, Hold.go); + const got = try openRegular(d, "list", false); + defer sys.close(got.fd); + const text = try readAll(gpa, got.fd, 200); + defer gpa.free(text); + try std.testing.expectEqualStrings("hi\n", text); +} diff --git a/src/json.zig b/src/json.zig index 8c732f9..5d50ac2 100644 --- a/src/json.zig +++ b/src/json.zig @@ -41,9 +41,10 @@ fn keyEql(raw: []const u8, want: []const u8) bool { while (i < raw.len) { var c: u21 = raw[i]; if (c == '\\') { + if (i + 1 >= raw.len) return false; const e = raw[i + 1]; if (e == 'u') { - c = std.fmt.parseInt(u16, raw[i + 2 .. i + 6], 16) catch return false; + c = hex4(raw, i + 2) catch return false; i += 6; } else { c = switch (e) { @@ -69,6 +70,13 @@ fn isHex(s: []const u8) bool { return true; } +/// The four hex digits of a \u escape at `raw[i..]`; parseInt alone would +/// also take a sign. +fn hex4(raw: []const u8, i: usize) error{Encoding}!u16 { + if (i + 4 > raw.len or !isHex(raw[i .. i + 4])) return error.Encoding; + return std.fmt.parseInt(u16, raw[i .. i + 4], 16) catch unreachable; +} + pub const ParseError = Allocator.Error || error{Syntax}; const max_depth = 32; @@ -222,9 +230,11 @@ pub fn parse(arena: Allocator, s: []const u8) ParseError!Value { pub const DecodeError = Allocator.Error || error{Encoding}; /// Code points of a raw JSON string, surrogate pairs joined; a lone -/// surrogate is returned as itself. +/// surrogate is returned as itself. An escape json.loads refuses (cut off, +/// unknown, or \u without four hex digits) is an error. fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { var out: std.ArrayList(u21) = .empty; + errdefer out.deinit(arena); var i: usize = 0; while (i < raw.len) { const c = raw[i]; @@ -235,6 +245,7 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { i += ch.len; continue; } + if (i + 1 >= raw.len) return error.Encoding; const e = raw[i + 1]; i += 2; const simple: ?u21 = switch (e) { @@ -246,16 +257,17 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { 'n' => '\n', 'r' => '\r', 't' => '\t', - else => null, + 'u' => null, + else => return error.Encoding, }; if (simple) |s| { try out.append(arena, s); continue; } - var cp: u21 = std.fmt.parseInt(u16, raw[i .. i + 4], 16) catch return error.Encoding; + var cp: u21 = try hex4(raw, i); i += 4; if (cp >= 0xD800 and cp <= 0xDBFF and i + 6 <= raw.len and raw[i] == '\\' and raw[i + 1] == 'u') { - const lo = std.fmt.parseInt(u16, raw[i + 2 .. i + 6], 16) catch return error.Encoding; + const lo = try hex4(raw, i + 2); if (lo >= 0xDC00 and lo <= 0xDFFF) { cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); i += 6; @@ -263,7 +275,7 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { } try out.append(arena, cp); } - return out.items; + return out.toOwnedSlice(arena); } /// Cache text: code points as UTF-8, a lone surrogate U+DC80..U+DCFF as the @@ -271,7 +283,9 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { /// environment and is an error. pub fn decodeCache(arena: Allocator, raw: []const u8) DecodeError![]u8 { const cps = try codePoints(arena, raw); + defer arena.free(cps); var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(arena); for (cps) |cp| { if (cp >= 0xDC80 and cp <= 0xDCFF) { try out.append(arena, @intCast(cp - 0xDC00)); @@ -283,13 +297,15 @@ pub fn decodeCache(arena: Allocator, raw: []const u8) DecodeError![]u8 { try out.appendSlice(arena, b[0..n]); } } - return out.items; + return out.toOwnedSlice(arena); } /// Wire text: every code point must be at most U+00FF and is one byte. pub fn decodeWire(arena: Allocator, raw: []const u8) DecodeError![]u8 { const cps = try codePoints(arena, raw); + defer arena.free(cps); const out = try arena.alloc(u8, cps.len); + errdefer arena.free(out); for (cps, 0..) |cp, i| { if (cp > 0xFF) return error.Encoding; out[i] = @intCast(cp); @@ -350,6 +366,13 @@ test "cache strings round-trip bytes that are not UTF-8" { const v = try parse(a, out.items); try std.testing.expectEqualStrings("a\"\xff\u{e9}\u{1F600}\n\x7f", try decodeCache(a, v.string)); try std.testing.expectError(error.Encoding, decodeCache(a, "\\ud800")); + // A truncated escape is an encoding error. It must not panic, and the + // bytes allocated before the error are freed when the caller did not + // pass an arena. + try std.testing.expectError(error.Encoding, decodeCache(std.testing.allocator, "a\\")); + try std.testing.expectError(error.Encoding, decodeCache(std.testing.allocator, "\\u12")); + const broken: Value = .{ .object = &.{.{ .key = "\\", .value = .null }} }; + try std.testing.expect(broken.get("x") == null); } test "wire strings are one byte per code point" { @@ -364,6 +387,16 @@ test "wire strings are one byte per code point" { try std.testing.expectError(error.Encoding, decodeWire(a, "\\u0100")); } +test "a truncated or malformed escape is an encoding error" { + const gpa = std.testing.allocator; + for ([_][]const u8{ "abc\\", "\\u12", "\\ud83d\\ude0", "\\u+123", "\\x0041" }) |raw| { + try std.testing.expectError(error.Encoding, decodeCache(gpa, raw)); + try std.testing.expectError(error.Encoding, decodeWire(gpa, raw)); + } + const obj: Value = .{ .object = &.{.{ .key = "a\\", .value = .null }} }; + try std.testing.expect(obj.get("a") == null); +} + test "parse objects, duplicates and nesting" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); diff --git a/src/logs.zig b/src/logs.zig index 6cdd7a8..e388aed 100644 --- a/src/logs.zig +++ b/src/logs.zig @@ -26,7 +26,18 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip const gz_name = try gpa.print("{s}.gz", .{name}); defer gpa.free(gz_name); - const dst = try sys.openat(dir, gz_name, .{ + // Refuse an existing archive before creating anything, as gzip does. + // The bytes then go to a temporary name, so a crash or a short write + // cannot leave a partial `name.gz` that a later run skips. + if (fsx.statAt(dir, gz_name, false)) |_| return error.Exists else |err| switch (err) { + error.NotFound => {}, + else => |e| return e, + } + var rnd: [4]u8 = undefined; + _ = try sys.call(linux.getrandom, .{ &rnd, rnd.len, 0 }); + const tmp_name = try gpa.print(".{s}.{d}.{x:0>8}.tmp", .{ gz_name, linux.getpid(), std.mem.readInt(u32, &rnd, .little) }); + defer gpa.free(tmp_name); + const dst = try sys.openat(dir, tmp_name, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, @@ -35,7 +46,7 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip var done = false; defer { sys.close(dst); - if (!done) sys.unlinkat(dir, gz_name, 0) catch {}; + if (!done) sys.unlinkat(dir, tmp_name, 0) catch {}; } const file: Io.File = .{ .handle = dst, .flags = .{ .nonblocking = false } }; @@ -81,8 +92,17 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip try sys.fchmod(dst, @intCast(src.stat.perm())); const times = [2]linux.timespec{ src.stat.atime, src.stat.mtime }; _ = try sys.call(linux.futimens, .{ dst, × }); - try sys.unlinkat(dir, name, 0); + try sys.fsync(dst); + try sys.renameat(dir, tmp_name, dir, gz_name); + sys.unlinkat(dir, name, 0) catch |err| { + // The archive is complete, but the original is still there. Drop + // the archive so a retry compresses the original instead of + // skipping an existing .gz and leaving both. + sys.unlinkat(dir, gz_name, 0) catch {}; + return err; + }; done = true; + try sys.fsync(dir); } /// The lines of a `.licensed-logs` list with every line found in `renames` @@ -110,6 +130,76 @@ pub fn renameInList(gpa: Allocator, content: []const u8, renames: *const std.Str return try out.toOwnedSlice(gpa); } +fn archiveIntact(path: [*:0]const u8) bool { + const pid: isize = @bitCast(linux.fork()); + if (pid == 0) { + const argv = [_:null]?[*:0]const u8{ "gzip", "-t", path, null }; + const envp = [_:null]?[*:0]const u8{}; + _ = linux.execve("/usr/bin/gzip", &argv, &envp); + linux.exit_group(127); + } + var status: u32 = 0; + _ = linux.wait4(@intCast(pid), @ptrCast(&status), 0, null); + return linux.W.IFEXITED(status) and linux.W.EXITSTATUS(status) == 0; +} + +test "gzip does not publish a partial archive under the final name" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const base = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/zig-common-gzip-{d}", .{ here, linux.getpid() }); + defer gpa.free(base); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(fsx.cwd, (try sys.z(&zb, base)).ptr, 0o700); + const dir = try fsx.openDir(fsx.cwd, base, false); + defer { + fsx.deleteTree(gpa, fsx.cwd, base) catch {}; + sys.close(dir); + } + const src = try sys.openat(dir, "a.log", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + const ur = try sys.openat(fsx.cwd, "/dev/urandom", .{}, 0); + defer sys.close(ur); + var buf: [64 * 1024]u8 = undefined; + var left: usize = 6 << 20; + while (left > 0) { + const n = try sys.read(ur, buf[0..@min(buf.len, left)]); + if (n == 0) return error.Unexpected; + try sys.writeAll(src, buf[0..n]); + left -= n; + } + sys.close(src); + const gz = try std.fmt.allocPrint(gpa, "{s}/a.log.gz", .{base}); + defer gpa.free(gz); + var gz_z: [sys.path_max:0]u8 = undefined; + const gz_sent = try sys.z(&gz_z, gz); + + const Watch = struct { + var stop = std.atomic.Value(bool).init(false); + var partial = std.atomic.Value(bool).init(false); + var path: [*:0]const u8 = undefined; + fn run() void { + while (!stop.load(.acquire)) { + if (fsx.statAt(fsx.cwd, std.mem.span(path), false)) |_| { + if (!archiveIntact(path)) partial.store(true, .release); + } else |_| {} + var ts: linux.timespec = .{ .sec = 0, .nsec = 2 * std.time.ns_per_ms }; + _ = linux.nanosleep(&ts, null); + } + } + }; + Watch.path = gz_sent.ptr; + Watch.stop.store(false, .release); + Watch.partial.store(false, .release); + const watch = try std.Thread.spawn(.{}, Watch.run, .{}); + const io = std.Io.Threaded.global_single_threaded.io(); + try gzipInPlace(gpa, io, dir, "a.log"); + Watch.stop.store(true, .release); + watch.join(); + try std.testing.expect(!Watch.partial.load(.acquire)); + try std.testing.expect(archiveIntact(gz_sent.ptr)); + try std.testing.expectError(error.NotFound, fsx.statAt(dir, "a.log", false)); +} + test "renameInList" { const gpa = std.testing.allocator; var m: std.StringHashMapUnmanaged([]const u8) = .empty; diff --git a/src/proc.zig b/src/proc.zig index 8fb12e5..a9b40e0 100644 --- a/src/proc.zig +++ b/src/proc.zig @@ -130,9 +130,18 @@ fn childExec( _ = linux.sigaction(@fromBackingInt(@intCast(s)), &act, null); } _ = linux.sigprocmask(linux.SIG.SETMASK, &empty, null); - if (linux.errno(linux.dup2(in_fd, 0)) != .SUCCESS) linux.exit_group(127); - if (linux.errno(linux.dup2(out_fd, 1)) != .SUCCESS) linux.exit_group(127); - if (linux.errno(linux.dup2(err_fd, 2)) != .SUCCESS) linux.exit_group(127); + // A source at 0, 1 or 2 (the caller started with that descriptor + // closed) moves above 2 first: a dup2 onto it would clobber it, and a + // dup2 onto itself keeps its close-on-exec flag. + var src = [3]sys.fd_t{ in_fd, out_fd, err_fd }; + for (&src) |*fd| if (fd.* < 3) { + const moved = linux.fcntl(fd.*, linux.F.DUPFD, 3); + if (linux.errno(moved) != .SUCCESS) linux.exit_group(127); + fd.* = @intCast(moved); + }; + for (src, 0..) |fd, target| { + if (linux.errno(linux.dup2(fd, @intCast(target))) != .SUCCESS) linux.exit_group(127); + } const keep_fd = exec_err_fd; if (keep_fd > 3) _ = linux.close_range(3, @intCast(keep_fd - 1), .{ .UNSHARE = false, .CLOEXEC = false }); _ = linux.close_range(if (keep_fd >= 3) @intCast(keep_fd + 1) else 3, std.math.maxInt(linux.fd_t), .{ .UNSHARE = false, .CLOEXEC = false }); @@ -168,16 +177,12 @@ fn exitedWithin(pid: linux.pid_t, ms: ?i64) bool { } } -fn reap(pid: linux.pid_t) Term { - while (true) { - var status: i32 = 0; - const rc = linux.wait4(pid, &status, 0, null); - switch (linux.errno(rc)) { - .SUCCESS => return decode(@bitCast(status)), - .INTR => continue, - else => return .{ .signaled = 9 }, - } - } +/// Reap the child. With SIGCHLD ignored (SIG_IGN or SA_NOCLDWAIT) the +/// kernel has reaped it and its status is gone: `error.NoChild`. +fn reap(pid: linux.pid_t) sys.Error!Term { + var status: i32 = 0; + _ = try sys.call(linux.wait4, .{ pid, &status, 0, null }); + return decode(@bitCast(status)); } /// True while a live (not zombie) process is in process group `pgid`. A @@ -189,6 +194,9 @@ pub fn groupAlive(pgid: linux.pid_t) bool { var dents: [16 * 1024]u8 align(8) = undefined; while (true) { const n = linux.getdents64(dir, &dents, dents.len); + // EINTR is not a failed scan. Treating it as "nobody is alive" + // would let waitGroupGone return while the group is still running. + if (linux.errno(n) == .INTR) continue; if (linux.errno(n) != .SUCCESS or n == 0) return false; var pos: usize = 0; while (pos < n) { @@ -267,6 +275,8 @@ fn keep(i: usize, bytes: []const u8, out: *std.ArrayList(u8), err_buf: *std.Arra } /// Run `exe` (an absolute path) with `argv` (argv[0] included) and `envp`. +/// The caller must not ignore SIGCHLD (SIG_IGN or SA_NOCLDWAIT): the kernel +/// then reaps the child and its status is lost, which is `error.NoChild`. pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp, opts: Options) Error!Result { if (exe.len == 0 or exe[0] != '/') return error.NotAbsolute; @@ -358,7 +368,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp }; if (got == 4) { if (opts.pgid_out) |o| o.store(0, .seq_cst); - _ = reap(pid); + _ = reap(pid) catch null; _ = try sys.result(@bitCast(-@as(isize, @intCast(e)))); return error.Unexpected; } @@ -481,7 +491,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp if (opts.new_session) kill(pid, true, .KILL); if (opts.new_session and opts.wait_for_group) waitGroupGone(pid); if (opts.pgid_out) |o| o.store(0, .seq_cst); - _ = reap(pid); + _ = reap(pid) catch null; return err; } // Members of the child's group outliving it are not ours to keep; the @@ -489,7 +499,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp if (opts.new_session) kill(pid, true, .KILL); if (opts.new_session and opts.wait_for_group) waitGroupGone(pid); if (opts.pgid_out) |o| o.store(0, .seq_cst); - const term = reap(pid); + const term = try reap(pid); const o = try out.toOwnedSlice(gpa); errdefer gpa.free(o); return .{ .term = term, .stdout = o, .stderr = try err_buf.toOwnedSlice(gpa) }; @@ -532,7 +542,91 @@ test "a set cancel flag kills the child" { try std.testing.expectError(error.Canceled, run(gpa, "/bin/sh", &.{ "sh", "-c", "sleep 30" }, no_env, .{ .cancel = &flag })); } +fn onUsr1Proc(_: linux.SIG) callconv(.c) void {} + +test "groupAlive stays true for a live group while signals arrive" { + const handler: linux.Sigaction = .{ .handler = .{ .handler = &onUsr1Proc }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.USR1, &handler, &prev); + defer _ = linux.sigaction(.USR1, &prev, null); + const Stop = struct { + var go = std.atomic.Value(bool).init(true); + fn buzz() void { + var blocked = linux.sigemptyset(); + linux.sigaddset(&blocked, .USR1); + _ = linux.sigprocmask(linux.SIG.BLOCK, &blocked, null); + while (go.load(.acquire)) _ = linux.kill(linux.getpid(), .USR1); + } + }; + Stop.go.store(true, .release); + const buzz = try std.Thread.spawn(.{}, Stop.buzz, .{}); + defer { + Stop.go.store(false, .release); + buzz.join(); + } + const pgid: linux.pid_t = @intCast(linux.getpgid(0)); + for (0..50) |_| try std.testing.expect(groupAlive(pgid)); +} + test "run refuses a relative program" { const gpa = std.testing.allocator; try std.testing.expectError(error.NotAbsolute, run(gpa, "sh", &.{"sh"}, no_env, .{})); } + +/// Run `body` in a forked child, which may close the descriptors the test +/// runner talks over, and expect it to exit 0. +fn expectChildPasses(comptime body: fn () u8) !void { + const pid: isize = @bitCast(linux.fork()); + if (pid < 0) return error.Unexpected; + if (pid == 0) linux.exit_group(body()); + var status: i32 = 0; + while (linux.errno(linux.wait4(@intCast(pid), &status, 0, null)) == .INTR) {} + try std.testing.expectEqual(Term{ .exited = 0 }, decode(@bitCast(status))); +} + +/// 0 when `script` under /bin/sh prints `want`, 1 when it prints something +/// else, 2 when it cannot be run. +fn shPrints(script: []const u8, stdin: ?[]const u8, want: []const u8) u8 { + const r = run(std.heap.page_allocator, "/bin/sh", &.{ "sh", "-c", script }, no_env, .{ .stdin = stdin }) catch return 2; + if (std.mem.eql(u8, r.stdout, want)) return 0; + std.debug.print("stdout \"{s}\", want \"{s}\"\n", .{ r.stdout, want }); + return 1; +} + +test "a closed stdout at start does not send the child's stderr to its stdout" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(1); + return shPrints("echo out; echo err >&2", null, "out\n"); + } + }.body); +} + +test "a closed stdin at start leaves the child's stdin and stderr on their files" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(0); + const r = shPrints("readlink /proc/self/fd/0 || echo closed", null, "/dev/null\n"); + if (r != 0) return r; + return shPrints("cat; readlink /proc/self/fd/2", "in\n", "in\n/dev/null\n"); + } + }.body); +} + +test "a closed stderr at start leaves the child's stderr on /dev/null" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(2); + return shPrints("readlink /proc/self/fd/2 || echo closed", null, "/dev/null\n"); + } + }.body); +} + +test "a status lost to an ignored SIGCHLD is an error, not a kill by signal 9" { + const ignore: linux.Sigaction = .{ .handler = .{ .handler = linux.SIG.IGN }, .mask = linux.sigemptyset(), .flags = 0 }; + var old: linux.Sigaction = undefined; + _ = linux.sigaction(.CHLD, &ignore, &old); + defer _ = linux.sigaction(.CHLD, &old, null); + try std.testing.expectError(error.NoChild, run(std.testing.allocator, "/bin/sh", &.{ "sh", "-c", "exit 3" }, no_env, .{})); + try std.testing.expectEqual(linux.E.CHILD, sys.last_errno); +} diff --git a/src/pytext.zig b/src/pytext.zig index 6e3bc35..c3c28d9 100644 --- a/src/pytext.zig +++ b/src/pytext.zig @@ -118,6 +118,7 @@ pub fn pyStrip(s: []const u8) []const u8 { /// str.split() with no separator: the runs between Unicode whitespace. pub fn pySplit(arena: Allocator, s: []const u8) Allocator.Error![]const []const u8 { var out: std.ArrayList([]const u8) = .empty; + errdefer out.deinit(arena); var i: usize = 0; var start: ?usize = null; while (i < s.len) { @@ -130,7 +131,7 @@ pub fn pySplit(arena: Allocator, s: []const u8) Allocator.Error![]const []const i += ch.len; } if (start) |st| try out.append(arena, s[st..]); - return out.items; + return try out.toOwnedSlice(arena); } /// shlex.quote: unchanged when every byte is in [A-Za-z0-9_@%+=:,./-] and @@ -167,6 +168,16 @@ test "pyStrip and pySplit use Unicode whitespace" { try std.testing.expectEqualStrings("waitpkg", parts[1]); } +fn splitOwned(a: std.mem.Allocator) !void { + const parts = try pySplit(a, "a b c"); + defer a.free(parts); + try std.testing.expectEqual(@as(usize, 3), parts.len); +} + +test "pySplit frees its list when an allocation fails" { + try std.testing.checkAllAllocationFailures(std.testing.allocator, splitOwned, .{}); +} + test "shellQuote" { const gpa = std.testing.allocator; const cases = [_][2][]const u8{ diff --git a/src/state.zig b/src/state.zig index d175b1e..b095941 100644 --- a/src/state.zig +++ b/src/state.zig @@ -43,12 +43,15 @@ pub fn openPrivate(dir: fsx.fd_t, name: []const u8, create: bool) Error!?fsx.fd_ pub const LockError = Error || error{TimedOut}; /// flock(LOCK_EX) on `fd`, giving up after `timeout_ms` when one is given. +/// The deadline is read from the monotonic clock. A signal that cuts a +/// sleep short does not consume the whole wait: the next check looks at +/// the clock again. pub fn lockExclusive(fd: fsx.fd_t, timeout_ms: ?u32) LockError!void { - const ms = timeout_ms orelse { + const deadline: ?i64 = if (timeout_ms) |ms| nowMs() + ms else null; + if (deadline == null) { _ = try sys.call(linux.flock, .{ fd, std.posix.LOCK.EX }); return; - }; - var left: i64 = ms; + } while (true) { const rc = linux.flock(fd, std.posix.LOCK.EX | std.posix.LOCK.NB); switch (linux.errno(rc)) { @@ -56,11 +59,10 @@ pub fn lockExclusive(fd: fsx.fd_t, timeout_ms: ?u32) LockError!void { .AGAIN, .INTR => {}, else => return error.Unexpected, } + const left = deadline.? - nowMs(); if (left <= 0) return error.TimedOut; - const step: i64 = @min(left, 20); - var ts: linux.timespec = .{ .sec = 0, .nsec = @intCast(step * std.time.ns_per_ms) }; + var ts: linux.timespec = .{ .sec = 0, .nsec = @intCast(@min(left, 20) * std.time.ns_per_ms) }; _ = linux.nanosleep(&ts, null); - left -= step; } } @@ -124,6 +126,60 @@ test "a file that is not private is not read" { try lockExclusive(lk, 100); } +fn onUsr1(_: linux.SIG) callconv(.c) void {} + +test "lockExclusive waits out the timeout while signals arrive" { + const gpa = std.testing.allocator; + var pb: [sys.path_max]u8 = undefined; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-locktime-{d}", .{ cwd, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(fsx.cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try fsx.openDir(fsx.cwd, base, false); + defer { + fsx.deleteTree(gpa, fsx.cwd, base) catch {}; + sys.close(d); + } + const held = try sys.openat(d, "L", .{ .ACCMODE = .RDWR, .CREAT = true }, 0o600); + defer sys.close(held); + const waiter = try sys.openat(d, "L", .{ .ACCMODE = .RDWR }, 0); + defer sys.close(waiter); + _ = try sys.call(linux.flock, .{ held, std.posix.LOCK.EX }); + defer _ = linux.flock(held, std.posix.LOCK.UN); + + const handler: linux.Sigaction = .{ .handler = .{ .handler = &onUsr1 }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.USR1, &handler, &prev); + defer _ = linux.sigaction(.USR1, &prev, null); + const Stop = struct { + var go = std.atomic.Value(bool).init(true); + fn buzz() void { + // This thread must not catch SIGUSR1, or the waiter's sleep is + // never the one the signal interrupts. + var blocked = linux.sigemptyset(); + linux.sigaddset(&blocked, .USR1); + _ = linux.sigprocmask(linux.SIG.BLOCK, &blocked, null); + while (go.load(.acquire)) { + _ = linux.kill(linux.getpid(), .USR1); + var ts: linux.timespec = .{ .sec = 0, .nsec = std.time.ns_per_ms }; + _ = linux.nanosleep(&ts, null); + } + } + }; + Stop.go.store(true, .release); + const buzz = try std.Thread.spawn(.{}, Stop.buzz, .{}); + var t0: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &t0); + try std.testing.expectError(error.TimedOut, lockExclusive(waiter, 400)); + var t1: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &t1); + Stop.go.store(false, .release); + buzz.join(); + const ms = (t1.sec - t0.sec) * 1000 + @divTrunc(t1.nsec - t0.nsec, 1_000_000); + try std.testing.expect(ms >= 300 and ms < 5000); +} + // ---- the state-root lock: a flock gate and a POSIX record lock ------------- /// A state root's lock. The flock gate excludes processes on one kernel; the @@ -136,8 +192,9 @@ test "a file that is not private is not read" { /// Others may be able to write the state root. The root is opened without /// following a symbolic link as its last component, both files through its /// descriptor without following one at their names, and each is trusted -/// only as a regular file with one link: a FIFO planted at a name is never -/// opened, and the file behind a planted hard link is never truncated. +/// only as a regular file of this user with one link: a FIFO planted at a +/// name is never opened, and the file behind a planted hard link, or a file +/// another user owns, is never truncated. pub const RootLock = struct { gate: fsx.fd_t, posix: fsx.fd_t, @@ -148,9 +205,10 @@ pub const RootLock = struct { pub const AcquireError = Error || error{ Busy, - /// STATE_ROOT/.lock.posix is not a regular file with one link. + /// STATE_ROOT/.lock.posix is not a regular file of this user with + /// one link. NotPrivate, - /// STATE_ROOT/.lock is not a regular file with one link. + /// STATE_ROOT/.lock is not a regular file of this user with one link. GateNotPrivate, OutOfMemory, }; @@ -207,10 +265,10 @@ pub const RootLock = struct { /// STATE_ROOT is created when absent and opened O_DIRECTORY|O_NOFOLLOW; /// a symbolic link there is NotDir. Each lock file is opened relative to /// that descriptor with O_NOFOLLOW, created mode 0600, only when its name - /// is absent or a regular file with one link, and its descriptor must be - /// that file. The gate is checked again once both locks are held, right - /// before it is truncated. Allocates nothing; `gpa` stays for source - /// compatibility. + /// is absent or a regular file of this user with one link, and its + /// descriptor must be that file. The gate is checked again once both + /// locks are held, right before it is truncated. Allocates nothing; + /// `gpa` stays for source compatibility. pub fn acquire(gpa: Allocator, state_root: []const u8, timeout_ms: ?u32) AcquireError!RootLock { _ = gpa; const atomic = @import("atomic.zig"); @@ -234,12 +292,12 @@ pub const RootLock = struct { } /// `dir/name` opened read-write through `open`; null unless the name is - /// absent or a regular file with one link before the open, so a FIFO, - /// device or symbolic link there is never opened, and the descriptor is - /// that file after it. + /// absent or a regular file of this user with one link before the open, + /// so a FIFO, device or symbolic link there is never opened, and the + /// descriptor is that file after it. fn openLockFile(dir: fsx.fd_t, name: [:0]const u8, open: *const fn (dir: fsx.fd_t, name: [*:0]const u8) usize) Error!?fsx.fd_t { if (fsx.statAt(dir, name, false)) |st| { - if (!st.isReg() or st.nlink != 1) return null; + if (!st.isReg() or st.nlink != 1 or st.uid != linux.geteuid()) return null; } else |err| switch (err) { error.NotFound => {}, else => |e| return e, @@ -254,13 +312,13 @@ pub const RootLock = struct { return null; } - /// Whether `fd` is a regular file whose one link is `dir/name`. The name - /// is looked up as well as the descriptor: a planted hard link unlinked - /// while the lock is awaited leaves the descriptor on the other name's - /// file, with one link. + /// Whether `fd` is a regular file of this user whose one link is + /// `dir/name`. The name is looked up as well as the descriptor: a + /// planted hard link unlinked while the lock is awaited leaves the + /// descriptor on the other name's file, with one link. fn isSoleLink(dir: fsx.fd_t, name: []const u8, fd: fsx.fd_t) Error!bool { const st = try fsx.fstat(fd); - if (!st.isReg() or st.nlink != 1) return false; + if (!st.isReg() or st.nlink != 1 or st.uid != linux.geteuid()) return false; const at = fsx.statAt(dir, name, false) catch |err| switch (err) { error.NotFound => return false, else => |e| return e, diff --git a/src/sys.zig b/src/sys.zig index fea0e2a..ec130c7 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -363,4 +363,18 @@ test "describe prints the errno the kernel gave, as Python's OSError does" { // An error that the last errno does not map to gets its usual errno. try std.testing.expectEqualStrings("[Errno 2] No such file or directory", describe(error.NotFound)); try std.testing.expectEqual(@as(?Error, null), asError(error.OutOfMemory)); + try std.testing.expectEqualStrings("[Errno 133] Memory page has hardware error", describeErrno(.HWPOISON)); + try std.testing.expectEqualStrings("[Errno 134] Unknown error 134", describeErrno(@fromBackingInt(@intCast(134)))); + const compat_fd = try openat(linux.AT.FDCWD, "COMPAT.md", .{}, 0); + defer close(compat_fd); + var compat: std.ArrayList(u8) = .empty; + defer compat.deinit(std.testing.allocator); + var chunk: [512]u8 = undefined; + while (true) { + const n = try read(compat_fd, &chunk); + if (n == 0) break; + try compat.appendSlice(std.testing.allocator, chunk[0..n]); + } + try std.testing.expect(std.mem.indexOf(u8, compat.items, "errno 133") != null); + try std.testing.expect(std.mem.indexOf(u8, compat.items, "errno 134,") == null); } diff --git a/towncrier.toml b/towncrier.toml new file mode 100644 index 0000000..19a34e1 --- /dev/null +++ b/towncrier.toml @@ -0,0 +1,37 @@ +[tool.towncrier] +name = "zig-common" +directory = "changelog.d" +filename = "CHANGELOG.md" +start_string = "\n" +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/stackhands/zig-common/pull/{issue})" +underlines = ["", "", ""] +# Setting ignore, even to nothing, is what makes `towncrier build` fail a +# fragment whose name has no configured type. Unset, that file is skipped +# and the draft still passes. .gitkeep stays ignored either way. +ignore = [] + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Fixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Miscellaneous" +showcontent = true