diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b28c1..9055909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,6 @@ # Changelog -## Unreleased - -* `state.RootLock.acquire` no longer truncates the file behind a hard link - planted at `.lock`. It opens the state root `O_DIRECTORY|O_NOFOLLOW` and - both lock files relative to it with `O_NOFOLLOW`, creating them mode - 0600; a lock file name that is not a regular file with one link is - refused before it is opened, so a FIFO or symbolic link there is never - followed; and once both locks are held, the gate must still be a regular - file whose one link is `.lock` before it is truncated. -* A refused gate is the new `RootLock.AcquireError.GateNotPrivate` - (`NotPrivate` stays the sidecar's), a symbolic link as the state root is - `NotDir`, and `LockOps.open_sidecar` takes the state root's descriptor - and a name. + ## 0.8.1 diff --git a/COMPAT.md b/COMPAT.md index d3d1875..682c753 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -21,6 +21,8 @@ breaks a tool shows before the tool moves to it. `sys.describe` prints glibc's `strerror` text, as Python's `os.strerror` does, for every errno the Zig standard library's `linux.E` names. An errno it does -not name prints `[Errno N] Unknown error N`; on x86_64 that is errno 134, -which glibc calls "Memory page has hardware error". +not name prints `[Errno N] Unknown error N`. On x86_64, errno 133 is +`HWPOISON`, `[Errno 133] Memory page has hardware error`, and errno 134 is +not a named errno: both glibc and this library print `[Errno 134] Unknown +error 134`. diff --git a/README.md b/README.md index 4d0bc54..63ffeb4 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,11 @@ the entry and its hash. | module | what it does | |---|---| | `sys` | `std.os.linux` calls with EINTR retried and errno mapped to a Zig error set | -| `fsx` | open without following a final symbolic link, checks on the descriptor (`fstat`), cp-style copy, line append, atomic replace keeping owner, mode and access ACLs, directory iteration, `rm -rf` that never follows links, `realpath` through `/proc/self/fd` | +| `fsx` | open without following a final symbolic link, checks on the descriptor (`fstat`), a copy that renames into place so a short write does not destroy the destination, line append, atomic replace keeping owner, mode and access ACLs, directory iteration, `rm -rf` that never follows links, `realpath` through `/proc/self/fd` | | `proc` | run a program by absolute path with an empty signal mask, default handlers and no descriptors above 2, by default in its own session; stdin fed and stdout and stderr read at once, each with a cap; an optional deadline, SIGTERM, a grace period, SIGKILL to the process group; a cancel flag a signal handler can set; group members left after the child exits are killed before it is reaped; `which` looks a name up in an explicit PATH | | `logs` | `gzip -9` in place with gzip's header (name, mtime, OS byte), owner, mode and times kept; rewrite of `.licensed-logs` entries | | `env` | the environment as raw bytes with getenv's first-entry rule, duplicate names found, a child environment built with each name once, shell name check | -| `state` | private files in a shared directory: trusted only when the open file is a regular file of this user with one link and no group or other bits, a lock trusted the same way, atomic 0600 writes; `RootLock`, a flock gate plus a POSIX record lock on a sidecar, with a deadline | +| `state` | private files in a shared directory: trusted only when the open file is a regular file of this user with one link and no group or other bits, a lock trusted the same way, atomic 0600 writes; `RootLock`, a flock gate plus a POSIX record lock on a sidecar, with a deadline read from the clock. The gate and the sidecar must each be a regular file of this user with one link | | `json` | a JSON reader with Python's two string encodings (surrogateescape text, one byte per code point), Python values, and `json.dumps(indent=N, sort_keys=True)` byte for byte, float repr included | | `pytext` | byte strings as Python text: the surrogateescape character, the 'replace' decoder, str.isspace, strip and split, shlex.quote | | `atomic` | `atomic_write`: replace a file only after its bytes, owner, mode and every ACL form (GPFS NFSv4, NFSv4, POSIX access) are durable and read back; fault points before write, fsync and rename | diff --git a/changelog.d/+append-short-write.bugfix.md b/changelog.d/+append-short-write.bugfix.md new file mode 100644 index 0000000..d50a541 --- /dev/null +++ b/changelog.d/+append-short-write.bugfix.md @@ -0,0 +1 @@ +`fsx.appendLine` rolls a short write back, so a full disk does not leave a partial line. diff --git a/changelog.d/+atomic-eintr.bugfix.md b/changelog.d/+atomic-eintr.bugfix.md new file mode 100644 index 0000000..075b0b9 --- /dev/null +++ b/changelog.d/+atomic-eintr.bugfix.md @@ -0,0 +1 @@ +`atomic` retries an ACL read interrupted by a signal, instead of refusing the replacement. diff --git a/changelog.d/+atomic-errno-text.bugfix.md b/changelog.d/+atomic-errno-text.bugfix.md new file mode 100644 index 0000000..7f3b2cb --- /dev/null +++ b/changelog.d/+atomic-errno-text.bugfix.md @@ -0,0 +1 @@ +`atomic` words a failed ACL read with `sys.describeErrno`, glibc's text for every errno Linux names (`[Errno 116] Stale file handle`), instead of its own six-entry table that printed the errno's name for the rest. diff --git a/changelog.d/+atomic-free.bugfix.md b/changelog.d/+atomic-free.bugfix.md new file mode 100644 index 0000000..058ab86 --- /dev/null +++ b/changelog.d/+atomic-free.bugfix.md @@ -0,0 +1 @@ +`atomic.atomicWrite` frees everything it allocates from the caller's allocator before it returns (the temporary path, the step names, the ACLs it reads and the refusals it wraps), which only an arena used to hide. A refusal's `why.msg` is the caller's, freed by the new `Why.deinit`; the ACLs `readAcls` and `readMetadata` return are freed by the new `Acls.deinit` and `Metadata.deinit`. diff --git a/changelog.d/+atomic-leak.bugfix.md b/changelog.d/+atomic-leak.bugfix.md new file mode 100644 index 0000000..2640931 --- /dev/null +++ b/changelog.d/+atomic-leak.bugfix.md @@ -0,0 +1 @@ +`atomicWrite` frees the path, the step names and the spare ACL read it allocates. diff --git a/changelog.d/+copy-offset.bugfix.md b/changelog.d/+copy-offset.bugfix.md new file mode 100644 index 0000000..1231fdb --- /dev/null +++ b/changelog.d/+copy-offset.bugfix.md @@ -0,0 +1 @@ +`fsx.copyTo` copies the whole source, whatever the descriptor's offset, and leaves the offset where it was, as its doc says; it copied from the offset to the end and left the offset there. diff --git a/changelog.d/+copy-short-write.bugfix.md b/changelog.d/+copy-short-write.bugfix.md new file mode 100644 index 0000000..da6671e --- /dev/null +++ b/changelog.d/+copy-short-write.bugfix.md @@ -0,0 +1 @@ +`fsx.copyTo` renames the new bytes into place, so a short write leaves the existing file unchanged. diff --git a/changelog.d/+errno-133.doc.md b/changelog.d/+errno-133.doc.md new file mode 100644 index 0000000..5ca82eb --- /dev/null +++ b/changelog.d/+errno-133.doc.md @@ -0,0 +1 @@ +`COMPAT.md` names errno 133 as the hardware-error text. Errno 134 is unnamed, which is what glibc prints too. diff --git a/changelog.d/+group-alive-eintr.bugfix.md b/changelog.d/+group-alive-eintr.bugfix.md new file mode 100644 index 0000000..5503de3 --- /dev/null +++ b/changelog.d/+group-alive-eintr.bugfix.md @@ -0,0 +1 @@ +`proc.groupAlive` retries a `/proc` read interrupted by a signal, instead of reporting that the process group is gone. diff --git a/changelog.d/+gzip-partial.bugfix.md b/changelog.d/+gzip-partial.bugfix.md new file mode 100644 index 0000000..4833319 --- /dev/null +++ b/changelog.d/+gzip-partial.bugfix.md @@ -0,0 +1 @@ +`logs.gzipInPlace` writes the archive under a temporary name and renames it into place, so a crash does not leave a partial `.gz` at the final name. diff --git a/changelog.d/+json-docs.bugfix.md b/changelog.d/+json-docs.bugfix.md new file mode 100644 index 0000000..cbb5b65 --- /dev/null +++ b/changelog.d/+json-docs.bugfix.md @@ -0,0 +1 @@ +`json.parse` refuses arrays and objects nested more than 32 deep, now `json.max_depth`, whether or not the innermost one is empty; 33 nested empty containers used to parse. The limit is documented, and the module summary covers the parser, both codecs, `toPy` and `dumps`. diff --git a/changelog.d/+json-escape.bugfix.md b/changelog.d/+json-escape.bugfix.md new file mode 100644 index 0000000..006d50b --- /dev/null +++ b/changelog.d/+json-escape.bugfix.md @@ -0,0 +1 @@ +`json` returns an encoding error for a truncated or unknown escape, instead of reading past the string. diff --git a/changelog.d/+json-escapes.bugfix.md b/changelog.d/+json-escapes.bugfix.md new file mode 100644 index 0000000..52a810b --- /dev/null +++ b/changelog.d/+json-escapes.bugfix.md @@ -0,0 +1 @@ +`json.decodeCache` and `json.decodeWire` return `error.Encoding` for an escape that is cut off, unknown or a `\u` without four hex digits, where they read past the string or took `\u+123` and `\x0041`; what they allocated before the error is freed. `Value.get` does not read past a key that ends in a backslash. diff --git a/changelog.d/+json-int-range.bugfix.md b/changelog.d/+json-int-range.bugfix.md new file mode 100644 index 0000000..57b8b18 --- /dev/null +++ b/changelog.d/+json-int-range.bugfix.md @@ -0,0 +1 @@ +`json.toPy` returns `error.Encoding` for an integer that does not fit an `i64`, which `json.loads` keeps exact, instead of a rounded float; a number with a fraction or an exponent is still a float. diff --git a/changelog.d/+lock-gate.bugfix.md b/changelog.d/+lock-gate.bugfix.md new file mode 100644 index 0000000..f387f65 --- /dev/null +++ b/changelog.d/+lock-gate.bugfix.md @@ -0,0 +1 @@ +`state.RootLock` also refuses a gate or sidecar that another user owns, as `GateNotPrivate` and `NotPrivate`: each must be a regular file of this user with one link. diff --git a/changelog.d/+lock-timeout.bugfix.md b/changelog.d/+lock-timeout.bugfix.md new file mode 100644 index 0000000..8c5eb11 --- /dev/null +++ b/changelog.d/+lock-timeout.bugfix.md @@ -0,0 +1 @@ +`state.lockExclusive` measures its deadline on the monotonic clock, so a signal that cuts a sleep short does not expire the wait. diff --git a/changelog.d/+proc-std-fds.bugfix.md b/changelog.d/+proc-std-fds.bugfix.md new file mode 100644 index 0000000..c586839 --- /dev/null +++ b/changelog.d/+proc-std-fds.bugfix.md @@ -0,0 +1 @@ +`proc.run` gives the child /dev/null or its pipes on fds 0 to 2 when the caller started with one of them closed: the child's stderr no longer lands in the captured stdout, and its stdin or stderr is no longer closed at exec. diff --git a/changelog.d/+proc-wait-status.bugfix.md b/changelog.d/+proc-wait-status.bugfix.md new file mode 100644 index 0000000..ed26eda --- /dev/null +++ b/changelog.d/+proc-wait-status.bugfix.md @@ -0,0 +1 @@ +`proc.run` returns `error.NoChild` when the child's status is lost because the caller ignores SIGCHLD, instead of reporting a kill by signal 9. diff --git a/changelog.d/+pysplit-oom.bugfix.md b/changelog.d/+pysplit-oom.bugfix.md new file mode 100644 index 0000000..8a550c8 --- /dev/null +++ b/changelog.d/+pysplit-oom.bugfix.md @@ -0,0 +1 @@ +`pytext.pySplit` frees its list when an allocation fails, and the slice it returns is the whole allocation. diff --git a/changelog.d/+replace-gpfs-acl.bugfix.md b/changelog.d/+replace-gpfs-acl.bugfix.md new file mode 100644 index 0000000..4135ae2 --- /dev/null +++ b/changelog.d/+replace-gpfs-acl.bugfix.md @@ -0,0 +1 @@ +`fsx.replaceContents` copies a GPFS native NFSv4 ACL (`system.gpfs_nfs4_acl`), which it dropped, and, as `atomic` does, copies only the most native ACL form the old file has. GPFS's EINVAL for an NFSv4 form of a file with a POSIX ACL no longer fails the replacement. `fsx.acl_attributes` is the one list of forms, and `atomic` uses it. diff --git a/changelog.d/+rootlock-api.bugfix.md b/changelog.d/+rootlock-api.bugfix.md new file mode 100644 index 0000000..59e0f64 --- /dev/null +++ b/changelog.d/+rootlock-api.bugfix.md @@ -0,0 +1 @@ +A refused gate is the new `RootLock.AcquireError.GateNotPrivate` (`NotPrivate` stays the sidecar's), a symbolic link as the state root is `NotDir`, and `LockOps.open_sidecar` takes the state root's descriptor and a name. diff --git a/changelog.d/+rootlock-truncate.bugfix.md b/changelog.d/+rootlock-truncate.bugfix.md new file mode 100644 index 0000000..72d1dd9 --- /dev/null +++ b/changelog.d/+rootlock-truncate.bugfix.md @@ -0,0 +1 @@ +`state.RootLock.acquire` no longer truncates the file behind a hard link planted at `.lock`. It opens the state root `O_DIRECTORY|O_NOFOLLOW` and both lock files relative to it with `O_NOFOLLOW`, creating them mode 0600; a lock file name that is not a regular file with one link is refused before it is opened, so a FIFO or symbolic link there is never followed; and once both locks are held, the gate must still be a regular file whose one link is `.lock` before it is truncated. diff --git a/src/atomic.zig b/src/atomic.zig index 49dc641..c565f73 100644 --- a/src/atomic.zig +++ b/src/atomic.zig @@ -12,7 +12,7 @@ const fsx = @import("fsx.zig"); const Allocator = std.mem.Allocator; -pub const acl_attributes = [_][:0]const u8{ "system.gpfs_nfs4_acl", "system.nfs4_acl", "system.posix_acl_access" }; +pub const acl_attributes = fsx.acl_attributes; const posix_access = "system.posix_acl_access"; /// The calls atomicWrite makes on a file descriptor, as raw system-call @@ -87,15 +87,25 @@ pub fn describe(err: Error) []const u8 { }; } -/// Why a replacement was refused; filled in when an error is returned. +/// Why a replacement was refused; filled in when an error is returned. The +/// message is allocated from the allocator given with `why` and belongs to +/// the caller: `deinit` frees it. pub const Why = struct { msg: []const u8 = "", + + pub fn deinit(why: *Why, a: Allocator) void { + if (why.msg.len > 0 and why.msg.ptr != unallocated_refusal.ptr) a.free(why.msg); + why.* = .{}; + } }; pub const Error = error{ Refused, OutOfMemory } || sys.Error; +/// The message of a refusal that could not allocate its own. +const unallocated_refusal: []const u8 = "refused"; + fn refuse(a: Allocator, why: *Why, comptime f: []const u8, args: anytype) Error { - why.msg = a.print(f, args) catch "refused"; + why.msg = a.print(f, args) catch unallocated_refusal; return error.Refused; } @@ -110,9 +120,21 @@ pub const Acls = struct { } return true; } + + /// Free the values, allocated from `a` by the read that returned them. + pub fn deinit(acls: Acls, a: Allocator) void { + for (acls.present) |p| if (p) |v| a.free(v); + } }; -pub const Metadata = struct { info: fsx.Stat, acls: Acls }; +pub const Metadata = struct { + info: fsx.Stat, + acls: Acls, + + pub fn deinit(m: Metadata, a: Allocator) void { + m.acls.deinit(a); + } +}; /// (dev, ino, uid, gid, mode, ctime): what a concurrent change moves. fn identity(s: fsx.Stat) [7]i128 { @@ -147,24 +169,23 @@ fn listedAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error![acl_attributes.len]b return listed; } -fn errnoText(e: linux.E) []const u8 { - return switch (e) { - .ACCES => "Permission denied", - .PERM => "Operation not permitted", - .IO => "Input/output error", - .INVAL => "Invalid argument", - .NODATA => "No data available", - .OPNOTSUPP => "Operation not supported", - else => @tagName(e), - }; +/// One `fgetxattr`, retrying when a signal interrupts it. `op` cannot be +/// used here: the errno itself decides whether a missing attribute is an +/// error, and `op` collapses errnos into `sys.Error`. +fn xattrGet(fd: sys.fd_t, name: [:0]const u8, buf: []u8) usize { + while (true) { + const rc = ops.fgetxattr(fd, name, buf); + if (linux.errno(rc) != .INTR) return rc; + } } fn readAllAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { const listed = try listedAcls(a, fd, why); var acls: Acls = .{}; + errdefer acls.deinit(a); var buf: [64 * 1024]u8 = undefined; for (acl_attributes, 0..) |name, i| { - const rc = ops.fgetxattr(fd, name, &buf); + const rc = xattrGet(fd, name, &buf); const e = linux.errno(rc); if (e == .SUCCESS) { acls.present[i] = try a.dupe(u8, buf[0..rc]); @@ -172,35 +193,41 @@ fn readAllAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { } if (listed[i]) { if (e == .NODATA) return refuse(a, why, "{s} was listed but is no longer readable", .{name}); - return refuse(a, why, "cannot read {s}: [Errno {d}] {s}", .{ name, @backingInt(e), errnoText(e) }); + return refuse(a, why, "cannot read {s}: {s}", .{ name, sys.describeErrno(e) }); } if (e == .NODATA or e == .OPNOTSUPP) continue; // An unlisted NFSv4-family name may answer EINVAL where the same // inspection lists a POSIX access ACL, which is read here. if (e == .INVAL and isNfs4Family(name) and listed[2]) continue; - return refuse(a, why, "cannot read {s}: [Errno {d}] {s}", .{ name, @backingInt(e), errnoText(e) }); + return refuse(a, why, "cannot read {s}: {s}", .{ name, sys.describeErrno(e) }); } if (listed[2] and acls.present[2] == null) return refuse(a, why, "{s} was listed but is no longer readable", .{posix_access}); return acls; } /// Every listed or virtual ACL of an open regular file, read twice; a change -/// between the reads, or of the file's identity, is refused. +/// between the reads, or of the file's identity, is refused. The values are +/// allocated from `a`; `Acls.deinit` frees them. pub fn readAcls(a: Allocator, fd: sys.fd_t, why: *Why) Error!Acls { const before = try fsx.fstat(fd); const acls = try readAllAcls(a, fd, why); + errdefer acls.deinit(a); const again = try readAllAcls(a, fd, why); + defer again.deinit(a); if (!acls.eql(again) or !sameIdentity(try fsx.fstat(fd), before)) return refuse(a, why, "access metadata changed during inspection", .{}); return acls; } +/// The stat and ACLs of the regular file at `path`; `Metadata.deinit` frees +/// the ACLs, allocated from `a`. pub fn readMetadata(a: Allocator, path: []const u8, why: *Why) Error!Metadata { const fd = try sys.openat(fsx.cwd, path, .{ .NOFOLLOW = true, .NONBLOCK = true }, 0); defer sys.close(fd); const info = try fsx.fstat(fd); if (!info.isReg()) return refuse(a, why, "metadata source is not a regular file: {s}", .{path}); const acls = try readAcls(a, fd, why); + errdefer acls.deinit(a); if (!sameIdentity(try fsx.fstat(fd), info)) return refuse(a, why, "access metadata changed during inspection", .{}); return .{ .info = info, .acls = acls }; } @@ -225,13 +252,16 @@ fn restoreMetadata(a: Allocator, fd: sys.fd_t, metadata: ?Metadata, mode: u32, w break; }; if (primary) |i| { - if (!(try readAcls(a, fd, why)).eql(m.acls)) { + const now = try readAcls(a, fd, why); + defer now.deinit(a); + if (!now.eql(m.acls)) { const v = m.acls.present[i].?; _ = try op(ops.fsetxattr, .{ fd, acl_attributes[i], v }); } } for (acl_attributes) |_| { const present = try readAcls(a, fd, why); + defer present.deinit(a); var extra: ?usize = null; for (present.present, m.acls.present, 0..) |p, want, i| if (p != null and want == null) { extra = i; @@ -243,7 +273,9 @@ fn restoreMetadata(a: Allocator, fd: sys.fd_t, metadata: ?Metadata, mode: u32, w const restored = try fsx.fstat(fd); if (restored.uid != m.info.uid or restored.gid != m.info.gid or restored.perm() != mode) return refuse(a, why, "ownership or mode did not round-trip: expected {d}:{d} {o}, found {d}:{d} {o}", .{ m.info.uid, m.info.gid, mode, restored.uid, restored.gid, restored.perm() }); - if (!(try readAcls(a, fd, why)).eql(m.acls)) return refuse(a, why, "access-control attributes did not round-trip", .{}); + const restored_acls = try readAcls(a, fd, why); + defer restored_acls.deinit(a); + if (!restored_acls.eql(m.acls)) return refuse(a, why, "access-control attributes did not round-trip", .{}); } /// Called before each named step; a test hook can stop the program there. @@ -349,7 +381,9 @@ pub const WriteOptions = struct { /// including the absence of one; `metadata_from` names another file whose /// metadata to apply. Fault points STEP.write, STEP.fsync and STEP.rename /// precede those operations, and in a replaceable build can fail them -/// (Faults.fails); the destination is never unlinked first. +/// (Faults.fails); the destination is never unlinked first. Everything it +/// allocates from `a` is freed before it returns, except the message of a +/// refusal (see `Why`). pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: WriteOptions, why: *Why) Error!void { sys.last_errno = .SUCCESS; const parent = if (std.fs.path.dirname(path)) |d| (if (d.len > 0) d else ".") else "."; @@ -359,12 +393,15 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write } else |_| {} const metadata_path: ?[]const u8 = opts.metadata_from orelse (if (exists(path)) path else null); var metadata: ?Metadata = null; + defer if (metadata) |m| m.deinit(a); if (metadata_path) |mp| { var inner: Why = .{}; + defer inner.deinit(a); metadata = readMetadata(a, mp, &inner) catch |err| return refuse(a, why, "cannot read access metadata for {s}: {s}", .{ mp, if (err == error.Refused) inner.msg else describe(err) }); } const mode: u32 = opts.mode orelse (if (metadata) |m| m.info.perm() else fileMode(path)); const tmp = try a.print("{s}/.{s}.new", .{ parent, std.fs.path.basename(path) }); + defer a.free(tmp); if (fsx.statAt(fsx.cwd, tmp, false)) |stale| { // A privileged writer hands its temporary file to the metadata owner // before a crash can interrupt it. @@ -377,6 +414,7 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write { const step = try a.print("{s}.write", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } const fd = try sys.openat(fsx.cwd, tmp, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, .NOFOLLOW = true }, 0o600); @@ -392,20 +430,25 @@ pub fn atomicWrite(a: Allocator, path: []const u8, data: []const u8, opts: Write defer sys.close(fd); try sys.writeAll(fd, data); var inner: Why = .{}; + defer inner.deinit(a); restoreMetadata(a, fd, metadata, mode, &inner) catch |err| return refuse(a, why, "cannot preserve access metadata for {s}: {s}", .{ path, if (err == error.Refused) inner.msg else describe(err) }); { const step = try a.print("{s}.fsync", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } _ = try op(ops.fsync, .{fd}); } if (metadata) |m| { var inner: Why = .{}; + defer inner.deinit(a); const current = readMetadata(a, metadata_path.?, &inner) catch |err| return refuse(a, why, "cannot recheck access metadata for {s}: {s}", .{ metadata_path.?, if (err == error.Refused) inner.msg else describe(err) }); + defer current.deinit(a); if (!sameMetadata(current, m)) return refuse(a, why, "access metadata of {s} changed before {s} was replaced", .{ metadata_path.?, path }); } { const step = try a.print("{s}.rename", .{opts.step}); + defer a.free(step); if (opts.faults.fails(step)) return injected(a, why, step); } try sys.renameat(fsx.cwd, tmp, fsx.cwd, path); @@ -417,6 +460,20 @@ fn discard(tmp: []const u8, created: fsx.Stat) void { if (cur.dev == created.dev and cur.ino == created.ino) sys.unlinkat(fsx.cwd, tmp, 0) catch {}; } +test "Why.deinit frees a refusal's message only when it was allocated" { + const gpa = std.testing.allocator; + var why: Why = .{}; + why.deinit(gpa); + try std.testing.expectEqual(error.Refused, refuse(gpa, &why, "cannot {s}", .{"write"})); + try std.testing.expectEqualStrings("cannot write", why.msg); + why.deinit(gpa); + try std.testing.expectEqualStrings("", why.msg); + try std.testing.expectEqual(error.Refused, refuse(std.testing.failing_allocator, &why, "cannot {s}", .{"write"})); + try std.testing.expectEqualStrings("refused", why.msg); + why.deinit(gpa); + try std.testing.expectEqualStrings("", why.msg); +} + test "atomicWrite keeps the mode and refuses a symlink" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); @@ -474,6 +531,55 @@ test "a step a test makes fail leaves the destination and no temporary file" { } } +test "atomicWrite frees what it allocates" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const dir = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/atomic-noleak-{d}", .{ cwd, linux.getpid() }); + defer gpa.free(dir); + try makedirs(dir); + defer fsx.deleteTree(gpa, fsx.cwd, dir) catch {}; + const path = try std.fmt.allocPrint(gpa, "{s}/j.json", .{dir}); + defer gpa.free(path); + var why: Why = .{}; + try atomicWrite(gpa, path, "one\n", .{}, &why); + try atomicWrite(gpa, path, "two\n", .{}, &why); + try std.testing.expectEqual(@as(usize, 0), why.msg.len); +} + +test "an interrupted ACL read is retried" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const dir = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/atomic-eintr-{d}", .{ cwd, linux.getpid() }); + defer gpa.free(dir); + try makedirs(dir); + defer fsx.deleteTree(gpa, fsx.cwd, dir) catch {}; + const path = try std.fmt.allocPrint(gpa, "{s}/f", .{dir}); + defer gpa.free(path); + const created = try sys.openat(fsx.cwd, path, .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o600); + sys.close(created); + const fd = try sys.openat(fsx.cwd, path, .{}, 0); + defer sys.close(fd); + const old = ops.*; + defer ops.* = old; + const S = struct { + var left: u8 = 0; + fn get(fd_: sys.fd_t, name: [:0]const u8, buf: []u8) usize { + if (left > 0) { + left -= 1; + return @bitCast(-@as(isize, @backingInt(linux.E.INTR))); + } + return linux.fgetxattr(fd_, name.ptr, buf.ptr, buf.len); + } + }; + S.left = 2; + ops.fgetxattr = S.get; + var why: Why = .{}; + const acls = try readAcls(gpa, fd, &why); + defer acls.deinit(gpa); +} + test "the umask is read without changing it" { const old: u32 = @intCast(linux.syscall1(.umask, 0o027)); defer _ = linux.syscall1(.umask, old); diff --git a/src/atomic_test.zig b/src/atomic_test.zig index 439a561..7978eb2 100644 --- a/src/atomic_test.zig +++ b/src/atomic_test.zig @@ -683,6 +683,22 @@ test "a denied read of a listed ACL is refused" { } } +test "a failed ACL read prints its errno's full text" { + begin(); + defer end(); + for (models()) |m0| { + var m = m0; + const s = scenario(&m); + defer cleanup(s.dir); + target = s.path; + hook_errno = .STALE; + m.hooks.get = hFail; + try expectRefused(write(s.path, "new", replace), "[Errno 116] Stale file handle"); + reset(&m); + try assertUnchanged(&m, s); + } +} + test "a listed, then unreadable ACL is refused" { begin(); defer end(); @@ -1684,3 +1700,34 @@ test "a privileged write refuses a temporary directory" { try testing.expect(isDir(pt[1])); try testing.expectEqualStrings("old", readFile(pt[0])); } + +// ---- ownership --------------------------------------------------------------- + +fn failFsync(_: ?*anyopaque, step: []const u8) bool { + return std.mem.eql(u8, step, "replace.fsync"); +} + +test "atomicWrite frees what it allocates but a refusal's message" { + begin(); + defer end(); + const gpa = testing.allocator; + for (models()) |m0| { + var m = m0; + const s = scenario(&m); + defer cleanup(s.dir); + var why: atomic.Why = .{}; + try atomic.atomicWrite(gpa, s.path, "new", replace, &why); + try atomic.atomicWrite(gpa, join(s.dir, "copy.lua"), "copy", .{ .metadata_from = s.path }, &why); + try testing.expectEqualStrings("", why.msg); + try testing.expectError(error.Refused, atomic.atomicWrite(gpa, s.path, "x", .{ .step = "replace", .faults = .{ .fail_fn = failFsync } }, &why)); + gpa.free(why.msg); + target = s.path; + hook_errno = .STALE; + m.hooks.get = hFail; + try testing.expectError(error.Refused, atomic.atomicWrite(gpa, s.path, "x", replace, &why)); + gpa.free(why.msg); + reset(&m); + try testing.expectEqualStrings("new", readFile(s.path)); + try expectAttrs(s.expected, m.getPath(s.path)); + } +} diff --git a/src/fsx.zig b/src/fsx.zig index e473fc0..0a82550 100644 --- a/src/fsx.zig +++ b/src/fsx.zig @@ -121,29 +121,65 @@ pub fn openDir(dir: fd_t, path: []const u8, follow: bool) Error!fd_t { fn copyFd(src: fd_t, dst: fd_t) Error!void { var buf: [64 * 1024]u8 = undefined; + var off: linux.off_t = 0; while (true) { - const n = try sys.read(src, &buf); + const n = try sys.call(linux.pread, .{ src, &buf, buf.len, off }); if (n == 0) return; try sys.writeAll(dst, buf[0..n]); + off += @intCast(n); } } -/// Copy an open regular file to `dir/name`, as cp(1) without options does: -/// an existing destination is truncated and keeps its mode, a new one gets -/// the source's permission bits less the umask. A destination that is a -/// symbolic link or not a regular file is refused. Returns the open -/// destination, which the caller closes. +/// Copy an open regular file to `dir/name`: all of it, whatever the +/// descriptor's offset, which is left where it was. The bytes are written +/// to a temporary file in `dir`, synced, and renamed over `name`, so a +/// short write (a full disk, a file-size limit) leaves an existing +/// destination as it was and does not publish a partial file under `name`. +/// An existing regular file keeps its owner, when this process may set it, +/// and its mode; a new file gets the source's permission bits less the +/// umask. A destination that is a symbolic link or not a regular file is +/// refused. A destination with other hard links keeps those links on the +/// previous inode; `name` is the new inode. Returns the open destination, +/// which the caller closes. pub fn copyTo(src: fd_t, src_stat: Stat, dir: fd_t, name: []const u8) Error!fd_t { - const fd = try sys.openat(dir, name, .{ + const existing: ?Stat = statAt(dir, name, false) catch |err| switch (err) { + error.NotFound => null, + else => |e| return e, + }; + if (existing) |st| { + if (st.isLink()) return error.SymLink; + if (st.isDir()) return error.IsDir; + if (!st.isReg()) return error.InvalidArgument; + } + var rnd: [4]u8 = undefined; + _ = try sys.call(linux.getrandom, .{ &rnd, rnd.len, 0 }); + var tmp_buf: [sys.path_max]u8 = undefined; + const tmp = std.fmt.bufPrint(&tmp_buf, ".{s}.{d}.{x:0>8}.tmp", .{ std.fs.path.basename(name), linux.getpid(), std.mem.readInt(u32, &rnd, .little) }) catch return error.NameTooLong; + const fd = try sys.openat(dir, tmp, .{ .ACCMODE = .WRONLY, .CREAT = true, + .EXCL = true, .NOFOLLOW = true, - .NONBLOCK = true, - }, @intCast(src_stat.mode & 0o777)); - errdefer sys.close(fd); - if (!(try fstat(fd)).isReg()) return error.InvalidArgument; - _ = try sys.call(linux.ftruncate, .{ fd, 0 }); + }, if (existing != null) 0o600 else @intCast(src_stat.mode & 0o777)); + var done = false; + errdefer { + sys.close(fd); + if (!done) sys.unlinkat(dir, tmp, 0) catch {}; + } try copyFd(src, fd); + if (existing) |st| { + if ((try fstat(fd)).uid != st.uid or (try fstat(fd)).gid != st.gid) { + sys.fchown(fd, st.uid, st.gid) catch |err| switch (err) { + error.AccessDenied => {}, + else => |e| return e, + }; + } + try sys.fchmod(fd, @intCast(st.perm())); + } + try sys.fsync(fd); + try sys.renameat(dir, tmp, dir, name); + done = true; + try sys.fsync(dir); return fd; } @@ -159,10 +195,22 @@ pub fn appendLine(gpa: Allocator, dir: fd_t, name: []const u8, line: []const u8) .NONBLOCK = true, }, 0o666); defer sys.close(fd); - if (!(try fstat(fd)).isReg()) return error.InvalidArgument; + const st = try fstat(fd); + if (!st.isReg()) return error.InvalidArgument; const bytes = try std.mem.concat(gpa, u8, &.{ line, "\n" }); defer gpa.free(bytes); - try sys.writeAll(fd, bytes); + // One write is atomic on Linux only when it completes. A short write + // (ENOSPC part way through) is rolled back to the size before it, so + // the list does not gain a partial line. Safe while this process is + // the only writer; a concurrent append in that window can be shortened. + sys.writeAll(fd, bytes) catch |err| { + if (fstat(fd)) |now| { + if (now.size > st.size and now.size - st.size < bytes.len) { + _ = linux.ftruncate(fd, @intCast(st.size)); + } + } else |_| {} + return err; + }; } /// Read a whole regular file opened by the caller; more than `limit` bytes @@ -180,25 +228,51 @@ pub fn readAll(gpa: Allocator, fd: fd_t, limit: usize) (Error || Allocator.Error return list.toOwnedSlice(gpa); } -const acl_names = [_][:0]const u8{ "system.posix_acl_access", "system.nfs4_acl" }; +/// The access ACL forms, most native first: the GPFS native NFSv4 ACL, the +/// generic NFSv4 ACL and the POSIX access ACL. A filesystem may expose one +/// ACL through several of them. +pub const acl_attributes = [_][:0]const u8{ "system.gpfs_nfs4_acl", "system.nfs4_acl", "system.posix_acl_access" }; +const posix_access = acl_attributes[2]; + +/// The xattr calls copyAcls makes, as raw system-call results; a test +/// emulates a filesystem's ACL model with its own. +const Xattrs = struct { + fn get(fd: fd_t, name: [:0]const u8, buf: []u8) usize { + return linux.fgetxattr(fd, name.ptr, buf.ptr, buf.len); + } + fn set(fd: fd_t, name: [:0]const u8, value: []const u8) usize { + return linux.fsetxattr(fd, name.ptr, value.ptr, value.len, 0); + } +}; -/// Copy the access ACL xattrs of `from` onto `to`. An ACL that exists on -/// `from` and cannot be set on `to` is an error. -fn copyAcls(from: fd_t, to: fd_t) Error!void { +/// Copy the access ACL of `from` onto `to` in the most native form `from` +/// has; the filesystem derives any other form it shows from that one. An +/// ACL that cannot be set on `to` is an error, and so is a form that cannot +/// be read when no other can be copied. +fn copyAcls(comptime X: type, from: fd_t, to: fd_t) Error!void { var buf: [64 * 1024]u8 = undefined; - for (acl_names) |name| { - const n = sys.call(linux.fgetxattr, .{ from, name.ptr, &buf, buf.len }) catch |err| switch (err) { + var unreadable = false; + for (acl_attributes) |name| { + const n = sys.call(X.get, .{ from, name, &buf }) catch |err| switch (err) { error.NoData, error.Unsupported => continue, + // GPFS answers EINVAL for an NFSv4 form of a file whose ACL is + // POSIX, which the last form reads. + error.InvalidArgument => if (std.mem.eql(u8, name, posix_access)) return err else { + unreadable = true; + continue; + }, else => return err, }; - _ = try sys.call(linux.fsetxattr, .{ to, name.ptr, &buf, n, 0 }); + _ = try sys.call(X.set, .{ to, name, buf[0..n] }); + return; } + if (unreadable) return error.InvalidArgument; } /// Replace the contents of the regular file `dir/name` atomically. The new /// file keeps the old one's owner (when this process may set it), mode and -/// access ACLs; it is written beside the old one, synced, then renamed over -/// it. +/// access ACL, in the most native of the `acl_attributes` forms the old one +/// has; it is written beside the old one, synced, then renamed over it. pub fn replaceContents(gpa: Allocator, dir: fd_t, name: []const u8, bytes: []const u8) (Error || Allocator.Error)!void { const old = try openRegular(dir, name, false); defer sys.close(old.fd); @@ -224,7 +298,7 @@ pub fn replaceContents(gpa: Allocator, dir: fd_t, name: []const u8, bytes: []con }; } try sys.fchmod(fd, @intCast(old.stat.perm())); - try copyAcls(old.fd, fd); + try copyAcls(Xattrs, old.fd, fd); try sys.fsync(fd); try sys.renameat(dir, tmp, dir, name); done = true; @@ -355,3 +429,190 @@ test "replaceContents keeps the mode" { defer gpa.free(got); try std.testing.expectEqualStrings("b\n", got); } + +fn withFileLimit(bytes: u64, body: *const fn () anyerror!void) anyerror!void { + var old: linux.rlimit = undefined; + if (linux.errno(linux.prlimit(0, .FSIZE, null, &old)) != .SUCCESS) return error.Unexpected; + const lim: linux.rlimit = .{ .cur = @intCast(bytes), .max = old.max }; + if (linux.errno(linux.prlimit(0, .FSIZE, &lim, null)) != .SUCCESS) return error.Unexpected; + const ign: linux.Sigaction = .{ .handler = .{ .handler = linux.SIG.IGN }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.XFSZ, &ign, &prev); + defer { + _ = linux.sigaction(.XFSZ, &prev, null); + _ = linux.prlimit(0, .FSIZE, &old, null); + } + try body(); +} + +test "copyTo leaves an existing file unchanged when the write cannot finish" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try realpath(cwd, ".", &cwd_buf); + var pb: [sys.path_max]u8 = undefined; + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-copy-{d}", .{ here, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try openDir(cwd, base, false); + defer { + deleteTree(gpa, cwd, base) catch {}; + sys.close(d); + } + const dst = try sys.openat(d, "dest", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o640); + try sys.writeAll(dst, "OLDDATA\n"); + sys.close(dst); + const srcfd = try sys.openat(d, "src", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + try sys.writeAll(srcfd, "NEWDATA-IS-LONGER-THAN-THE-LIMIT\n"); + sys.close(srcfd); + const opened = try openRegular(d, "src", false); + defer sys.close(opened.fd); + + const Hold = struct { + var dir: fd_t = -1; + var src: fd_t = -1; + var st: Stat = undefined; + fn go() Error!void { + if (copyTo(src, st, dir, "dest")) |fd| sys.close(fd) else |_| {} + } + }; + Hold.dir = d; + Hold.src = opened.fd; + Hold.st = opened.stat; + try withFileLimit(8, Hold.go); + + const got = try openRegular(d, "dest", false); + defer sys.close(got.fd); + const text = try readAll(gpa, got.fd, 100); + defer gpa.free(text); + try std.testing.expectEqualStrings("OLDDATA\n", text); + try std.testing.expectEqual(@as(u32, 0o640), got.stat.perm()); +} + +test "appendLine does not leave a partial line when the write cannot finish" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try realpath(cwd, ".", &cwd_buf); + var pb: [sys.path_max]u8 = undefined; + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-append-{d}", .{ here, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try openDir(cwd, base, false); + defer { + deleteTree(gpa, cwd, base) catch {}; + sys.close(d); + } + const f = try sys.openat(d, "list", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + try sys.writeAll(f, "hi\n"); + sys.close(f); + const Hold = struct { + var dir: fd_t = -1; + fn go() Error!void { + appendLine(std.testing.allocator, dir, "list", "this line is far longer than the file size limit") catch {}; + } + }; + Hold.dir = d; + try withFileLimit(8, Hold.go); + const got = try openRegular(d, "list", false); + defer sys.close(got.fd); + const text = try readAll(gpa, got.fd, 200); + defer gpa.free(text); + try std.testing.expectEqualStrings("hi\n", text); +} + +test "copyTo copies the whole source and leaves its offset" { + const gpa = std.testing.allocator; + var tmp_path_buf: [sys.path_max]u8 = undefined; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try realpath(cwd, ".", &cwd_buf); + const base = try std.mem.print(&tmp_path_buf, "{s}/.zig-cache/zig-common-copy-{d}", .{ here, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try openDir(cwd, base, false); + defer { + deleteTree(gpa, cwd, base) catch {}; + sys.close(d); + } + const f = try sys.openat(d, "src", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o600); + try sys.writeAll(f, "header\nbody\n"); + sys.close(f); + const src = try openRegular(d, "src", false); + defer sys.close(src.fd); + var head: [7]u8 = undefined; + try std.testing.expectEqual(@as(usize, 7), try sys.read(src.fd, &head)); + sys.close(try copyTo(src.fd, src.stat, d, "dst")); + const dst = try openRegular(d, "dst", false); + defer sys.close(dst.fd); + const got = try readAll(gpa, dst.fd, 100); + defer gpa.free(got); + try std.testing.expectEqualStrings("header\nbody\n", got); + try std.testing.expectEqual(@as(u64, 7), linux.lseek(src.fd, 0, linux.SEEK.CUR)); +} + +/// A filesystem's answer to each ACL form, GPFS native, generic NFSv4 and +/// POSIX access: the bytes of a form it shows, or its errno. Any other name +/// is unsupported. Records what copyAcls sets. +const FakeAcls = struct { + const Answer = union(enum) { bytes: []const u8, errno: linux.E }; + const forms = [_][]const u8{ "system.gpfs_nfs4_acl", "system.nfs4_acl", "system.posix_acl_access" }; + var answers: [3]Answer = undefined; + var sets: usize = 0; + var set_name: []const u8 = ""; + var set_buf: [32]u8 = undefined; + var set_len: usize = 0; + + fn reset(a: [3]Answer) void { + answers = a; + sets = 0; + set_name = ""; + set_len = 0; + } + fn rc(e: linux.E) usize { + return @bitCast(-@as(isize, @backingInt(e))); + } + fn get(_: fd_t, name: [:0]const u8, buf: []u8) usize { + for (forms, answers) |form, answer| if (std.mem.eql(u8, form, name)) return switch (answer) { + .bytes => |b| blk: { + @memcpy(buf[0..b.len], b); + break :blk b.len; + }, + .errno => |e| rc(e), + }; + return rc(.OPNOTSUPP); + } + fn set(_: fd_t, name: [:0]const u8, value: []const u8) usize { + sets += 1; + set_name = name; + @memcpy(set_buf[0..value.len], value); + set_len = value.len; + return 0; + } +}; + +test "replaceContents copies the most native ACL form, GPFS's included" { + const F = FakeAcls; + const unsupported: F.Answer = .{ .errno = .OPNOTSUPP }; + const cases = [_]struct { answers: [3]F.Answer, name: []const u8, bytes: []const u8 }{ + // GPFS: its native NFSv4 ACL alone. + .{ .answers = .{ .{ .bytes = "gpfs" }, unsupported, unsupported }, .name = "system.gpfs_nfs4_acl", .bytes = "gpfs" }, + // GPFS showing the same ACL through the generic and POSIX forms too. + .{ .answers = .{ .{ .bytes = "gpfs" }, .{ .bytes = "generic" }, .{ .bytes = "posix" } }, .name = "system.gpfs_nfs4_acl", .bytes = "gpfs" }, + // An NFSv4 mount. + .{ .answers = .{ unsupported, .{ .bytes = "nfs4" }, unsupported }, .name = "system.nfs4_acl", .bytes = "nfs4" }, + // A POSIX ACL on GPFS, which answers EINVAL for the NFSv4 forms. + .{ .answers = .{ .{ .errno = .INVAL }, .{ .errno = .INVAL }, .{ .bytes = "posix" } }, .name = "system.posix_acl_access", .bytes = "posix" }, + }; + for (cases) |c| { + F.reset(c.answers); + try copyAcls(F, 0, 1); + try std.testing.expectEqual(@as(usize, 1), F.sets); + try std.testing.expectEqualStrings(c.name, F.set_name); + try std.testing.expectEqualStrings(c.bytes, F.set_buf[0..F.set_len]); + } + // No ACL at all. + F.reset(.{ unsupported, unsupported, .{ .errno = .NODATA } }); + try copyAcls(F, 0, 1); + try std.testing.expectEqual(@as(usize, 0), F.sets); + // A form that could not be read and no other: refused, not dropped. + F.reset(.{ .{ .errno = .INVAL }, unsupported, .{ .errno = .NODATA } }); + try std.testing.expectError(error.InvalidArgument, copyAcls(F, 0, 1)); +} diff --git a/src/json.zig b/src/json.zig index 8c732f9..8ed13ec 100644 --- a/src/json.zig +++ b/src/json.zig @@ -1,10 +1,15 @@ -//! The JSON the guard reads and writes, with the two string encodings the -//! Python guard used: -//! - cache: json.dumps(ensure_ascii=True) of text decoded with -//! surrogateescape, so a byte that is not UTF-8 is a lone \udcXX; -//! - wire: the restore helper's request and reply, one character per byte +//! JSON as the Python tools these programs replace read and wrote it. +//! `parse` reads one document into `Value`s, keeping strings and numbers +//! as their JSON text until a codec or `toPy` decodes them. The two string +//! codecs are the encodings those tools used: +//! - cache (`decodeCache`, `writeCacheString`): json.dumps(ensure_ascii=True) +//! of text decoded with surrogateescape, so a byte that is not UTF-8 is a +//! lone \udcXX; +//! - wire (`decodeWire`, `writeWireString`): one character per byte //! (latin-1), so every byte is \u00XX or itself. -//! Strings are kept as their raw JSON text until a codec decodes them. +//! `toPy` turns a document into `Py` values, strings as cache text, and +//! `dumps` writes `Py` values as json.dumps(indent=N, sort_keys=...) does, +//! byte for byte, float repr included. const std = @import("std"); const text = @import("pytext.zig"); @@ -41,9 +46,10 @@ fn keyEql(raw: []const u8, want: []const u8) bool { while (i < raw.len) { var c: u21 = raw[i]; if (c == '\\') { + if (i + 1 >= raw.len) return false; const e = raw[i + 1]; if (e == 'u') { - c = std.fmt.parseInt(u16, raw[i + 2 .. i + 6], 16) catch return false; + c = hex4(raw, i + 2) catch return false; i += 6; } else { c = switch (e) { @@ -69,9 +75,18 @@ fn isHex(s: []const u8) bool { return true; } +/// The four hex digits of a \u escape at `raw[i..]`; parseInt alone would +/// also take a sign. +fn hex4(raw: []const u8, i: usize) error{Encoding}!u16 { + if (i + 4 > raw.len or !isHex(raw[i .. i + 4])) return error.Encoding; + return std.fmt.parseInt(u16, raw[i .. i + 4], 16) catch unreachable; +} + pub const ParseError = Allocator.Error || error{Syntax}; -const max_depth = 32; +/// Arrays and objects nest at most this deep. The parser recurses once per +/// level, so the bound keeps its stack use fixed. +pub const max_depth = 32; const Parser = struct { s: []const u8, @@ -90,8 +105,8 @@ const Parser = struct { return false; } + /// A value inside `depth` arrays and objects. fn value(p: *Parser, depth: usize) ParseError!Value { - if (depth > max_depth) return error.Syntax; p.ws(); if (p.i >= p.s.len) return error.Syntax; switch (p.s[p.i]) { @@ -100,6 +115,7 @@ const Parser = struct { 'f' => return if (p.lit("false")) .{ .bool = false } else error.Syntax, '"' => return .{ .string = try p.string() }, '[' => { + if (depth >= max_depth) return error.Syntax; p.i += 1; var items: std.ArrayList(Value) = .empty; p.ws(); @@ -123,6 +139,7 @@ const Parser = struct { } }, '{' => { + if (depth >= max_depth) return error.Syntax; p.i += 1; var members: std.ArrayList(Member) = .empty; p.ws(); @@ -210,7 +227,9 @@ const Parser = struct { } }; -/// Parse one JSON document; all memory comes from `arena`. +/// Parse one JSON document; all memory comes from `arena`. Arrays and +/// objects nested more than `max_depth` (32) deep are `error.Syntax`, +/// though json.loads reads them. pub fn parse(arena: Allocator, s: []const u8) ParseError!Value { var p: Parser = .{ .s = s, .arena = arena }; const v = try p.value(0); @@ -222,9 +241,11 @@ pub fn parse(arena: Allocator, s: []const u8) ParseError!Value { pub const DecodeError = Allocator.Error || error{Encoding}; /// Code points of a raw JSON string, surrogate pairs joined; a lone -/// surrogate is returned as itself. +/// surrogate is returned as itself. An escape json.loads refuses (cut off, +/// unknown, or \u without four hex digits) is an error. fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { var out: std.ArrayList(u21) = .empty; + errdefer out.deinit(arena); var i: usize = 0; while (i < raw.len) { const c = raw[i]; @@ -235,6 +256,7 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { i += ch.len; continue; } + if (i + 1 >= raw.len) return error.Encoding; const e = raw[i + 1]; i += 2; const simple: ?u21 = switch (e) { @@ -246,16 +268,17 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { 'n' => '\n', 'r' => '\r', 't' => '\t', - else => null, + 'u' => null, + else => return error.Encoding, }; if (simple) |s| { try out.append(arena, s); continue; } - var cp: u21 = std.fmt.parseInt(u16, raw[i .. i + 4], 16) catch return error.Encoding; + var cp: u21 = try hex4(raw, i); i += 4; if (cp >= 0xD800 and cp <= 0xDBFF and i + 6 <= raw.len and raw[i] == '\\' and raw[i + 1] == 'u') { - const lo = std.fmt.parseInt(u16, raw[i + 2 .. i + 6], 16) catch return error.Encoding; + const lo = try hex4(raw, i + 2); if (lo >= 0xDC00 and lo <= 0xDFFF) { cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); i += 6; @@ -263,7 +286,7 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { } try out.append(arena, cp); } - return out.items; + return out.toOwnedSlice(arena); } /// Cache text: code points as UTF-8, a lone surrogate U+DC80..U+DCFF as the @@ -271,7 +294,9 @@ fn codePoints(arena: Allocator, raw: []const u8) DecodeError![]u21 { /// environment and is an error. pub fn decodeCache(arena: Allocator, raw: []const u8) DecodeError![]u8 { const cps = try codePoints(arena, raw); + defer arena.free(cps); var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(arena); for (cps) |cp| { if (cp >= 0xDC80 and cp <= 0xDCFF) { try out.append(arena, @intCast(cp - 0xDC00)); @@ -283,13 +308,15 @@ pub fn decodeCache(arena: Allocator, raw: []const u8) DecodeError![]u8 { try out.appendSlice(arena, b[0..n]); } } - return out.items; + return out.toOwnedSlice(arena); } /// Wire text: every code point must be at most U+00FF and is one byte. pub fn decodeWire(arena: Allocator, raw: []const u8) DecodeError![]u8 { const cps = try codePoints(arena, raw); + defer arena.free(cps); const out = try arena.alloc(u8, cps.len); + errdefer arena.free(out); for (cps, 0..) |cp, i| { if (cp > 0xFF) return error.Encoding; out[i] = @intCast(cp); @@ -350,6 +377,13 @@ test "cache strings round-trip bytes that are not UTF-8" { const v = try parse(a, out.items); try std.testing.expectEqualStrings("a\"\xff\u{e9}\u{1F600}\n\x7f", try decodeCache(a, v.string)); try std.testing.expectError(error.Encoding, decodeCache(a, "\\ud800")); + // A truncated escape is an encoding error. It must not panic, and the + // bytes allocated before the error are freed when the caller did not + // pass an arena. + try std.testing.expectError(error.Encoding, decodeCache(std.testing.allocator, "a\\")); + try std.testing.expectError(error.Encoding, decodeCache(std.testing.allocator, "\\u12")); + const broken: Value = .{ .object = &.{.{ .key = "\\", .value = .null }} }; + try std.testing.expect(broken.get("x") == null); } test "wire strings are one byte per code point" { @@ -364,6 +398,16 @@ test "wire strings are one byte per code point" { try std.testing.expectError(error.Encoding, decodeWire(a, "\\u0100")); } +test "a truncated or malformed escape is an encoding error" { + const gpa = std.testing.allocator; + for ([_][]const u8{ "abc\\", "\\u12", "\\ud83d\\ude0", "\\u+123", "\\x0041" }) |raw| { + try std.testing.expectError(error.Encoding, decodeCache(gpa, raw)); + try std.testing.expectError(error.Encoding, decodeWire(gpa, raw)); + } + const obj: Value = .{ .object = &.{.{ .key = "a\\", .value = .null }} }; + try std.testing.expect(obj.get("a") == null); +} + test "parse objects, duplicates and nesting" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); @@ -379,6 +423,27 @@ test "parse objects, duplicates and nesting" { try std.testing.expectError(error.Syntax, parse(a, "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]")); } +/// `inner` inside `n` arrays, or objects when `open` opens one. +fn nested(a: Allocator, n: usize, open: []const u8, inner: []const u8, close: []const u8) Allocator.Error![]u8 { + var out: std.ArrayList(u8) = .empty; + for (0..n) |_| try out.appendSlice(a, open); + try out.appendSlice(a, inner); + for (0..n) |_| try out.appendSlice(a, close); + return out.items; +} + +test "arrays and objects nest at most 32 deep" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const a = arena_state.allocator(); + _ = try parse(a, try nested(a, 32, "[", "1", "]")); + _ = try parse(a, try nested(a, 31, "[", "[]", "]")); + _ = try parse(a, try nested(a, 31, "{\"k\":", "{}", "}")); + try std.testing.expectError(error.Syntax, parse(a, try nested(a, 33, "[", "1", "]"))); + try std.testing.expectError(error.Syntax, parse(a, try nested(a, 32, "[", "[]", "]"))); + try std.testing.expectError(error.Syntax, parse(a, try nested(a, 32, "{\"k\":", "{}", "}"))); +} + // ---- Python values and json.dumps ----------------------------------------- /// A value as a Python program built it for json.dumps. Strings are byte @@ -403,12 +468,24 @@ pub const Py = union(enum) { } }; -/// The parsed document as Python values, strings decoded as cache text. +/// A number as json.loads reads it: an int when the text has no fraction or +/// exponent, a float otherwise. An int that does not fit an i64, which +/// Python keeps exact, cannot be represented and is an error, not a +/// rounded float. +fn pyNumber(n: []const u8) error{Encoding}!Py { + if (std.mem.findAny(u8, n, ".eE") != null) { + return .{ .float = std.fmt.parseFloat(f64, n) catch return error.Encoding }; + } + return .{ .int = std.fmt.parseInt(i64, n, 10) catch return error.Encoding }; +} + +/// The parsed document as Python values, strings decoded as cache text; an +/// integer past i64 is `error.Encoding`. pub fn toPy(arena: Allocator, v: Value) DecodeError!Py { return switch (v) { .null => .null, .bool => |b| .{ .bool = b }, - .number => |n| if (std.fmt.parseInt(i64, n, 10)) |i| .{ .int = i } else |_| .{ .float = std.fmt.parseFloat(f64, n) catch return error.Encoding }, + .number => |n| try pyNumber(n), .string => |raw| .{ .str = try decodeCache(arena, raw) }, .array => |items| blk: { const out = try arena.alloc(Py, items.len); @@ -588,3 +665,14 @@ test "dumps matches json.dumps(indent=2, sort_keys=True)" { const parsed = try toPy(a, try parse(a, s)); try std.testing.expectEqualStrings(s, try dumps(a, parsed, 2, true)); } + +test "an integer past i64 is an error, not a rounded float" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const a = arena_state.allocator(); + try std.testing.expectError(error.Encoding, toPy(a, try parse(a, "18446744073709551616"))); + try std.testing.expectError(error.Encoding, toPy(a, try parse(a, "[-9223372036854775809]"))); + try std.testing.expectEqual(Py{ .int = std.math.minInt(i64) }, try toPy(a, try parse(a, "-9223372036854775808"))); + try std.testing.expectEqual(Py{ .float = 1e19 }, try toPy(a, try parse(a, "1e19"))); + try std.testing.expectEqual(Py{ .float = 3.0 }, try toPy(a, try parse(a, "3.0"))); +} diff --git a/src/logs.zig b/src/logs.zig index 6cdd7a8..e388aed 100644 --- a/src/logs.zig +++ b/src/logs.zig @@ -26,7 +26,18 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip const gz_name = try gpa.print("{s}.gz", .{name}); defer gpa.free(gz_name); - const dst = try sys.openat(dir, gz_name, .{ + // Refuse an existing archive before creating anything, as gzip does. + // The bytes then go to a temporary name, so a crash or a short write + // cannot leave a partial `name.gz` that a later run skips. + if (fsx.statAt(dir, gz_name, false)) |_| return error.Exists else |err| switch (err) { + error.NotFound => {}, + else => |e| return e, + } + var rnd: [4]u8 = undefined; + _ = try sys.call(linux.getrandom, .{ &rnd, rnd.len, 0 }); + const tmp_name = try gpa.print(".{s}.{d}.{x:0>8}.tmp", .{ gz_name, linux.getpid(), std.mem.readInt(u32, &rnd, .little) }); + defer gpa.free(tmp_name); + const dst = try sys.openat(dir, tmp_name, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true, @@ -35,7 +46,7 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip var done = false; defer { sys.close(dst); - if (!done) sys.unlinkat(dir, gz_name, 0) catch {}; + if (!done) sys.unlinkat(dir, tmp_name, 0) catch {}; } const file: Io.File = .{ .handle = dst, .flags = .{ .nonblocking = false } }; @@ -81,8 +92,17 @@ pub fn gzipInPlace(gpa: Allocator, io: Io, dir: fsx.fd_t, name: []const u8) Gzip try sys.fchmod(dst, @intCast(src.stat.perm())); const times = [2]linux.timespec{ src.stat.atime, src.stat.mtime }; _ = try sys.call(linux.futimens, .{ dst, × }); - try sys.unlinkat(dir, name, 0); + try sys.fsync(dst); + try sys.renameat(dir, tmp_name, dir, gz_name); + sys.unlinkat(dir, name, 0) catch |err| { + // The archive is complete, but the original is still there. Drop + // the archive so a retry compresses the original instead of + // skipping an existing .gz and leaving both. + sys.unlinkat(dir, gz_name, 0) catch {}; + return err; + }; done = true; + try sys.fsync(dir); } /// The lines of a `.licensed-logs` list with every line found in `renames` @@ -110,6 +130,76 @@ pub fn renameInList(gpa: Allocator, content: []const u8, renames: *const std.Str return try out.toOwnedSlice(gpa); } +fn archiveIntact(path: [*:0]const u8) bool { + const pid: isize = @bitCast(linux.fork()); + if (pid == 0) { + const argv = [_:null]?[*:0]const u8{ "gzip", "-t", path, null }; + const envp = [_:null]?[*:0]const u8{}; + _ = linux.execve("/usr/bin/gzip", &argv, &envp); + linux.exit_group(127); + } + var status: u32 = 0; + _ = linux.wait4(@intCast(pid), @ptrCast(&status), 0, null); + return linux.W.IFEXITED(status) and linux.W.EXITSTATUS(status) == 0; +} + +test "gzip does not publish a partial archive under the final name" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const base = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/zig-common-gzip-{d}", .{ here, linux.getpid() }); + defer gpa.free(base); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(fsx.cwd, (try sys.z(&zb, base)).ptr, 0o700); + const dir = try fsx.openDir(fsx.cwd, base, false); + defer { + fsx.deleteTree(gpa, fsx.cwd, base) catch {}; + sys.close(dir); + } + const src = try sys.openat(dir, "a.log", .{ .ACCMODE = .WRONLY, .CREAT = true }, 0o644); + const ur = try sys.openat(fsx.cwd, "/dev/urandom", .{}, 0); + defer sys.close(ur); + var buf: [64 * 1024]u8 = undefined; + var left: usize = 6 << 20; + while (left > 0) { + const n = try sys.read(ur, buf[0..@min(buf.len, left)]); + if (n == 0) return error.Unexpected; + try sys.writeAll(src, buf[0..n]); + left -= n; + } + sys.close(src); + const gz = try std.fmt.allocPrint(gpa, "{s}/a.log.gz", .{base}); + defer gpa.free(gz); + var gz_z: [sys.path_max:0]u8 = undefined; + const gz_sent = try sys.z(&gz_z, gz); + + const Watch = struct { + var stop = std.atomic.Value(bool).init(false); + var partial = std.atomic.Value(bool).init(false); + var path: [*:0]const u8 = undefined; + fn run() void { + while (!stop.load(.acquire)) { + if (fsx.statAt(fsx.cwd, std.mem.span(path), false)) |_| { + if (!archiveIntact(path)) partial.store(true, .release); + } else |_| {} + var ts: linux.timespec = .{ .sec = 0, .nsec = 2 * std.time.ns_per_ms }; + _ = linux.nanosleep(&ts, null); + } + } + }; + Watch.path = gz_sent.ptr; + Watch.stop.store(false, .release); + Watch.partial.store(false, .release); + const watch = try std.Thread.spawn(.{}, Watch.run, .{}); + const io = std.Io.Threaded.global_single_threaded.io(); + try gzipInPlace(gpa, io, dir, "a.log"); + Watch.stop.store(true, .release); + watch.join(); + try std.testing.expect(!Watch.partial.load(.acquire)); + try std.testing.expect(archiveIntact(gz_sent.ptr)); + try std.testing.expectError(error.NotFound, fsx.statAt(dir, "a.log", false)); +} + test "renameInList" { const gpa = std.testing.allocator; var m: std.StringHashMapUnmanaged([]const u8) = .empty; diff --git a/src/proc.zig b/src/proc.zig index 8fb12e5..a9b40e0 100644 --- a/src/proc.zig +++ b/src/proc.zig @@ -130,9 +130,18 @@ fn childExec( _ = linux.sigaction(@fromBackingInt(@intCast(s)), &act, null); } _ = linux.sigprocmask(linux.SIG.SETMASK, &empty, null); - if (linux.errno(linux.dup2(in_fd, 0)) != .SUCCESS) linux.exit_group(127); - if (linux.errno(linux.dup2(out_fd, 1)) != .SUCCESS) linux.exit_group(127); - if (linux.errno(linux.dup2(err_fd, 2)) != .SUCCESS) linux.exit_group(127); + // A source at 0, 1 or 2 (the caller started with that descriptor + // closed) moves above 2 first: a dup2 onto it would clobber it, and a + // dup2 onto itself keeps its close-on-exec flag. + var src = [3]sys.fd_t{ in_fd, out_fd, err_fd }; + for (&src) |*fd| if (fd.* < 3) { + const moved = linux.fcntl(fd.*, linux.F.DUPFD, 3); + if (linux.errno(moved) != .SUCCESS) linux.exit_group(127); + fd.* = @intCast(moved); + }; + for (src, 0..) |fd, target| { + if (linux.errno(linux.dup2(fd, @intCast(target))) != .SUCCESS) linux.exit_group(127); + } const keep_fd = exec_err_fd; if (keep_fd > 3) _ = linux.close_range(3, @intCast(keep_fd - 1), .{ .UNSHARE = false, .CLOEXEC = false }); _ = linux.close_range(if (keep_fd >= 3) @intCast(keep_fd + 1) else 3, std.math.maxInt(linux.fd_t), .{ .UNSHARE = false, .CLOEXEC = false }); @@ -168,16 +177,12 @@ fn exitedWithin(pid: linux.pid_t, ms: ?i64) bool { } } -fn reap(pid: linux.pid_t) Term { - while (true) { - var status: i32 = 0; - const rc = linux.wait4(pid, &status, 0, null); - switch (linux.errno(rc)) { - .SUCCESS => return decode(@bitCast(status)), - .INTR => continue, - else => return .{ .signaled = 9 }, - } - } +/// Reap the child. With SIGCHLD ignored (SIG_IGN or SA_NOCLDWAIT) the +/// kernel has reaped it and its status is gone: `error.NoChild`. +fn reap(pid: linux.pid_t) sys.Error!Term { + var status: i32 = 0; + _ = try sys.call(linux.wait4, .{ pid, &status, 0, null }); + return decode(@bitCast(status)); } /// True while a live (not zombie) process is in process group `pgid`. A @@ -189,6 +194,9 @@ pub fn groupAlive(pgid: linux.pid_t) bool { var dents: [16 * 1024]u8 align(8) = undefined; while (true) { const n = linux.getdents64(dir, &dents, dents.len); + // EINTR is not a failed scan. Treating it as "nobody is alive" + // would let waitGroupGone return while the group is still running. + if (linux.errno(n) == .INTR) continue; if (linux.errno(n) != .SUCCESS or n == 0) return false; var pos: usize = 0; while (pos < n) { @@ -267,6 +275,8 @@ fn keep(i: usize, bytes: []const u8, out: *std.ArrayList(u8), err_buf: *std.Arra } /// Run `exe` (an absolute path) with `argv` (argv[0] included) and `envp`. +/// The caller must not ignore SIGCHLD (SIG_IGN or SA_NOCLDWAIT): the kernel +/// then reaps the child and its status is lost, which is `error.NoChild`. pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp, opts: Options) Error!Result { if (exe.len == 0 or exe[0] != '/') return error.NotAbsolute; @@ -358,7 +368,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp }; if (got == 4) { if (opts.pgid_out) |o| o.store(0, .seq_cst); - _ = reap(pid); + _ = reap(pid) catch null; _ = try sys.result(@bitCast(-@as(isize, @intCast(e)))); return error.Unexpected; } @@ -481,7 +491,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp if (opts.new_session) kill(pid, true, .KILL); if (opts.new_session and opts.wait_for_group) waitGroupGone(pid); if (opts.pgid_out) |o| o.store(0, .seq_cst); - _ = reap(pid); + _ = reap(pid) catch null; return err; } // Members of the child's group outliving it are not ours to keep; the @@ -489,7 +499,7 @@ pub fn run(gpa: Allocator, exe: []const u8, argv: []const []const u8, envp: Envp if (opts.new_session) kill(pid, true, .KILL); if (opts.new_session and opts.wait_for_group) waitGroupGone(pid); if (opts.pgid_out) |o| o.store(0, .seq_cst); - const term = reap(pid); + const term = try reap(pid); const o = try out.toOwnedSlice(gpa); errdefer gpa.free(o); return .{ .term = term, .stdout = o, .stderr = try err_buf.toOwnedSlice(gpa) }; @@ -532,7 +542,91 @@ test "a set cancel flag kills the child" { try std.testing.expectError(error.Canceled, run(gpa, "/bin/sh", &.{ "sh", "-c", "sleep 30" }, no_env, .{ .cancel = &flag })); } +fn onUsr1Proc(_: linux.SIG) callconv(.c) void {} + +test "groupAlive stays true for a live group while signals arrive" { + const handler: linux.Sigaction = .{ .handler = .{ .handler = &onUsr1Proc }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.USR1, &handler, &prev); + defer _ = linux.sigaction(.USR1, &prev, null); + const Stop = struct { + var go = std.atomic.Value(bool).init(true); + fn buzz() void { + var blocked = linux.sigemptyset(); + linux.sigaddset(&blocked, .USR1); + _ = linux.sigprocmask(linux.SIG.BLOCK, &blocked, null); + while (go.load(.acquire)) _ = linux.kill(linux.getpid(), .USR1); + } + }; + Stop.go.store(true, .release); + const buzz = try std.Thread.spawn(.{}, Stop.buzz, .{}); + defer { + Stop.go.store(false, .release); + buzz.join(); + } + const pgid: linux.pid_t = @intCast(linux.getpgid(0)); + for (0..50) |_| try std.testing.expect(groupAlive(pgid)); +} + test "run refuses a relative program" { const gpa = std.testing.allocator; try std.testing.expectError(error.NotAbsolute, run(gpa, "sh", &.{"sh"}, no_env, .{})); } + +/// Run `body` in a forked child, which may close the descriptors the test +/// runner talks over, and expect it to exit 0. +fn expectChildPasses(comptime body: fn () u8) !void { + const pid: isize = @bitCast(linux.fork()); + if (pid < 0) return error.Unexpected; + if (pid == 0) linux.exit_group(body()); + var status: i32 = 0; + while (linux.errno(linux.wait4(@intCast(pid), &status, 0, null)) == .INTR) {} + try std.testing.expectEqual(Term{ .exited = 0 }, decode(@bitCast(status))); +} + +/// 0 when `script` under /bin/sh prints `want`, 1 when it prints something +/// else, 2 when it cannot be run. +fn shPrints(script: []const u8, stdin: ?[]const u8, want: []const u8) u8 { + const r = run(std.heap.page_allocator, "/bin/sh", &.{ "sh", "-c", script }, no_env, .{ .stdin = stdin }) catch return 2; + if (std.mem.eql(u8, r.stdout, want)) return 0; + std.debug.print("stdout \"{s}\", want \"{s}\"\n", .{ r.stdout, want }); + return 1; +} + +test "a closed stdout at start does not send the child's stderr to its stdout" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(1); + return shPrints("echo out; echo err >&2", null, "out\n"); + } + }.body); +} + +test "a closed stdin at start leaves the child's stdin and stderr on their files" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(0); + const r = shPrints("readlink /proc/self/fd/0 || echo closed", null, "/dev/null\n"); + if (r != 0) return r; + return shPrints("cat; readlink /proc/self/fd/2", "in\n", "in\n/dev/null\n"); + } + }.body); +} + +test "a closed stderr at start leaves the child's stderr on /dev/null" { + try expectChildPasses(struct { + fn body() u8 { + sys.close(2); + return shPrints("readlink /proc/self/fd/2 || echo closed", null, "/dev/null\n"); + } + }.body); +} + +test "a status lost to an ignored SIGCHLD is an error, not a kill by signal 9" { + const ignore: linux.Sigaction = .{ .handler = .{ .handler = linux.SIG.IGN }, .mask = linux.sigemptyset(), .flags = 0 }; + var old: linux.Sigaction = undefined; + _ = linux.sigaction(.CHLD, &ignore, &old); + defer _ = linux.sigaction(.CHLD, &old, null); + try std.testing.expectError(error.NoChild, run(std.testing.allocator, "/bin/sh", &.{ "sh", "-c", "exit 3" }, no_env, .{})); + try std.testing.expectEqual(linux.E.CHILD, sys.last_errno); +} diff --git a/src/pytext.zig b/src/pytext.zig index 6e3bc35..c3c28d9 100644 --- a/src/pytext.zig +++ b/src/pytext.zig @@ -118,6 +118,7 @@ pub fn pyStrip(s: []const u8) []const u8 { /// str.split() with no separator: the runs between Unicode whitespace. pub fn pySplit(arena: Allocator, s: []const u8) Allocator.Error![]const []const u8 { var out: std.ArrayList([]const u8) = .empty; + errdefer out.deinit(arena); var i: usize = 0; var start: ?usize = null; while (i < s.len) { @@ -130,7 +131,7 @@ pub fn pySplit(arena: Allocator, s: []const u8) Allocator.Error![]const []const i += ch.len; } if (start) |st| try out.append(arena, s[st..]); - return out.items; + return try out.toOwnedSlice(arena); } /// shlex.quote: unchanged when every byte is in [A-Za-z0-9_@%+=:,./-] and @@ -167,6 +168,16 @@ test "pyStrip and pySplit use Unicode whitespace" { try std.testing.expectEqualStrings("waitpkg", parts[1]); } +fn splitOwned(a: std.mem.Allocator) !void { + const parts = try pySplit(a, "a b c"); + defer a.free(parts); + try std.testing.expectEqual(@as(usize, 3), parts.len); +} + +test "pySplit frees its list when an allocation fails" { + try std.testing.checkAllAllocationFailures(std.testing.allocator, splitOwned, .{}); +} + test "shellQuote" { const gpa = std.testing.allocator; const cases = [_][2][]const u8{ diff --git a/src/state.zig b/src/state.zig index d175b1e..b095941 100644 --- a/src/state.zig +++ b/src/state.zig @@ -43,12 +43,15 @@ pub fn openPrivate(dir: fsx.fd_t, name: []const u8, create: bool) Error!?fsx.fd_ pub const LockError = Error || error{TimedOut}; /// flock(LOCK_EX) on `fd`, giving up after `timeout_ms` when one is given. +/// The deadline is read from the monotonic clock. A signal that cuts a +/// sleep short does not consume the whole wait: the next check looks at +/// the clock again. pub fn lockExclusive(fd: fsx.fd_t, timeout_ms: ?u32) LockError!void { - const ms = timeout_ms orelse { + const deadline: ?i64 = if (timeout_ms) |ms| nowMs() + ms else null; + if (deadline == null) { _ = try sys.call(linux.flock, .{ fd, std.posix.LOCK.EX }); return; - }; - var left: i64 = ms; + } while (true) { const rc = linux.flock(fd, std.posix.LOCK.EX | std.posix.LOCK.NB); switch (linux.errno(rc)) { @@ -56,11 +59,10 @@ pub fn lockExclusive(fd: fsx.fd_t, timeout_ms: ?u32) LockError!void { .AGAIN, .INTR => {}, else => return error.Unexpected, } + const left = deadline.? - nowMs(); if (left <= 0) return error.TimedOut; - const step: i64 = @min(left, 20); - var ts: linux.timespec = .{ .sec = 0, .nsec = @intCast(step * std.time.ns_per_ms) }; + var ts: linux.timespec = .{ .sec = 0, .nsec = @intCast(@min(left, 20) * std.time.ns_per_ms) }; _ = linux.nanosleep(&ts, null); - left -= step; } } @@ -124,6 +126,60 @@ test "a file that is not private is not read" { try lockExclusive(lk, 100); } +fn onUsr1(_: linux.SIG) callconv(.c) void {} + +test "lockExclusive waits out the timeout while signals arrive" { + const gpa = std.testing.allocator; + var pb: [sys.path_max]u8 = undefined; + var cwd_buf: [sys.path_max]u8 = undefined; + const cwd = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const base = try std.mem.print(&pb, "{s}/.zig-cache/zig-common-locktime-{d}", .{ cwd, linux.getpid() }); + var zb: [sys.path_max:0]u8 = undefined; + _ = linux.mkdirat(fsx.cwd, (try sys.z(&zb, base)).ptr, 0o700); + const d = try fsx.openDir(fsx.cwd, base, false); + defer { + fsx.deleteTree(gpa, fsx.cwd, base) catch {}; + sys.close(d); + } + const held = try sys.openat(d, "L", .{ .ACCMODE = .RDWR, .CREAT = true }, 0o600); + defer sys.close(held); + const waiter = try sys.openat(d, "L", .{ .ACCMODE = .RDWR }, 0); + defer sys.close(waiter); + _ = try sys.call(linux.flock, .{ held, std.posix.LOCK.EX }); + defer _ = linux.flock(held, std.posix.LOCK.UN); + + const handler: linux.Sigaction = .{ .handler = .{ .handler = &onUsr1 }, .mask = linux.sigemptyset(), .flags = 0 }; + var prev: linux.Sigaction = undefined; + _ = linux.sigaction(.USR1, &handler, &prev); + defer _ = linux.sigaction(.USR1, &prev, null); + const Stop = struct { + var go = std.atomic.Value(bool).init(true); + fn buzz() void { + // This thread must not catch SIGUSR1, or the waiter's sleep is + // never the one the signal interrupts. + var blocked = linux.sigemptyset(); + linux.sigaddset(&blocked, .USR1); + _ = linux.sigprocmask(linux.SIG.BLOCK, &blocked, null); + while (go.load(.acquire)) { + _ = linux.kill(linux.getpid(), .USR1); + var ts: linux.timespec = .{ .sec = 0, .nsec = std.time.ns_per_ms }; + _ = linux.nanosleep(&ts, null); + } + } + }; + Stop.go.store(true, .release); + const buzz = try std.Thread.spawn(.{}, Stop.buzz, .{}); + var t0: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &t0); + try std.testing.expectError(error.TimedOut, lockExclusive(waiter, 400)); + var t1: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &t1); + Stop.go.store(false, .release); + buzz.join(); + const ms = (t1.sec - t0.sec) * 1000 + @divTrunc(t1.nsec - t0.nsec, 1_000_000); + try std.testing.expect(ms >= 300 and ms < 5000); +} + // ---- the state-root lock: a flock gate and a POSIX record lock ------------- /// A state root's lock. The flock gate excludes processes on one kernel; the @@ -136,8 +192,9 @@ test "a file that is not private is not read" { /// Others may be able to write the state root. The root is opened without /// following a symbolic link as its last component, both files through its /// descriptor without following one at their names, and each is trusted -/// only as a regular file with one link: a FIFO planted at a name is never -/// opened, and the file behind a planted hard link is never truncated. +/// only as a regular file of this user with one link: a FIFO planted at a +/// name is never opened, and the file behind a planted hard link, or a file +/// another user owns, is never truncated. pub const RootLock = struct { gate: fsx.fd_t, posix: fsx.fd_t, @@ -148,9 +205,10 @@ pub const RootLock = struct { pub const AcquireError = Error || error{ Busy, - /// STATE_ROOT/.lock.posix is not a regular file with one link. + /// STATE_ROOT/.lock.posix is not a regular file of this user with + /// one link. NotPrivate, - /// STATE_ROOT/.lock is not a regular file with one link. + /// STATE_ROOT/.lock is not a regular file of this user with one link. GateNotPrivate, OutOfMemory, }; @@ -207,10 +265,10 @@ pub const RootLock = struct { /// STATE_ROOT is created when absent and opened O_DIRECTORY|O_NOFOLLOW; /// a symbolic link there is NotDir. Each lock file is opened relative to /// that descriptor with O_NOFOLLOW, created mode 0600, only when its name - /// is absent or a regular file with one link, and its descriptor must be - /// that file. The gate is checked again once both locks are held, right - /// before it is truncated. Allocates nothing; `gpa` stays for source - /// compatibility. + /// is absent or a regular file of this user with one link, and its + /// descriptor must be that file. The gate is checked again once both + /// locks are held, right before it is truncated. Allocates nothing; + /// `gpa` stays for source compatibility. pub fn acquire(gpa: Allocator, state_root: []const u8, timeout_ms: ?u32) AcquireError!RootLock { _ = gpa; const atomic = @import("atomic.zig"); @@ -234,12 +292,12 @@ pub const RootLock = struct { } /// `dir/name` opened read-write through `open`; null unless the name is - /// absent or a regular file with one link before the open, so a FIFO, - /// device or symbolic link there is never opened, and the descriptor is - /// that file after it. + /// absent or a regular file of this user with one link before the open, + /// so a FIFO, device or symbolic link there is never opened, and the + /// descriptor is that file after it. fn openLockFile(dir: fsx.fd_t, name: [:0]const u8, open: *const fn (dir: fsx.fd_t, name: [*:0]const u8) usize) Error!?fsx.fd_t { if (fsx.statAt(dir, name, false)) |st| { - if (!st.isReg() or st.nlink != 1) return null; + if (!st.isReg() or st.nlink != 1 or st.uid != linux.geteuid()) return null; } else |err| switch (err) { error.NotFound => {}, else => |e| return e, @@ -254,13 +312,13 @@ pub const RootLock = struct { return null; } - /// Whether `fd` is a regular file whose one link is `dir/name`. The name - /// is looked up as well as the descriptor: a planted hard link unlinked - /// while the lock is awaited leaves the descriptor on the other name's - /// file, with one link. + /// Whether `fd` is a regular file of this user whose one link is + /// `dir/name`. The name is looked up as well as the descriptor: a + /// planted hard link unlinked while the lock is awaited leaves the + /// descriptor on the other name's file, with one link. fn isSoleLink(dir: fsx.fd_t, name: []const u8, fd: fsx.fd_t) Error!bool { const st = try fsx.fstat(fd); - if (!st.isReg() or st.nlink != 1) return false; + if (!st.isReg() or st.nlink != 1 or st.uid != linux.geteuid()) return false; const at = fsx.statAt(dir, name, false) catch |err| switch (err) { error.NotFound => return false, else => |e| return e, diff --git a/src/sys.zig b/src/sys.zig index fea0e2a..ec130c7 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -363,4 +363,18 @@ test "describe prints the errno the kernel gave, as Python's OSError does" { // An error that the last errno does not map to gets its usual errno. try std.testing.expectEqualStrings("[Errno 2] No such file or directory", describe(error.NotFound)); try std.testing.expectEqual(@as(?Error, null), asError(error.OutOfMemory)); + try std.testing.expectEqualStrings("[Errno 133] Memory page has hardware error", describeErrno(.HWPOISON)); + try std.testing.expectEqualStrings("[Errno 134] Unknown error 134", describeErrno(@fromBackingInt(@intCast(134)))); + const compat_fd = try openat(linux.AT.FDCWD, "COMPAT.md", .{}, 0); + defer close(compat_fd); + var compat: std.ArrayList(u8) = .empty; + defer compat.deinit(std.testing.allocator); + var chunk: [512]u8 = undefined; + while (true) { + const n = try read(compat_fd, &chunk); + if (n == 0) break; + try compat.appendSlice(std.testing.allocator, chunk[0..n]); + } + try std.testing.expect(std.mem.indexOf(u8, compat.items, "errno 133") != null); + try std.testing.expect(std.mem.indexOf(u8, compat.items, "errno 134,") == null); } diff --git a/towncrier.toml b/towncrier.toml new file mode 100644 index 0000000..19a34e1 --- /dev/null +++ b/towncrier.toml @@ -0,0 +1,37 @@ +[tool.towncrier] +name = "zig-common" +directory = "changelog.d" +filename = "CHANGELOG.md" +start_string = "\n" +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/stackhands/zig-common/pull/{issue})" +underlines = ["", "", ""] +# Setting ignore, even to nothing, is what makes `towncrier build` fail a +# fragment whose name has no configured type. Unset, that file is skipped +# and the draft still passes. .gitkeep stays ignored either way. +ignore = [] + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Fixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Miscellaneous" +showcontent = true