diff --git a/packages/plugin/src/features/magic-context/storage-db.ts b/packages/plugin/src/features/magic-context/storage-db.ts index 2d5e5dc0d..8eece567c 100644 --- a/packages/plugin/src/features/magic-context/storage-db.ts +++ b/packages/plugin/src/features/magic-context/storage-db.ts @@ -184,8 +184,8 @@ function ensureSecureStorageDir(dir: string): void { ensureStorageDirectorySync(dir); } -function tightenStorageTree(dir: string): void { - const { failures } = tightenStorageTreeSync(dir); +function tightenStorageTree(dbPath: string): void { + const { failures } = tightenStorageTreeSync(dirname(dbPath), dbPath); log(`[magic-context] storage permission tightening failures=${failures}`); } @@ -1131,7 +1131,7 @@ function finishDatabaseOpen( // created them. Externally managed trusted-group storage skips this entirely. // SQLite creates WAL sidecars itself; the startup tree pass also covers files // materialized during this open. The helper logs failure counts only. - tightenStorageTree(dirname(dbPath)); + tightenStorageTree(dbPath); databases.set(dbPath, db); pathByDatabase.set(db, dbPath); persistenceByDatabase.set(db, true); @@ -2691,7 +2691,7 @@ export function openDatabase(dbPathOrOptions?: string | OpenDatabaseOptions): Da migrateLegacyStorageIfNeeded(dbPath, dbDir); } ensureSecureStorageDir(dbDir); - tightenStorageTree(dbDir); + tightenStorageTree(dbPath); if (!existsSync(dbPath) && shouldEnforcePrivateStoragePermissions()) { try { writeStorageFileSync(dbPath, new Uint8Array(), { flag: "wx" }); @@ -2813,7 +2813,7 @@ export async function openDatabaseAsync( try { if (!explicitDbPath) migrateLegacyStorageIfNeeded(dbPath, dbDir); ensureSecureStorageDir(dbDir); - tightenStorageTree(dbDir); + tightenStorageTree(dbPath); if (!existsSync(dbPath) && shouldEnforcePrivateStoragePermissions()) { try { writeStorageFileSync(dbPath, new Uint8Array(), { flag: "wx" }); diff --git a/packages/plugin/src/hooks/magic-context/lkg-recovery-fence.test.ts b/packages/plugin/src/hooks/magic-context/lkg-recovery-fence.test.ts index fb4ba8922..119fa2281 100644 --- a/packages/plugin/src/hooks/magic-context/lkg-recovery-fence.test.ts +++ b/packages/plugin/src/hooks/magic-context/lkg-recovery-fence.test.ts @@ -1,6 +1,7 @@ import { Database as BunDatabase } from "bun:sqlite"; -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs"; import { dirname, join } from "node:path"; import { @@ -10,7 +11,7 @@ import { import { clearSession, closeDatabase, openDatabase } from "../../features/magic-context/storage"; import { deleteSessionScopedRows } from "../../features/magic-context/storage-session-tables"; import type { Database } from "../../shared/sqlite"; -import { writeStorageFileSync } from "../../shared/storage-permissions"; +import { tightenStorageTreeSync, writeStorageFileSync } from "../../shared/storage-permissions"; import { cleanupTestTempDir, createTestTempDir } from "../../shared/test-temp-dir"; import { clearPersistedLkgSlotStrict, @@ -117,6 +118,37 @@ ${code}`, } describe.skipIf(process.platform === "win32")("durable recovery fence", () => { + test("locked publications remove their directory after two operations on the same fence", () => { + const f = fixture(); + try { + expect(existsSync(`${f.path}.locks`)).toBe(false); + f.dirty(); + expect(existsSync(`${f.path}.locks`)).toBe(false); + expect(saveLkgSlotToDb(f.db, "session", slot, undefined, { rebuild: true })).toBe(true); + expect(existsSync(`${f.path}.locks`)).toBe(false); + } finally { + f.raw.close(); + } + }); + test("temp cleanup removes only the known name without listing the fence folder", () => { + const f = fixture(); + const known = `${f.path}.tmp`; + const unrelated = `${f.path}.unrelated.tmp`; + writeStorageFileSync(known, "stale", { forcePrivate: true }); + writeStorageFileSync(unrelated, "keep", { forcePrivate: true }); + const list = spyOn(fs, "readdirSync"); + try { + f.dirty(); + expect(existsSync(known)).toBe(false); + expect(readFileSync(unrelated, "utf8")).toBe("keep"); + expect( + list.mock.calls.filter(([path]) => String(path) === dirname(f.path)), + ).toHaveLength(0); + } finally { + list.mockRestore(); + f.raw.close(); + } + }); test("warm synchronous captures perform zero fence I/O", () => { const f = fixture(); const operations: string[] = []; @@ -400,14 +432,23 @@ for (let i=0, attempts=0; i<12;) { f.raw.close(); } }); - test("session clear removes the certificate and orphan temporary records", () => { + test("session clear removes the certificate and its known temporary record", () => { const f = fixture(); try { - writeStorageFileSync(`${f.path}.dead.tmp`, "orphan", { forcePrivate: true }); + writeStorageFileSync(`${f.path}.tmp`, "orphan", { forcePrivate: true }); + writeStorageFileSync(`${f.path}.dead.tmp`, "legacy", { forcePrivate: true }); + const old = new Date(Date.now() - 11 * 60_000); + fs.utimesSync(`${f.path}.dead.tmp`, old, old); + const unrelated = join(dirname(f.path), "unrelated.tmp"); + writeStorageFileSync(unrelated, "keep", { forcePrivate: true }); clearPersistedLkgSlotStrict(f.db, "session"); expect(f.load()).toBeUndefined(); expect(existsSync(f.path)).toBe(false); + expect(existsSync(`${f.path}.tmp`)).toBe(false); + expect(existsSync(`${f.path}.dead.tmp`)).toBe(true); + expect(tightenStorageTreeSync(dirname(dirname(f.path))).failures).toBe(0); expect(existsSync(`${f.path}.dead.tmp`)).toBe(false); + expect(readFileSync(unrelated, "utf8")).toBe("keep"); clearLkgRecoveryFence("session"); } finally { f.raw.close(); diff --git a/packages/plugin/src/hooks/magic-context/lkg-recovery-fence.ts b/packages/plugin/src/hooks/magic-context/lkg-recovery-fence.ts index 647967f6a..be7f13a21 100644 --- a/packages/plugin/src/hooks/magic-context/lkg-recovery-fence.ts +++ b/packages/plugin/src/hooks/magic-context/lkg-recovery-fence.ts @@ -1,6 +1,6 @@ import { createHash, randomUUID } from "node:crypto"; -import { lstatSync, readdirSync, readFileSync, rmdirSync, rmSync } from "node:fs"; -import { basename, dirname, join } from "node:path"; +import { lstatSync, readFileSync, rmdirSync, rmSync } from "node:fs"; +import { dirname, join } from "node:path"; import { BoundedSessionMap } from "../../shared/bounded-session-map"; import { getMagicContextStorageDir } from "../../shared/data-path"; import { getHarness } from "../../shared/harness"; @@ -123,6 +123,7 @@ function publishRecord(record: Omit): void { lkgRecoveryFencePath(record.sessionId), JSON.stringify({ ...record, checksum: hash(JSON.stringify(payload(record))) }), () => observer?.("temp-synced"), + `${lkgRecoveryFencePath(record.sessionId)}.tmp`, ); if (record.state === "dirty") observer?.("dirty"); remember(record.sessionId, { @@ -141,6 +142,8 @@ function publishRecord(record: Omit): void { writeStorageFileDurableAtomicSync( lkgRecoveryFencePath(record.sessionId), JSON.stringify({ ...dirty, checksum: hash(JSON.stringify(payload(dirty))) }), + undefined, + `${lkgRecoveryFencePath(record.sessionId)}.tmp`, ); } remember(record.sessionId, { state: "invalid", record: null }); @@ -201,10 +204,8 @@ function locked(sessionId: string, work: () => T): T { ioObserver?.("mutex"); return withStorageFileMutex(`${lkgRecoveryFencePath(sessionId)}.locks`, () => { const path = lkgRecoveryFencePath(sessionId); - for (const name of readdirSync(dirname(path))) { - if (name.startsWith(`${basename(path)}.`) && name.endsWith(".tmp")) - rmSync(join(dirname(path), name), { force: true }); - } + // The mutex prevents concurrent use of the temp name. A folder listing is not necessary. + rmSync(`${path}.tmp`, { force: true }); return work(); }); } diff --git a/packages/plugin/src/shared/storage-file-mutex-review.test.ts b/packages/plugin/src/shared/storage-file-mutex-review.test.ts index fc0b1e9d1..6d9d38e11 100644 --- a/packages/plugin/src/shared/storage-file-mutex-review.test.ts +++ b/packages/plugin/src/shared/storage-file-mutex-review.test.ts @@ -1,6 +1,6 @@ import { expect, spyOn, test } from "bun:test"; import { spawnSync } from "node:child_process"; -import { readdirSync, readFileSync, rmdirSync, writeFileSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync, rmdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { __resetRpcIdentityTestHooks, __setRpcIdentityTestHooks } from "./rpc-utils"; import { withStorageFileMutex } from "./storage-file-mutex"; @@ -9,6 +9,46 @@ import { createTestTempDir } from "./test-temp-dir"; const mutex = join(import.meta.dir, "storage-file-mutex.ts"); +test("mutex removes its empty directory after sequential operations and thrown work", () => { + const { dir: root, cleanup } = createTestTempDir("mc-mutex-sequence-"); + const directory = join(root, "locks"); + try { + for (let i = 0; i < 2; i++) { + expect(withStorageFileMutex(directory, () => i)).toBe(i); + expect(existsSync(directory)).toBe(false); + } + expect(() => + withStorageFileMutex(directory, () => { + throw new Error("work failed"); + }), + ).toThrow("work failed"); + expect(existsSync(directory)).toBe(false); + } finally { + cleanup(); + } +}); + +test("mutex directory cleanup preserves a ticket published during locked work", () => { + const { dir: root, cleanup } = createTestTempDir("mc-mutex-ticket-"); + const directory = join(root, "locks"); + const contender = join(directory, `${process.pid}-00000000-0000-0000-0000-000000000000.lock`); + try { + expect( + withStorageFileMutex(directory, () => { + writeFileSync(contender, "choosing", { mode: 0o600 }); + return "done"; + }), + ).toBe("done"); + expect(readFileSync(contender, "utf8")).toBe("choosing"); + expect(readdirSync(directory)).toHaveLength(1); + rmSync(contender); + withStorageFileMutex(directory, () => undefined); + expect(existsSync(directory)).toBe(false); + } finally { + cleanup(); + } +}); + test("mutex retries ticket creation when empty-directory cleanup wins the mkdir race", () => { const { dir: root, cleanup } = createTestTempDir("mc-mutex-rmdir-"); const directory = join(root, "locks"); @@ -101,7 +141,7 @@ test.skipIf(process.platform === "win32")( 200, ); expect(reached).toBe(true); - expect(readdirSync(join(root, "locks"))).toEqual([]); + expect(existsSync(join(root, "locks"))).toBe(false); } finally { if (child.exitCode === null) { child.kill("SIGKILL"); diff --git a/packages/plugin/src/shared/storage-file-mutex.ts b/packages/plugin/src/shared/storage-file-mutex.ts index fce93e2bb..13e1607dc 100644 --- a/packages/plugin/src/shared/storage-file-mutex.ts +++ b/packages/plugin/src/shared/storage-file-mutex.ts @@ -1,6 +1,6 @@ import { execFileSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { readdirSync, readFileSync, rmSync } from "node:fs"; +import { readdirSync, readFileSync, rmdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { isPidAlive, readProcessStartTime } from "./rpc-utils"; import { @@ -136,5 +136,10 @@ export function withStorageFileMutex(directory: string, work: () => T, timeou return result; } finally { rmSync(ownPath, { force: true }); + try { + rmdirSync(directory); + } catch { + // Do not remove a live contender's ticket. + } } } diff --git a/packages/plugin/src/shared/storage-permissions-creators.test.ts b/packages/plugin/src/shared/storage-permissions-creators.test.ts new file mode 100644 index 000000000..f1a45d391 --- /dev/null +++ b/packages/plugin/src/shared/storage-permissions-creators.test.ts @@ -0,0 +1,193 @@ +import { afterEach, beforeAll, describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { lstatSync, mkdirSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { cleanupTestTempDir, createTestTempDirFromPath } from "./test-temp-dir"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) cleanupTestTempDir(root); +}); + +function child(root: string, script: string) { + return spawnSync(process.execPath, ["--eval", script], { + cwd: import.meta.dir, + encoding: "utf8", + timeout: 120_000, + windowsHide: true, + env: { + ...process.env, + NODE_ENV: "test", + XDG_DATA_HOME: root, + XDG_CONFIG_HOME: root, + MAGIC_CONTEXT_TEST_DATA_DIR: root, + MAGIC_CONTEXT_STORAGE_DIR: root, + }, + }); +} + +describe.skipIf(process.platform === "win32")("storage creation under umask 0", () => { + beforeAll(() => mkdirSync("/tmp/opencode/mc-walk-pr", { recursive: true, mode: 0o700 })); + it("creates helper outputs and public storage records privately before any startup walk", () => { + const root = createTestTempDirFromPath("/tmp/opencode/mc-walk-pr/creators-"); + roots.push(root); + const result = child( + root, + ` +import { closeSync, lstatSync, readdirSync } from 'node:fs'; +import { once } from 'node:events'; +import { join } from 'node:path'; +import * as p from './storage-permissions.ts'; +import { withStorageFileMutex } from './storage-file-mutex.ts'; +import { markAnnouncementSeen } from './announcement.ts'; +import { rememberGitIdentity } from '../features/magic-context/memory/project-identity-cache.ts'; +import { refreshModelLimitsFromApi } from './models-dev-cache.ts'; +import { appendWindowReport } from '../features/magic-context/window-report-ledger.ts'; +import { markLkgRecoveryDirty } from '../hooks/magic-context/lkg-recovery-fence.ts'; +import { MagicContextRpcServer } from './rpc-server.ts'; +import { rpcPortDir } from './rpc-utils.ts'; +import { emitStallReport } from './host-stall-profiler-common.ts'; +const root = process.env.MAGIC_CONTEXT_STORAGE_DIR; +const check = (path) => { + const stat = lstatSync(path); + if (stat.isSymbolicLink()) return; + const expected = stat.isDirectory() ? 0o700 : 0o600; + if ((stat.mode & 0o777) !== expected) throw new Error(path + ' mode=' + (stat.mode & 0o777).toString(8)); + if (stat.isDirectory()) for (const entry of readdirSync(path)) check(join(path, entry)); +}; +const previous = process.umask(0); +let rpc; +try { + p.ensureStorageDirectorySync(join(root, 'helpers', 'nested')); + p.ensurePrivateStorageDirectorySync(join(root, 'private')); + p.writeStorageFileSync(join(root, 'sync'), 'x'); + p.writeStorageFileAtomicSync(join(root, 'atomic'), 'x'); + p.writeStorageFileDurableAtomicSync(join(root, 'durable', 'record'), 'x'); + p.copyStorageFileSync(join(root, 'sync'), join(root, 'copy')); + p.copyStorageTreeSync(join(root, 'durable'), join(root, 'copy-tree')); + closeSync(p.openStorageFileSync(join(root, 'open'), 'wx')); + await p.writeStorageFileAsync(join(root, 'async', 'file'), 'x'); + const stream = p.createStorageWriteStream(join(root, 'stream')); + const finished = once(stream, 'close'); + stream.end('x'); + await finished; + const handle = await p.createStorageFileHandleAsync(join(root, 'handle'), 'wx'); + await p.writeStorageFileHandleAsync(handle, 'x'); + await handle.close(); + withStorageFileMutex(join(root, 'mutex'), () => check(root)); + markAnnouncementSeen('0.49.37'); + rememberGitIdentity(join(root, 'project'), 'git:0123456789abcdef'); + await refreshModelLimitsFromApi({ config: { providers: async () => ({ data: { providers: [ + { id: 'test', models: { model: { limit: { context: 100000, output: 10000 } } } } + ] } }) } }); + appendWindowReport({ access_path: 'api', geometry: 'unknown', observed_at_ms: 123 }); + markLkgRecoveryDirty({ sessionId: 'private-creation', processLocalProof: false, slot: { + jsonPrefix: '[{"text":"old"}]', inputIdSeq: ['one'], inputContentDigests: ['digest'], + lastInputMessageId: 'one', modelKey: 'provider/model', providerKey: 'provider', capturedAt: 1, + } }); + emitStallReport({ profilerDir: () => join(root, 'host-profiler'), wallNow: () => 1000, + now: () => 1000, pid: process.pid, bunVersion: process.versions.bun, log: () => {}, + }, { startMs: 0, endMs: 1000, traces: [], windowed: [], windowFilter: 'all', + options: { thresholdMs: 500, sampleIntervalMs: 10 }, before: null, + after: { at: 'fixture', heap_size_bytes: null, jsc_memory: null, rss_bytes: null, + percent_available_memory_in_use: null }, backend: 'jsc' }); + rpc = new MagicContextRpcServer(root, join(root, 'project')); + if (await rpc.start() <= 0) throw new Error('RPC server did not start'); + if (!readdirSync(rpcPortDir(root, join(root, 'project'))).some(name => name.endsWith('.json'))) + throw new Error('RPC discovery record was not published'); + check(root); + console.log('PRIVATE_CREATORS_OK'); +} finally { + rpc?.stop(); + process.umask(previous); +} +`, + ); + expect(result.status, result.stdout + result.stderr).toBe(0); + expect(result.stdout).toContain("PRIVATE_CREATORS_OK"); + console.log(result.stdout.trim()); + const storage = join(root, "cortexkit", "magic-context"); + for (const name of ["project-identities", "lkg-recovery-fences"]) + expect(lstatSync(join(storage, name)).mode & 0o777).toBe(0o700); + expect(lstatSync(join(root, "host-profiler")).mode & 0o777).toBe(0o700); + expect(readdirSync(storage)).toContain("window-reports.jsonl"); + expect(readdirSync(storage)).toContain("model-context-limits-opencode.json"); + }); + + it("keeps SQLite DB, WAL and SHM private and bounded on a later real open", () => { + const root = createTestTempDirFromPath("/tmp/opencode/mc-walk-pr/sqlite-"); + roots.push(root); + const result = child( + root, + ` +import { expect, spyOn } from 'bun:test'; +import * as fs from 'node:fs'; +import { join } from 'node:path'; +import { openDatabase, closeDatabase } from '../features/magic-context/storage-db.ts'; +const root = process.env.MAGIC_CONTEXT_STORAGE_DIR; +const path = join(root, 'custom.sqlite'); +const previous = process.umask(0); +try { + const db = openDatabase({ dbPath: path }); + db.exec('CREATE TABLE permission_probe (id INTEGER); INSERT INTO permission_probe VALUES (1)'); + for (const suffix of ['', '-wal', '-shm']) { + expect(fs.lstatSync(path + suffix).mode & 0o777).toBe(0o600); + } + closeDatabase(); + fs.mkdirSync(join(root, 'lkg-recovery-fences'), { mode: 0o700 }); + for (let i = 0; i < 2000; i++) fs.writeFileSync(join(root, 'lkg-recovery-fences', i + '.json'), '{}', { mode: 0o600 }); + const lstat = spyOn(fs, 'lstatSync'); + try { + expect(openDatabase({ dbPath: path })).not.toBeNull(); + expect(lstat.mock.calls.length).toBeLessThanOrEqual(16); + console.log('SQLITE_PRIVATE_OK lstat=' + lstat.mock.calls.length); + } finally { lstat.mockRestore(); } +} finally { + closeDatabase(); + process.umask(previous); +} +`, + ); + expect(result.status, result.stdout + result.stderr).toBe(0); + expect(result.stdout).toContain("SQLITE_PRIVATE_OK"); + console.log(result.stdout.trim()); + }); + + it("retains the full check for third-party model downloads under umask 0", () => { + const root = createTestTempDirFromPath("/tmp/opencode/mc-walk-pr/models-"); + roots.push(root); + const result = child( + root, + ` +import { expect, spyOn } from 'bun:test'; +import * as fs from 'node:fs'; +import { join } from 'node:path'; +import { FileCache } from '../../node_modules/@huggingface/transformers/src/utils/cache/FileCache.js'; +import { tightenStorageTreeSync, writeStorageFileSync } from './storage-permissions.ts'; +const root = process.env.MAGIC_CONTEXT_STORAGE_DIR; +const cache = join(root, 'models'); +const file = join(cache, 'org/model/onnx/weights.bin'); +const previous = process.umask(0); +try { + await new FileCache(cache).put('org/model/onnx/weights.bin', new Response('fixture')); + expect(fs.lstatSync(cache).mode & 0o777).toBe(0o777); + expect(fs.lstatSync(file).mode & 0o777).toBe(0o666); + writeStorageFileSync(join(root, '.storage-permissions-v1'), '1\\n'); + const lstat = spyOn(fs, 'lstatSync'); + try { + expect(tightenStorageTreeSync(root)).toEqual({ tightened: 5, failures: 0 }); + expect(lstat.mock.calls.length).toBeLessThanOrEqual(11); + } finally { lstat.mockRestore(); } + expect(fs.lstatSync(cache).mode & 0o777).toBe(0o700); + expect(fs.lstatSync(file).mode & 0o777).toBe(0o600); + expect(fs.existsSync(join(root, '.storage-permissions-v1'))).toBe(false); + console.log('MODELS_CHECK_OK entries=5 before=0777/0666 after=0700/0600'); +} finally { process.umask(previous); } +`, + ); + expect(result.status, result.stdout + result.stderr).toBe(0); + expect(result.stdout).toContain("MODELS_CHECK_OK entries=5"); + console.log(result.stdout.trim()); + }); +}); diff --git a/packages/plugin/src/shared/storage-permissions-startup.test.ts b/packages/plugin/src/shared/storage-permissions-startup.test.ts new file mode 100644 index 000000000..e80b03551 --- /dev/null +++ b/packages/plugin/src/shared/storage-permissions-startup.test.ts @@ -0,0 +1,212 @@ +import { afterEach, describe, expect, it, spyOn } from "bun:test"; +import * as fs from "node:fs"; +import { join } from "node:path"; +import { + __resetStoragePrivatePermissionEnforcementForTests, + ensurePrivateStorageDirectorySync, + setStoragePrivatePermissionEnforcement, + tightenStorageTreeSync, +} from "./storage-permissions"; +import { cleanupTestTempDir, createTestTempDirFromPath } from "./test-temp-dir"; + +const roots: string[] = []; +const markerName = ".storage-permissions-v1"; + +function fixture(): string { + fs.mkdirSync("/tmp/opencode/mc-walk-pr", { recursive: true, mode: 0o700 }); + const root = createTestTempDirFromPath("/tmp/opencode/mc-walk-pr/startup-"); + roots.push(root); + fs.mkdirSync(join(root, "lkg-recovery-fences"), { mode: 0o700 }); + for (let i = 0; i < 2000; i++) { + fs.writeFileSync(join(root, "lkg-recovery-fences", `${i}.json`), "{}", { mode: 0o600 }); + } + fs.writeFileSync(join(root, "context.db"), "", { mode: 0o600 }); + return root; +} + +afterEach(() => { + __resetStoragePrivatePermissionEnforcementForTests(); + for (const root of roots.splice(0)) cleanupTestTempDir(root); +}); + +describe.skipIf(process.platform === "win32")("startup permission certificate", () => { + it("keeps fresh legacy fence temp files and removes only ones older than ten minutes", () => { + const root = fixture(); + const fences = join(root, "lkg-recovery-fences"); + const fresh = join(fences, "fresh.json.publisher.tmp"); + const old = join(fences, "old.json.publisher.tmp"); + const unrelated = join(fences, "unrelated.tmp"); + const fixed = join(fences, "fixed.json.tmp"); + const now = Date.now(); + for (const path of [fresh, old, unrelated, fixed]) { + fs.writeFileSync(path, "keep", { mode: 0o600 }); + const age = path === fresh ? 9 * 60_000 : 11 * 60_000; + fs.utimesSync(path, new Date(now - age), new Date(now - age)); + } + expect(tightenStorageTreeSync(root).failures).toBe(0); + expect(fs.existsSync(fresh)).toBe(true); + expect(fs.existsSync(old)).toBe(false); + expect(fs.readFileSync(unrelated, "utf8")).toBe("keep"); + expect(fs.readFileSync(fixed, "utf8")).toBe("keep"); + }); + + it("sweeps only empty orphan fence lock directories during the full pass", () => { + const root = fixture(); + const fences = join(root, "lkg-recovery-fences"); + for (const name of ["orphan.json.locks", "held.json.locks", "0.json.locks", "other.locks"]) + fs.mkdirSync(join(fences, name), { mode: 0o700 }); + fs.writeFileSync(join(fences, "held.json.locks", "ticket"), "held", { mode: 0o600 }); + fs.symlinkSync("orphan.json.locks", join(fences, "link.json.locks")); + expect(tightenStorageTreeSync(root).failures).toBe(0); + expect(fs.existsSync(join(fences, "orphan.json.locks"))).toBe(false); + for (const name of ["held.json.locks", "0.json.locks", "other.locks"]) + expect(fs.existsSync(join(fences, name))).toBe(true); + expect(fs.lstatSync(join(fences, "link.json.locks")).isSymbolicLink()).toBe(true); + fs.mkdirSync(join(fences, "later.json.locks"), { mode: 0o700 }); + tightenStorageTreeSync(root); + expect(fs.existsSync(join(fences, "later.json.locks"))).toBe(true); + }); + + it("checks only startup entries after a certified walk, not 2000 recovery fences", () => { + const root = fixture(); + fs.writeFileSync(join(root, markerName), "1\n", { mode: 0o600 }); + const lstat = spyOn(fs, "lstatSync"); + try { + expect(tightenStorageTreeSync(root).failures).toBe(0); + expect(lstat.mock.calls.length).toBeLessThanOrEqual(8); + console.log(`CERTIFIED_LSTAT calls=${lstat.mock.calls.length} fences=2000`); + } finally { + lstat.mockRestore(); + } + }); + + it("certifies the first full walk only after all entries succeed", () => { + const root = fixture(); + const lstat = spyOn(fs, "lstatSync"); + try { + expect(tightenStorageTreeSync(root).failures).toBe(0); + expect(lstat.mock.calls.length).toBeGreaterThan(2000); + expect(fs.lstatSync(join(root, markerName)).mode & 0o777).toBe(0o600); + } finally { + lstat.mockRestore(); + } + }); + + for (const target of ["context.db", ".", "context.db-wal", "context.db-shm"]) { + it(`invalidates after repairing ${target}, then walks fully on the next open`, () => { + const root = fixture(); + fs.writeFileSync(join(root, markerName), "1\n", { mode: 0o600 }); + if (target.endsWith("-wal") || target.endsWith("-shm")) { + fs.writeFileSync(join(root, target), ""); + } + fs.chmodSync(join(root, target), target === "." ? 0o755 : 0o644); + expect(tightenStorageTreeSync(root)).toMatchObject({ tightened: 1, failures: 0 }); + expect(fs.lstatSync(join(root, target)).mode & 0o777).toBe( + target === "." ? 0o700 : 0o600, + ); + expect(fs.existsSync(join(root, markerName))).toBe(false); + const lstat = spyOn(fs, "lstatSync"); + try { + tightenStorageTreeSync(root); + expect(lstat.mock.calls.length).toBeGreaterThan(2000); + } finally { + lstat.mockRestore(); + } + }); + } + + it("does not trust a symlink or a permissive certificate", () => { + for (const kind of ["symlink", "mode"]) { + const root = fixture(); + const marker = join(root, markerName); + if (kind === "symlink") fs.symlinkSync("context.db", marker); + else fs.writeFileSync(marker, "1\n", { mode: 0o644 }); + const lstat = spyOn(fs, "lstatSync"); + try { + tightenStorageTreeSync(root); + expect(lstat.mock.calls.length).toBeGreaterThan(2000); + } finally { + lstat.mockRestore(); + } + expect(fs.lstatSync(marker).isSymbolicLink()).toBe(false); + expect(fs.lstatSync(marker).mode & 0o777).toBe(0o600); + } + }); + + it("never certifies a failed walk or enforcement-off storage", () => { + const root = fixture(); + const realLstat = fs.lstatSync; + const lstat = spyOn(fs, "lstatSync").mockImplementation((...args) => { + if (String(args[0]).endsWith("/12.json")) throw new Error("injected lstat failure"); + return realLstat(...args); + }); + try { + expect(tightenStorageTreeSync(root).failures).toBe(1); + expect(fs.existsSync(join(root, markerName))).toBe(false); + } finally { + lstat.mockRestore(); + } + setStoragePrivatePermissionEnforcement(false); + expect(tightenStorageTreeSync(root)).toEqual({ tightened: 0, failures: 0 }); + expect(fs.existsSync(join(root, markerName))).toBe(false); + }); + + it("invalidates certification when a bounded entry cannot be checked", () => { + const root = fixture(); + fs.writeFileSync(join(root, markerName), "1\n", { mode: 0o600 }); + const realLstat = fs.lstatSync; + const lstat = spyOn(fs, "lstatSync").mockImplementation((...args) => { + if (String(args[0]) === join(root, "context.db")) { + throw Object.assign(new Error("injected permission failure"), { code: "EACCES" }); + } + return realLstat(...args); + }); + try { + expect(tightenStorageTreeSync(root)).toMatchObject({ failures: 1 }); + expect(fs.existsSync(join(root, markerName))).toBe(false); + } finally { + lstat.mockRestore(); + } + }); + + for (const subtree of ["models", "logs"]) { + it(`still tightens third-party ${subtree} entries after certification`, () => { + const root = fixture(); + fs.writeFileSync(join(root, markerName), "1\n", { mode: 0o600 }); + fs.mkdirSync(join(root, subtree), { mode: 0o755 }); + fs.writeFileSync(join(root, subtree, "entry"), "fixture", { mode: 0o644 }); + fs.chmodSync(join(root, subtree), 0o755); + fs.chmodSync(join(root, subtree, "entry"), 0o644); + expect(tightenStorageTreeSync(root)).toMatchObject({ tightened: 2, failures: 0 }); + expect(fs.lstatSync(join(root, subtree)).mode & 0o777).toBe(0o700); + expect(fs.lstatSync(join(root, subtree, "entry")).mode & 0o777).toBe(0o600); + expect(fs.existsSync(join(root, markerName))).toBe(false); + }); + } + + it("invalidates when the existing private-directory check finds a wrong mode", () => { + const root = fixture(); + fs.writeFileSync(join(root, markerName), "1\n", { mode: 0o600 }); + const directory = join(root, "lkg-recovery-fences"); + fs.chmodSync(directory, 0o755); + ensurePrivateStorageDirectorySync(directory); + expect(fs.existsSync(join(root, markerName))).toBe(false); + expect(fs.lstatSync(directory).mode & 0o777).toBe(0o700); + }); + + it("removes certification while enforcement is off before a later private open", () => { + const root = fixture(); + fs.writeFileSync(join(root, markerName), "1\n", { mode: 0o600 }); + setStoragePrivatePermissionEnforcement(false); + expect(tightenStorageTreeSync(root)).toEqual({ tightened: 0, failures: 0 }); + expect(fs.existsSync(join(root, markerName))).toBe(false); + setStoragePrivatePermissionEnforcement(true); + const lstat = spyOn(fs, "lstatSync"); + try { + tightenStorageTreeSync(root); + expect(lstat.mock.calls.length).toBeGreaterThan(2000); + } finally { + lstat.mockRestore(); + } + }); +}); diff --git a/packages/plugin/src/shared/storage-permissions.ts b/packages/plugin/src/shared/storage-permissions.ts index 8e73a3da9..54f9f1ee6 100644 --- a/packages/plugin/src/shared/storage-permissions.ts +++ b/packages/plugin/src/shared/storage-permissions.ts @@ -18,6 +18,7 @@ import { readlinkSync, readSync, renameSync, + rmdirSync, rmSync, symlinkSync, writeFileSync, @@ -25,9 +26,19 @@ import { } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import { open as openAsync, writeFile as writeFileAsync } from "node:fs/promises"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; let enforcePrivateStoragePermissions = true; +const STORAGE_PERMISSIONS_POLICY_VERSION = 1; +const STORAGE_PERMISSIONS_MARKER = `.storage-permissions-v${STORAGE_PERMISSIONS_POLICY_VERSION}`; +const LEGACY_FENCE_TEMP_MIN_AGE_MS = 10 * 60_000; + +function invalidateStoragePermissionMarkers(directory: string): void { + for (let current = directory; ; current = dirname(current)) { + rmSync(join(current, STORAGE_PERMISSIONS_MARKER), { force: true }); + if (dirname(current) === current) return; + } +} export function setStoragePrivatePermissionEnforcement(enforce: boolean): void { enforcePrivateStoragePermissions = enforce; @@ -61,7 +72,10 @@ export function ensurePrivateStorageDirectorySync(directory: string): void { const stat = lstatSync(directory); if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("Private storage directory is not a real directory"); - if (process.platform !== "win32") chmodSync(directory, 0o700); + if (process.platform !== "win32") { + if ((stat.mode & 0o777) !== 0o700) invalidateStoragePermissionMarkers(directory); + chmodSync(directory, 0o700); + } } /** Create or replace a storage file using a private mode from the first byte. */ @@ -105,15 +119,17 @@ export function syncStorageDirectorySync(directory: string): void { } } -/** Publish a safety record only after both its bytes and directory entry are durable. */ +/** Publish a safety record only after both its bytes and directory entry are durable. + * Hold the mutex through this call to use a constant tempPath. + */ export function writeStorageFileDurableAtomicSync( filePath: string, content: string, afterTempSync?: () => void, + tempPath = `${filePath}.${process.pid}.${randomBytes(12).toString("hex")}.tmp`, ): void { const directory = dirname(filePath); ensurePrivateStorageDirectorySync(directory); - const tempPath = `${filePath}.${process.pid}.${randomBytes(12).toString("hex")}.tmp`; let fd: number | undefined; try { fd = openStorageFileSync(tempPath, "wx", true); @@ -219,21 +235,78 @@ export async function createStorageFileHandleAsync( } /** Tighten existing entries during storage startup; symlinks are never followed. */ -export function tightenStorageTreeSync(root: string): { tightened: number; failures: number } { - if (!shouldEnforcePrivateStoragePermissions() || process.platform === "win32") { +export function tightenStorageTreeSync( + root: string, + dbPath = join(root, "context.db"), +): { tightened: number; failures: number } { + if (process.platform === "win32") { return { tightened: 0, failures: 0 }; } + const marker = join(root, STORAGE_PERMISSIONS_MARKER); let tightened = 0; let failures = 0; - const visit = (entryPath: string): void => { - let stat: ReturnType; + const removeMarker = (): void => { try { - stat = lstatSync(entryPath); + rmSync(marker, { force: true }); } catch { failures++; + } + }; + if (!shouldEnforcePrivateStoragePermissions()) { + removeMarker(); + return { tightened, failures }; + } + let certified = false; + try { + const stat = lstatSync(marker); + certified = stat.isFile() && (stat.mode & 0o7777) === 0o600; + if (!certified) removeMarker(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") failures++; + } + const fenceDirectory = join(root, "lkg-recovery-fences"); + const visit = (entryPath: string, recursive = true, optional = false): void => { + let stat: ReturnType; + try { + stat = lstatSync(entryPath); + } catch (error) { + if (!optional || (error as NodeJS.ErrnoException).code !== "ENOENT") failures++; return; } if (stat.isSymbolicLink()) return; + if (!certified && dirname(entryPath) === fenceDirectory) { + const name = basename(entryPath); + // A legacy publisher can use a recent temp file. + if ( + stat.isFile() && + /^.+\.json\..+\.tmp$/.test(name) && + Date.now() - stat.mtimeMs > LEGACY_FENCE_TEMP_MIN_AGE_MS + ) { + try { + rmSync(entryPath, { force: true }); + } catch { + failures++; + } + return; + } + if (stat.isDirectory() && name.endsWith(".json.locks")) { + try { + lstatSync(entryPath.slice(0, -".locks".length)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + try { + // A live ticket prevents removal of its directory. + rmdirSync(entryPath); + return; + } catch (cleanupError) { + const code = (cleanupError as NodeJS.ErrnoException).code; + if (code === "ENOENT") return; + if (code !== "ENOTEMPTY" && code !== "EEXIST") failures++; + } + } else failures++; + } + } + } const mode = stat.isDirectory() ? 0o700 : 0o600; if ((stat.mode & 0o077) !== 0 || (stat.mode & 0o777) !== mode) { try { @@ -243,7 +316,7 @@ export function tightenStorageTreeSync(root: string): { tightened: number; failu failures++; } } - if (!stat.isDirectory()) return; + if (!recursive || !stat.isDirectory()) return; let entries: string[]; try { entries = readdirSync(entryPath); @@ -253,6 +326,27 @@ export function tightenStorageTreeSync(root: string): { tightened: number; failu } for (const entry of entries) visit(join(entryPath, entry)); }; - visit(root); + if (certified) { + visit(root, false); + visit(dbPath, false, true); + visit(`${dbPath}-wal`, false, true); + visit(`${dbPath}-shm`, false, true); + // Transformers does not set cache entry modes. Keep this subtree checked. + visit(join(root, "models"), true, true); + // The Rust logger does not use the storage helpers. + visit(join(root, "logs"), true, true); + if (tightened > 0 || failures > 0) removeMarker(); + } else { + visit(root); + if (failures === 0) { + try { + writeStorageFileSync(marker, `${STORAGE_PERMISSIONS_POLICY_VERSION}\n`, { + flag: "wx", + }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") failures++; + } + } + } return { tightened, failures }; }