fix: poll directories the system has no watcher left for, instead of reporting their files as removed

.changeset/watcher-limit-polling.md

@@ -0,0 +1,5 @@
+---
+"watchpack": patch
+---
+
+When the system runs out of file watchers (`ENOSPC`/`EMFILE`, e.g. `fs.inotify.max_user_watches` on Linux), poll the directories it cannot watch instead of reporting their files as removed and retrying in a loop, and log the error once with a hint (fixes #46).

lib/DirectoryWatcher.js

@@ -25,6 +25,12 @@ const EXISTANCE_ONLY_TIME_ENTRY = Object.freeze({});
 
 let FS_ACCURACY = 2000;
 
+// the OS has no watcher left for the directory, which is still there
+const WATCHER_LIMIT_ERRORS = new Set(["ENOSPC", "EMFILE"]);
+// the polling interval of `poll: true`
+const WATCHER_LIMIT_POLL_INTERVAL = 5007;
+let watcherLimitReported = false;
+
 const IS_OSX = require("os").platform() === "darwin";
 const IS_WIN = require("os").platform() === "win32";
 
@@ -719,6 +725,16 @@ class DirectoryWatcher extends EventEmitter {
 	onWatcherError(err) {
 		if (this.closed) return;
 		if (err) {
+			if (
+				WATCHER_LIMIT_ERRORS.has(
+					/** @type {string} */ (
+						/** @type {NodeJS.ErrnoException} */ (err).code
+					),
+				)
+			) {
+				this.pollAfterWatcherLimit(err);
+				return;
+			}
 			if (
 				/** @type {NodeJS.ErrnoException} */
 				(err).code !== "EPERM" &&
@@ -732,6 +748,25 @@ class DirectoryWatcher extends EventEmitter {
 		}
 	}
 
+	/**
+	 * Polls the directory, which still exists, when the OS cannot watch it.
+	 * @param {unknown} err watcher limit error
+	 */
+	pollAfterWatcherLimit(err) {
+		if (!watcherLimitReported) {
+			watcherLimitReported = true;
+			// eslint-disable-next-line no-console
+			console.error(
+				`Watchpack Error (watcher): ${err}\nWatchpack polls the directories it cannot watch. Raise the limit of the system (e.g. fs.inotify.max_user_watches on Linux) or set WATCHPACK_POLLING.`,
+			);
+		}
+		if (this.watcher) this.watcher.close();
+		this.polledWatching = WATCHER_LIMIT_POLL_INTERVAL;
+		this.createWatcher();
+		// a finishing initial scan starts the polling by itself
+		if (!this.initialScan) this.doScan(false);
+	}
+
 	/**
 	 * @param {Error | NodeJS.ErrnoException=} err error
 	 */

test/WatcherLimit.test.js

@@ -0,0 +1,81 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const Watchpack = require("../lib");
+const TestHelper = require("./helpers/TestHelper");
+
+const fixtures = path.join(__dirname, "fixtures");
+const testHelper = new TestHelper(fixtures);
+
+// eslint-disable-next-line jest/no-confusing-set-timeout
+jest.setTimeout(20000);
+
+describe("Watcher limit of the system (#46)", () => {
+	beforeEach((done) => {
+		testHelper.before(done);
+	});
+
+	/** @type {InstanceType<typeof Watchpack>[]} */
+	let watchers = [];
+
+	afterEach((done) => {
+		// closed here, not in the test, so a failure cannot leave one running
+		for (const w of watchers) w.close();
+		watchers = [];
+		jest.restoreAllMocks();
+		testHelper.after(done);
+	});
+
+	it("should poll a directory it cannot watch instead of reporting its files as removed", (done) => {
+		const dir = path.join(fixtures, "dir");
+		const file = path.join(dir, "a");
+		testHelper.dir("dir");
+		testHelper.file(path.join("dir", "a"));
+		const { watch } = fs;
+		let watchCalls = 0;
+		jest.spyOn(fs, "watch").mockImplementation((target, ...args) => {
+			if (target === dir) {
+				watchCalls++;
+				throw Object.assign(
+					new Error(
+						`ENOSPC: System limit for number of file watchers reached, watch '${dir}'`,
+					),
+					{ code: "ENOSPC" },
+				);
+			}
+			return /** @type {(...a: unknown[]) => fs.FSWatcher} */ (watch)(
+				target,
+				...args,
+			);
+		});
+		const errors = jest.spyOn(console, "error").mockImplementation(() => {});
+		testHelper.tick(1000, () => {
+			const w = new Watchpack({ aggregateTimeout: 100 });
+			watchers.push(w);
+			/** @type {string[]} */
+			const removed = [];
+			w.on("remove", (item) => {
+				removed.push(item);
+			});
+			let edited = false;
+			w.on("change", (item) => {
+				// coarse timestamps can report the file in the initial scan
+				if (!edited) return;
+				expect(item).toBe(file);
+				expect(removed).toEqual([]);
+				// tried once, not again and again
+				expect(watchCalls).toBeLessThanOrEqual(1);
+				expect(
+					errors.mock.calls.filter((call) => /ENOSPC/.test(String(call[0]))),
+				).toHaveLength(watchCalls);
+				done();
+			});
+			w.watch({ files: [file], startTime: Date.now() });
+			testHelper.tick(1000, () => {
+				edited = true;
+				testHelper.file(path.join("dir", "a"));
+			});
+		});
+	});
+});

types/DirectoryWatcher.d.ts

@@ -151,6 +151,11 @@ declare class DirectoryWatcher extends EventEmitter<{
 	 * @param {unknown=} err error
 	 */
 	onWatcherError(err?: unknown | undefined): void;
+	/**
+	 * Polls the directory, which still exists, when the OS cannot watch it.
+	 * @param {unknown} err watcher limit error
+	 */
+	pollAfterWatcherLimit(err: unknown): void;
 	/**
 	 * @param {Error | NodeJS.ErrnoException=} err error
 	 */