security(ignore): preserve fail-closed sibling pending canonical TOCTOU repair

.jules/sentinel.md

@@ -99,3 +99,7 @@
 **Root cause:** The protected implementation added canonical names to the exclusion set but did not compare each observed directory entry through a locale-stable normalized key.
 **Prevention:** Build one `Locale.ROOT` lowercase set from the canonical sensitive names, compare every observed name against it, and add the original spelling to the exclusion set so downstream exact membership remains correct.
 **Evidence:** `testProcessIgnoreFileTreatsSensitiveNamesCaseInsensitively` failed on test-only commit `472b916cd40f70693c4e1eb48956042a25353feb` (CI run `31469596932`) and passed with the source fix at `bb113d858ccfc42ddaecf6729749b238e5ade2d0` (CI run `31469921661`).
+## 2025-05-24 - [CRITICAL] Fail-closed Security Policy Files (.html4ignore)
+**Vulnerability:** TOCTOU and Policy bypass via broken/invalid .html4ignore symlinks or directories.
+**Learning:** Security policy files (like .html4ignore) were previously ignoring unreadable or invalid variants (e.g. symlinks, directories, unreadable files). This meant that if a policy file couldn't be read (due to TOCTOU manipulation or broken symlinks), the application failed open and generated the index anyway, potentially exposing sensitive files that should have been excluded.
+**Prevention:** Implement `IgnoreFileReadException` and fail-closed architecture. When encountering a security policy file, check if it exists or is a symlink first, then explicitly verify it's a valid readable file. If invalid/unreadable, throw the exception to abruptly stop traversal (suppress publication) for that specific directory subtree.

src/main/kotlin/html4tree/main.kt

@@ -200,20 +200,27 @@ internal fun crawl_directories(
         val dirFilesNames = dirFiles?.let { files ->
             Array(files.size) { index -> files[index].name }
         }
-        val exclude = processIgnoreFile(lle.file, dirFilesNames)
-
-        if(maxLevel == -1 || currentLevel <= maxLevel)
-           processDirectory(lle.file, exclude, dirFiles)
-
-        if(maxLevel == -1 || currentLevel < maxLevel) {
-            dirFiles?.forEach {
-                // ⚡ Bolt Performance Optimization: Short-circuit OS stat calls
-                // by checking cheap in-memory string exclusion rules first
-                if(!it.name.isHiddenFile() && it.name !in exclude) {
-                    val childAttrs = readAttributes(it)
-                    if(childAttrs != null && childAttrs.isDirectory && !childAttrs.isSymbolicLink) {
-                        val childEntry = LinkedListEntry(it, currentLevel+1, readIdentity(it).key)
-                        ll.push(childEntry)
+
+        val exclude = try {
+            processIgnoreFile(lle.file, dirFilesNames)
+        } catch (e: IgnoreFileReadException) {
+            null
+        }
+
+        if (exclude != null) {
+            if(maxLevel == -1 || currentLevel <= maxLevel)
+               processDirectory(lle.file, exclude, dirFiles)
+
+            if(maxLevel == -1 || currentLevel < maxLevel) {
+                dirFiles?.forEach {
+                    // ⚡ Bolt Performance Optimization: Short-circuit OS stat calls
+                    // by checking cheap in-memory string exclusion rules first
+                    if(!it.name.isHiddenFile() && it.name !in exclude) {
+                        val childAttrs = readAttributes(it)
+                        if(childAttrs != null && childAttrs.isDirectory && !childAttrs.isSymbolicLink) {
+                            val childEntry = LinkedListEntry(it, currentLevel+1, readIdentity(it).key)
+                            ll.push(childEntry)
+                        }
                     }
                 }
             }
@@ -293,6 +300,13 @@ fun String.urlEncodePath(): String {
     return encoded?.toString() ?: this
 }
 
+/**
+ * Exception thrown when a security policy file (e.g., .html4ignore) is detected
+ * but cannot be read or disappears (TOCTOU race condition).
+ * Enforces fail-closed behavior to suppress directory publication and traversal.
+ */
+class IgnoreFileReadException(message: String) : Exception(message)
+
 fun process_ignore_file(curr_dir: File, dirFilesNames: Array<String>? = null): Set<String> {
 
     val ignore_filename = ".html4ignore"
@@ -303,6 +317,13 @@ fun process_ignore_file(curr_dir: File, dirFilesNames: Array<String>? = null): S
 
     val files_to_exclude = mutableSetOf<String>()
 
+    val ignoreExists = ignore_file.exists() || Files.isSymbolicLink(ignore_file.toPath())
+    if (ignoreExists) {
+        if (!ignore_file.isFile || Files.isSymbolicLink(ignore_file.toPath()) || !ignore_file.canRead() || ignore_file.length() > 1048576) {
+            throw IgnoreFileReadException("Policy file $ignore_filename is inaccessible or invalid. Failing closed to prevent TOCTOU bypass.")
+        }
+    }
+
     // 보안 향상: .html4ignore 파일이 일반 파일인지 확인하고, 심볼릭 링크인 경우 무시하여 DoS 및 경로 조작을 방지합니다.
     // 보안 향상: 파일 크기(1MB 제한) 및 줄 수(1000줄), 정규식 길이(100자)를 제한하여 ReDoS 및 메모리 고갈(OOM) 방지
     // 보안 향상: 권한이 없는 파일 접근 시 발생하는 예외(DoS)를 방지하기 위해 canRead() 추가 확인

src/test/kotlin/html4tree/MainTest.kt

@@ -718,9 +718,36 @@ class MainTest {
         val ignoreDir = File(tempDir, ".html4ignore")
         ignoreDir.mkdir()
 
-        // This should not crash or parse the directory
-        val excluded = process_ignore_file(tempDir, null)
-        assertTrue(excluded.contains("index.html"))
+        // This should fail-closed and throw IgnoreFileReadException
+        assertFailsWith<IgnoreFileReadException>("Expected IgnoreFileReadException to be thrown") {
+            process_ignore_file(tempDir, null)
+        }
+    }
+
+    @Test
+    fun testCrawlDirectoriesIgnoreFileReadException() {
+        val ignoreDir = File(tempDir, ".html4ignore")
+        ignoreDir.mkdir()
+        val ll = LinkedList()
+        val topEntry = LinkedListEntry(tempDir, 0, read_file_identity(tempDir).key)
+        ll.push(topEntry)
+
+        crawl_directories(
+            ll = ll,
+            maxLevel = -1,
+            processIgnoreFile = { _, _ -> throw IgnoreFileReadException("mock") }
+        )
+    }
+
+    @Test
+    fun testIgnoreFileTooLarge() {
+        val ignoreFile = File(tempDir, ".html4ignore")
+        val largeBytes = ByteArray(1048576 + 1)
+        ignoreFile.writeBytes(largeBytes)
+
+        assertFailsWith<IgnoreFileReadException>("Expected IgnoreFileReadException to be thrown") {
+            process_ignore_file(tempDir, null)
+        }
     }
 
     @Test
@@ -762,10 +789,10 @@ class MainTest {
 
         File(tempDir, "test.txt").createNewFile()
 
-        // Should ignore the symlink and NOT parse it
-        val excluded = process_ignore_file(tempDir, null)
-        assertFalse(excluded.contains("test.txt"))
-        assertTrue(excluded.contains("index.html"))
+        // Should fail-closed and throw IgnoreFileReadException
+        assertFailsWith<IgnoreFileReadException>("Expected IgnoreFileReadException to be thrown") {
+            process_ignore_file(tempDir, null)
+        }
     }
 
     @Test
@@ -777,10 +804,10 @@ class MainTest {
 
         File(tempDir, "test.txt").createNewFile()
 
-        // Should ignore the file because it's too large
-        val excluded = process_ignore_file(tempDir, null)
-        assertFalse(excluded.contains("test.txt"))
-        assertTrue(excluded.contains("index.html"))
+        // Should fail-closed and throw IgnoreFileReadException
+        assertFailsWith<IgnoreFileReadException>("Expected IgnoreFileReadException to be thrown") {
+            process_ignore_file(tempDir, null)
+        }
     }
 
     @Test