Exercise IB document extraction on the hostile corpus

.github/workflows/idric-core.yml

@@ -26,6 +26,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Checkout canonical hostile-input corpus
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          repository: isomorphisms/ai-ci
+          ref: c1e423d506f6adfc8c1c9659a4faaf94ae84eaf0
+          path: .tools/ai-ci
+
       - name: Install system dependencies through Grease
         run: sh bin/ci_browser_foundation.grease install-dependencies
 
@@ -50,6 +57,15 @@ jobs:
       - name: Exercise HTML-first information renderer
         run: sh bin/ci_browser_foundation.grease exercise-information
 
+      - name: Exercise pinned hostile-input document receipts
+        run: sh bin/ci_browser_foundation.grease exercise-hostile-ingestion
+
+      - name: Exercise the pinned reusable AICI action
+        uses: isomorphisms/ai-ci/ingestion@c1e423d506f6adfc8c1c9659a4faaf94ae84eaf0
+        with:
+          receipt: /tmp/ib-valid-ingestion.receipt.tsv
+          role: consumer
+
       - name: Exercise browser-owned Idric core
         run: sh bin/ci_browser_foundation.grease exercise-core
 

bin/ci_browser_foundation.grease

@@ -5,6 +5,7 @@ repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
 idric_prefix=${IDRIC_PREFIX:-/tmp/idric}
 idric_ref=${IDRIC_REF:-61970be77769f607cca8650bf424c0f0b22ddee7}
 pdf_harvester_ref=${PDF_HARVESTER_REF:-77d85dc6f7e89d109cb9e06f42706bc34f00e4a3}
+aici_ref=${AICI_REF:-c1e423d506f6adfc8c1c9659a4faaf94ae84eaf0}
 
 install_dependencies() {
   sudo apt-get update
@@ -109,6 +110,47 @@ exercise_information() {
   sh tests/test_arxiv_progressive_prepaint.grease
 }
 
+exercise_hostile_ingestion() {
+  aici_root="$repository_root/.tools/ai-ci"
+  test -d "$aici_root/.git"
+  test "$(git -C "$aici_root" rev-parse HEAD)" = "$aici_ref"
+  (cd "$aici_root/ingestion/corpus-v1" && sha256sum -c CORPUS.sha256)
+  cc -std=c17 -Wall -Wextra -Werror -pedantic -O2 \
+    -o /tmp/aici-ingestion "$aici_root/ingestion/src/aici_ingestion.c"
+
+  cd "$repository_root/src"
+  "$idric_prefix/bin/idris2" HostileIngestionReceipt.idric \
+    -o ib-hostile-ingestion 2>&1 | tee /tmp/idric-hostile-ingestion-compile.txt
+  test -x build/exec/ib-hostile-ingestion
+  ! grep -q '^Error:' /tmp/idric-hostile-ingestion-compile.txt
+
+  implementation_revision=$(git -C "$repository_root" rev-parse HEAD)
+  executable_sha256=$(sha256sum \
+    build/exec/ib-hostile-ingestion_app/ib-hostile-ingestion.so | cut -d ' ' -f1)
+  corpus_root="$aici_root/ingestion/corpus-v1"
+
+  build/exec/ib-hostile-ingestion \
+    valid_utf8 "$corpus_root/bodies/valid.html" \
+    'Hostile fixture' 'reading list' /reading \
+    "$aici_ref" "$implementation_revision" "$executable_sha256" \
+    >/tmp/ib-valid-ingestion.receipt.tsv
+
+  build/exec/ib-hostile-ingestion \
+    malformed_html "$corpus_root/bodies/malformed.html" \
+    'Recovered fixture' syllabus /syllabus \
+    "$aici_ref" "$implementation_revision" "$executable_sha256" \
+    >/tmp/ib-malformed-ingestion.receipt.tsv
+
+  /tmp/aici-ingestion verify "$corpus_root" \
+    /tmp/ib-valid-ingestion.receipt.tsv consumer -
+  /tmp/aici-ingestion verify "$corpus_root" \
+    /tmp/ib-malformed-ingestion.receipt.tsv consumer -
+  grep -Fx "$(printf 'summary\tfirst_failure\tnone')" /tmp/ib-valid-ingestion.receipt.tsv
+  grep -Fx "$(printf 'summary\tfirst_failure\tnone')" /tmp/ib-malformed-ingestion.receipt.tsv
+  grep -F 'document_log_subset_v0' /tmp/ib-malformed-ingestion.receipt.tsv
+  grep -F 'expected_link_extracted' /tmp/ib-malformed-ingestion.receipt.tsv
+}
+
 exercise_scientific_media() {
   cd "$repository_root"
   sh -n lib/scientific_media.grease
@@ -151,13 +193,14 @@ case "${1:-}" in
   build-idric) build_idric ;;
   verify-pdf-harvester) verify_pdf_harvester ;;
   exercise-information) exercise_information ;;
+  exercise-hostile-ingestion) exercise_hostile_ingestion ;;
   exercise-core) exercise_core ;;
   exercise-workbench) exercise_workbench ;;
   exercise-scientific-media) exercise_scientific_media ;;
   exercise-live-arxiv) exercise_live_arxiv ;;
   exercise-live-arxiv-prepaint) exercise_live_arxiv_prepaint ;;
   *)
-    printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2
+    printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-hostile-ingestion|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2
     exit 2
     ;;
 esac

src/HostileIngestionReceipt.idric

@@ -0,0 +1,131 @@
+module Main
+
+import IB.Information
+import IB.RecoveredInformation
+import System
+import System.File
+
+emit_meta : String → String → IO ()
+emit_meta key value = putStrLn ("meta\t" ++ key ++ "\t" ++ value)
+
+emit_stage : String → String → String → String → String → String → IO ()
+emit_stage fixture stage status code owner evidence =
+  putStrLn
+    ("stage\t" ++ fixture ++ "\t" ++ stage ++ "\t" ++ status ++ "\t" ++
+     code ++ "\t" ++ owner ++ "\t" ++ evidence)
+
+emit_summary : String → String → IO ()
+emit_summary first_failure first_incomplete = do
+  putStrLn ("summary\tfirst_failure\t" ++ first_failure)
+  putStrLn ("summary\tfirst_incomplete\t" ++ first_incomplete)
+
+headers : String → String → String → String → IO ()
+headers fixture corpus_repository_revision implementation_revision executable_sha256 = do
+  emit_meta "schema" "aici-ingestion-receipt-v1"
+  emit_meta "corpus_revision" "hostile-web-v1-2026-08-30"
+  emit_meta "corpus_repository_revision" corpus_repository_revision
+  emit_meta "fixture_id" fixture
+  emit_meta "implementation_role" "consumer"
+  emit_meta "implementation_name" "ib-information-idric"
+  emit_meta "implementation_revision" implementation_revision
+  emit_meta "executable_sha256" executable_sha256
+  emit_meta "fallback" "none"
+  emit_meta "scope" "local-document-consumer"
+  emit_meta "producer" "idric"
+
+blocked_after_input : String → IO ()
+blocked_after_input fixture = do
+  emit_stage fixture "network" "SKIP" "blocked_by_input_acquisition" "ib" "not-run"
+  emit_stage fixture "decompression" "SKIP" "blocked_by_input_acquisition" "ib" "not-run"
+  emit_stage fixture "decoding" "SKIP" "blocked_by_input_acquisition" "ib" "not-run"
+  emit_stage fixture "html_recovery" "SKIP" "blocked_by_input_acquisition" "ib" "not-run"
+  emit_stage fixture "document_construction" "SKIP" "blocked_by_input_acquisition" "ib" "not-run"
+  emit_stage fixture "downstream_extraction" "SKIP" "blocked_by_input_acquisition" "ib" "not-run"
+  emit_summary "input_acquisition" "input_acquisition"
+
+blocked_after_recovery : String → IO ()
+blocked_after_recovery fixture = do
+  emit_stage fixture "document_construction" "SKIP" "blocked_by_html_recovery" "idric" "not-run"
+  emit_stage fixture "downstream_extraction" "SKIP" "blocked_by_html_recovery" "idric" "not-run"
+  emit_summary "html_recovery" "html_recovery"
+
+recovery_contract_holds : String → RecoveredDocumentLog → Bool
+recovery_contract_holds fixture log =
+  document_log_source log == fixture &&
+  (if fixture == "malformed_html"
+    then has_implied_end "p" log &&
+         has_implied_end "td" log &&
+         has_implied_end "tr" log &&
+         has_implied_end "a" log
+    else True)
+
+document_contract_holds : String → InformationView → Bool
+document_contract_holds fixture view =
+  if fixture == "malformed_html"
+    then has_table_row ["A", "B"] view
+    else True
+
+run : String → String → String → String → String → String → String → String → IO ()
+run fixture path expected_title expected_link_text expected_link_href
+    corpus_repository_revision implementation_revision executable_sha256 = do
+  headers fixture corpus_repository_revision implementation_revision executable_sha256
+  input ← readFile path
+  case input of
+    Left _ ⇒ do
+      emit_stage fixture "input_acquisition" "FAIL" "local_fixture_read_failure" "idric" "read-failed"
+      blocked_after_input fixture
+      exitFailure
+    Right html ⇒ do
+      emit_stage fixture "input_acquisition" "PASS" "local_fixture_read" "idric" "canonical-corpus"
+      emit_stage fixture "network" "SKIP" "not_applicable_local_fixture" "ib" "not-run"
+      emit_stage fixture "decompression" "PASS" "identity" "idric" "identity"
+      emit_stage fixture "decoding" "PASS" "utf8_valid" "idric"
+        ("characters=" ++ show (length (unpack html)))
+      let log = recover_document_log fixture html
+      if not (recovery_contract_holds fixture log)
+        then do
+          emit_stage fixture "html_recovery" "FAIL" "document_log_contract_violation" "idric"
+            ("contract=" ++ document_log_contract)
+          blocked_after_recovery fixture
+          exitFailure
+        else do
+          emit_stage fixture "html_recovery" "PASS" document_log_contract "idric"
+            ("source=" ++ document_log_source log)
+          let view = information_from_log "fixture://hostile" "fixture://hostile" log
+          if view_title view /= expected_title || not (document_contract_holds fixture view)
+            then do
+              emit_stage fixture "document_construction" "FAIL" "useful_document_subset_missing" "idric"
+                ("title=" ++ view_title view)
+              emit_stage fixture "downstream_extraction" "SKIP"
+                "blocked_by_document_construction" "idric" "not-run"
+              emit_summary "document_construction" "document_construction"
+              exitFailure
+            else do
+              emit_stage fixture "document_construction" "PASS" "information_view_built" "idric"
+                ("title=" ++ expected_title)
+              if has_link expected_link_text expected_link_href view
+                then do
+                  emit_stage fixture "downstream_extraction" "PASS" "expected_link_extracted" "idric"
+                    ("href=" ++ expected_link_href)
+                  emit_summary "none" "none"
+                else do
+                  emit_stage fixture "downstream_extraction" "FAIL" "expected_link_missing" "idric"
+                    ("href=" ++ expected_link_href)
+                  emit_summary "downstream_extraction" "downstream_extraction"
+                  exitFailure
+
+usage : IO ()
+usage = do
+  putStrLn
+    "usage: ib-hostile-ingestion FIXTURE PATH TITLE LINK_TEXT LINK_HREF CORPUS_REPOSITORY_REVISION IMPLEMENTATION_REVISION EXECUTABLE_SHA256"
+  exitFailure
+
+main : IO ()
+main = do
+  arguments ← getArgs
+  case arguments of
+    _ :: fixture :: path :: expected_title :: expected_link_text :: expected_link_href ::
+      corpus_repository_revision :: implementation_revision :: executable_sha256 :: [] ⇒
+        run fixture path expected_title expected_link_text expected_link_href
+          corpus_repository_revision implementation_revision executable_sha256
+    _ ⇒ usage