diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 0d3bffd..0b1e027 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -26,6 +26,13 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Checkout canonical hostile-input corpus + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + repository: isomorphisms/ai-ci + ref: c1e423d506f6adfc8c1c9659a4faaf94ae84eaf0 + path: .tools/ai-ci + - name: Install system dependencies through Grease run: sh bin/ci_browser_foundation.grease install-dependencies @@ -50,6 +57,15 @@ jobs: - name: Exercise HTML-first information renderer run: sh bin/ci_browser_foundation.grease exercise-information + - name: Exercise pinned hostile-input document receipts + run: sh bin/ci_browser_foundation.grease exercise-hostile-ingestion + + - name: Exercise the pinned reusable AICI action + uses: isomorphisms/ai-ci/ingestion@c1e423d506f6adfc8c1c9659a4faaf94ae84eaf0 + with: + receipt: /tmp/ib-valid-ingestion.receipt.tsv + role: consumer + - name: Exercise browser-owned Idric core run: sh bin/ci_browser_foundation.grease exercise-core diff --git a/bin/ci_browser_foundation.grease b/bin/ci_browser_foundation.grease index 105a255..e4277ab 100755 --- a/bin/ci_browser_foundation.grease +++ b/bin/ci_browser_foundation.grease @@ -5,6 +5,7 @@ repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) idric_prefix=${IDRIC_PREFIX:-/tmp/idric} idric_ref=${IDRIC_REF:-61970be77769f607cca8650bf424c0f0b22ddee7} pdf_harvester_ref=${PDF_HARVESTER_REF:-77d85dc6f7e89d109cb9e06f42706bc34f00e4a3} +aici_ref=${AICI_REF:-c1e423d506f6adfc8c1c9659a4faaf94ae84eaf0} install_dependencies() { sudo apt-get update @@ -109,6 +110,47 @@ exercise_information() { sh tests/test_arxiv_progressive_prepaint.grease } +exercise_hostile_ingestion() { + aici_root="$repository_root/.tools/ai-ci" + test -d "$aici_root/.git" + test "$(git -C "$aici_root" rev-parse HEAD)" = "$aici_ref" + (cd "$aici_root/ingestion/corpus-v1" && sha256sum -c CORPUS.sha256) + cc -std=c17 -Wall -Wextra -Werror -pedantic -O2 \ + -o /tmp/aici-ingestion "$aici_root/ingestion/src/aici_ingestion.c" + + cd "$repository_root/src" + "$idric_prefix/bin/idris2" HostileIngestionReceipt.idric \ + -o ib-hostile-ingestion 2>&1 | tee /tmp/idric-hostile-ingestion-compile.txt + test -x build/exec/ib-hostile-ingestion + ! grep -q '^Error:' /tmp/idric-hostile-ingestion-compile.txt + + implementation_revision=$(git -C "$repository_root" rev-parse HEAD) + executable_sha256=$(sha256sum \ + build/exec/ib-hostile-ingestion_app/ib-hostile-ingestion.so | cut -d ' ' -f1) + corpus_root="$aici_root/ingestion/corpus-v1" + + build/exec/ib-hostile-ingestion \ + valid_utf8 "$corpus_root/bodies/valid.html" \ + 'Hostile fixture' 'reading list' /reading \ + "$aici_ref" "$implementation_revision" "$executable_sha256" \ + >/tmp/ib-valid-ingestion.receipt.tsv + + build/exec/ib-hostile-ingestion \ + malformed_html "$corpus_root/bodies/malformed.html" \ + 'Recovered fixture' syllabus /syllabus \ + "$aici_ref" "$implementation_revision" "$executable_sha256" \ + >/tmp/ib-malformed-ingestion.receipt.tsv + + /tmp/aici-ingestion verify "$corpus_root" \ + /tmp/ib-valid-ingestion.receipt.tsv consumer - + /tmp/aici-ingestion verify "$corpus_root" \ + /tmp/ib-malformed-ingestion.receipt.tsv consumer - + grep -Fx "$(printf 'summary\tfirst_failure\tnone')" /tmp/ib-valid-ingestion.receipt.tsv + grep -Fx "$(printf 'summary\tfirst_failure\tnone')" /tmp/ib-malformed-ingestion.receipt.tsv + grep -F 'document_log_subset_v0' /tmp/ib-malformed-ingestion.receipt.tsv + grep -F 'expected_link_extracted' /tmp/ib-malformed-ingestion.receipt.tsv +} + exercise_scientific_media() { cd "$repository_root" sh -n lib/scientific_media.grease @@ -151,13 +193,14 @@ case "${1:-}" in build-idric) build_idric ;; verify-pdf-harvester) verify_pdf_harvester ;; exercise-information) exercise_information ;; + exercise-hostile-ingestion) exercise_hostile_ingestion ;; exercise-core) exercise_core ;; exercise-workbench) exercise_workbench ;; exercise-scientific-media) exercise_scientific_media ;; exercise-live-arxiv) exercise_live_arxiv ;; exercise-live-arxiv-prepaint) exercise_live_arxiv_prepaint ;; *) - printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2 + printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-hostile-ingestion|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2 exit 2 ;; esac diff --git a/src/HostileIngestionReceipt.idric b/src/HostileIngestionReceipt.idric new file mode 100644 index 0000000..719ab16 --- /dev/null +++ b/src/HostileIngestionReceipt.idric @@ -0,0 +1,131 @@ +module Main + +import IB.Information +import IB.RecoveredInformation +import System +import System.File + +emit_meta : String → String → IO () +emit_meta key value = putStrLn ("meta\t" ++ key ++ "\t" ++ value) + +emit_stage : String → String → String → String → String → String → IO () +emit_stage fixture stage status code owner evidence = + putStrLn + ("stage\t" ++ fixture ++ "\t" ++ stage ++ "\t" ++ status ++ "\t" ++ + code ++ "\t" ++ owner ++ "\t" ++ evidence) + +emit_summary : String → String → IO () +emit_summary first_failure first_incomplete = do + putStrLn ("summary\tfirst_failure\t" ++ first_failure) + putStrLn ("summary\tfirst_incomplete\t" ++ first_incomplete) + +headers : String → String → String → String → IO () +headers fixture corpus_repository_revision implementation_revision executable_sha256 = do + emit_meta "schema" "aici-ingestion-receipt-v1" + emit_meta "corpus_revision" "hostile-web-v1-2026-08-30" + emit_meta "corpus_repository_revision" corpus_repository_revision + emit_meta "fixture_id" fixture + emit_meta "implementation_role" "consumer" + emit_meta "implementation_name" "ib-information-idric" + emit_meta "implementation_revision" implementation_revision + emit_meta "executable_sha256" executable_sha256 + emit_meta "fallback" "none" + emit_meta "scope" "local-document-consumer" + emit_meta "producer" "idric" + +blocked_after_input : String → IO () +blocked_after_input fixture = do + emit_stage fixture "network" "SKIP" "blocked_by_input_acquisition" "ib" "not-run" + emit_stage fixture "decompression" "SKIP" "blocked_by_input_acquisition" "ib" "not-run" + emit_stage fixture "decoding" "SKIP" "blocked_by_input_acquisition" "ib" "not-run" + emit_stage fixture "html_recovery" "SKIP" "blocked_by_input_acquisition" "ib" "not-run" + emit_stage fixture "document_construction" "SKIP" "blocked_by_input_acquisition" "ib" "not-run" + emit_stage fixture "downstream_extraction" "SKIP" "blocked_by_input_acquisition" "ib" "not-run" + emit_summary "input_acquisition" "input_acquisition" + +blocked_after_recovery : String → IO () +blocked_after_recovery fixture = do + emit_stage fixture "document_construction" "SKIP" "blocked_by_html_recovery" "idric" "not-run" + emit_stage fixture "downstream_extraction" "SKIP" "blocked_by_html_recovery" "idric" "not-run" + emit_summary "html_recovery" "html_recovery" + +recovery_contract_holds : String → RecoveredDocumentLog → Bool +recovery_contract_holds fixture log = + document_log_source log == fixture && + (if fixture == "malformed_html" + then has_implied_end "p" log && + has_implied_end "td" log && + has_implied_end "tr" log && + has_implied_end "a" log + else True) + +document_contract_holds : String → InformationView → Bool +document_contract_holds fixture view = + if fixture == "malformed_html" + then has_table_row ["A", "B"] view + else True + +run : String → String → String → String → String → String → String → String → IO () +run fixture path expected_title expected_link_text expected_link_href + corpus_repository_revision implementation_revision executable_sha256 = do + headers fixture corpus_repository_revision implementation_revision executable_sha256 + input ← readFile path + case input of + Left _ ⇒ do + emit_stage fixture "input_acquisition" "FAIL" "local_fixture_read_failure" "idric" "read-failed" + blocked_after_input fixture + exitFailure + Right html ⇒ do + emit_stage fixture "input_acquisition" "PASS" "local_fixture_read" "idric" "canonical-corpus" + emit_stage fixture "network" "SKIP" "not_applicable_local_fixture" "ib" "not-run" + emit_stage fixture "decompression" "PASS" "identity" "idric" "identity" + emit_stage fixture "decoding" "PASS" "utf8_valid" "idric" + ("characters=" ++ show (length (unpack html))) + let log = recover_document_log fixture html + if not (recovery_contract_holds fixture log) + then do + emit_stage fixture "html_recovery" "FAIL" "document_log_contract_violation" "idric" + ("contract=" ++ document_log_contract) + blocked_after_recovery fixture + exitFailure + else do + emit_stage fixture "html_recovery" "PASS" document_log_contract "idric" + ("source=" ++ document_log_source log) + let view = information_from_log "fixture://hostile" "fixture://hostile" log + if view_title view /= expected_title || not (document_contract_holds fixture view) + then do + emit_stage fixture "document_construction" "FAIL" "useful_document_subset_missing" "idric" + ("title=" ++ view_title view) + emit_stage fixture "downstream_extraction" "SKIP" + "blocked_by_document_construction" "idric" "not-run" + emit_summary "document_construction" "document_construction" + exitFailure + else do + emit_stage fixture "document_construction" "PASS" "information_view_built" "idric" + ("title=" ++ expected_title) + if has_link expected_link_text expected_link_href view + then do + emit_stage fixture "downstream_extraction" "PASS" "expected_link_extracted" "idric" + ("href=" ++ expected_link_href) + emit_summary "none" "none" + else do + emit_stage fixture "downstream_extraction" "FAIL" "expected_link_missing" "idric" + ("href=" ++ expected_link_href) + emit_summary "downstream_extraction" "downstream_extraction" + exitFailure + +usage : IO () +usage = do + putStrLn + "usage: ib-hostile-ingestion FIXTURE PATH TITLE LINK_TEXT LINK_HREF CORPUS_REPOSITORY_REVISION IMPLEMENTATION_REVISION EXECUTABLE_SHA256" + exitFailure + +main : IO () +main = do + arguments ← getArgs + case arguments of + _ :: fixture :: path :: expected_title :: expected_link_text :: expected_link_href :: + corpus_repository_revision :: implementation_revision :: executable_sha256 :: [] ⇒ + run fixture path expected_title expected_link_text expected_link_href + corpus_repository_revision implementation_revision executable_sha256 + _ ⇒ usage diff --git a/src/IB/RecoveredInformation.idric b/src/IB/RecoveredInformation.idric new file mode 100644 index 0000000..0850135 --- /dev/null +++ b/src/IB/RecoveredInformation.idric @@ -0,0 +1,182 @@ +module IB.RecoveredInformation + +import IB.Information + +%default total + +data HtmlToken + = HtmlText String + | HtmlTag String + +public export +data DocumentEvent + = StartTag String String + | EndTag String + | ImpliedEndTag String + | TextEvent String + +public export +data RecoveredDocumentLog = DocumentLog String (List DocumentEvent) + +data RecoveryState = RecoverState (Maybe String) (Maybe String) Bool Bool + +finish_text_token : List Char → List HtmlToken → List HtmlToken +finish_text_token chars tokens = + let text = pack (reverse chars) in + if text == "" then tokens else HtmlText text :: tokens + +finish_tag_token : List Char → List HtmlToken → List HtmlToken +finish_tag_token chars tokens = HtmlTag (pack (reverse chars)) :: tokens + +tokenize_chars : Bool → List Char → List Char → List HtmlToken → List HtmlToken +tokenize_chars in_tag current [] tokens = + reverse (if in_tag then finish_tag_token current tokens else finish_text_token current tokens) +tokenize_chars False current ('<' :: rest) tokens = + tokenize_chars True [] rest (finish_text_token current tokens) +tokenize_chars False current (value :: rest) tokens = + tokenize_chars False (value :: current) rest tokens +tokenize_chars True current ('>' :: rest) tokens = + tokenize_chars False [] rest (finish_tag_token current tokens) +tokenize_chars True current (value :: rest) tokens = + tokenize_chars True (value :: current) rest tokens + +tokenize : String → List HtmlToken +tokenize text = tokenize_chars False [] (unpack text) [] + +is_space_char : Char → Bool +is_space_char value = + value == ' ' || value == '\n' || value == '\r' || value == '\t' + +drop_space_chars : List Char → List Char +drop_space_chars [] = [] +drop_space_chars (value :: rest) = + if is_space_char value then drop_space_chars rest else value :: rest + +is_closing_tag : String → Bool +is_closing_tag tag = + case drop_space_chars (unpack tag) of + '/' :: _ ⇒ True + _ ⇒ False + +heading_tag : String → Bool +heading_tag name = elem name ["h1", "h2", "h3", "h4", "h5", "h6"] + +paragraph_boundary : String → Bool +paragraph_boundary name = + heading_tag name || + elem name ["table", "tr", "div", "section", "article", "aside", "nav", + "ul", "ol", "form"] + +close_matching : String → Maybe String → Maybe String +close_matching wanted Nothing = Nothing +close_matching wanted (Just value) = + if wanted == value then Nothing else Just value + +close_maybe : Maybe String → List DocumentEvent +close_maybe Nothing = [] +close_maybe (Just name) = [ImpliedEndTag name] + +eof_events : RecoveryState → List DocumentEvent +eof_events (RecoverState paragraph cell row_open link_open) = + (if link_open then [ImpliedEndTag "a"] else []) ++ + close_maybe cell ++ + (if row_open then [ImpliedEndTag "tr"] else []) ++ + close_maybe paragraph + +recover_tokens : List HtmlToken → RecoveryState → List DocumentEvent +recover_tokens [] state = eof_events state +recover_tokens (HtmlText text :: rest) state = + TextEvent text :: recover_tokens rest state +recover_tokens (HtmlTag tag :: rest) (RecoverState paragraph cell row_open link_open) = + let name = tag_name tag in + if is_closing_tag tag + then + if name == "p" || name == "li" + then EndTag name :: recover_tokens rest + (RecoverState (close_matching name paragraph) cell row_open link_open) + else if name == "td" || name == "th" + then EndTag name :: recover_tokens rest + (RecoverState paragraph Nothing row_open link_open) + else if name == "tr" + then close_maybe cell ++ [EndTag "tr"] ++ recover_tokens rest + (RecoverState paragraph Nothing False link_open) + else if name == "table" + then close_maybe cell ++ + (if row_open then [ImpliedEndTag "tr"] else []) ++ + [EndTag "table"] ++ recover_tokens rest + (RecoverState paragraph Nothing False link_open) + else if name == "a" + then EndTag "a" :: recover_tokens rest + (RecoverState paragraph cell row_open False) + else EndTag name :: recover_tokens rest + (RecoverState paragraph cell row_open link_open) + else + if name == "p" || name == "li" + then close_maybe paragraph ++ [StartTag name tag] ++ recover_tokens rest + (RecoverState (Just name) cell row_open link_open) + else if name == "tr" + then close_maybe paragraph ++ close_maybe cell ++ + (if row_open then [ImpliedEndTag "tr"] else []) ++ + [StartTag name tag] ++ recover_tokens rest + (RecoverState Nothing Nothing True link_open) + else if name == "td" || name == "th" + then close_maybe cell ++ [StartTag name tag] ++ recover_tokens rest + (RecoverState paragraph (Just name) row_open link_open) + else if name == "a" + then (if link_open then [ImpliedEndTag "a"] else []) ++ + [StartTag name tag] ++ recover_tokens rest + (RecoverState paragraph cell row_open True) + else if paragraph_boundary name + then close_maybe paragraph ++ [StartTag name tag] ++ recover_tokens rest + (RecoverState Nothing cell row_open link_open) + else StartTag name tag :: recover_tokens rest + (RecoverState paragraph cell row_open link_open) + +public export +recover_document_log : String → String → RecoveredDocumentLog +recover_document_log source html = + DocumentLog source + (recover_tokens (tokenize html) (RecoverState Nothing Nothing False False)) + +public export +document_log_contract : String +document_log_contract = "document_log_subset_v0" + +public export +document_log_source : RecoveredDocumentLog → String +document_log_source (DocumentLog source _) = source + +public export +document_log_events : RecoveredDocumentLog → List DocumentEvent +document_log_events (DocumentLog _ events) = events + +has_implied_end_event : String → List DocumentEvent → Bool +has_implied_end_event wanted [] = False +has_implied_end_event wanted (ImpliedEndTag name :: rest) = + name == wanted || has_implied_end_event wanted rest +has_implied_end_event wanted (_ :: rest) = has_implied_end_event wanted rest + +public export +has_implied_end : String → RecoveredDocumentLog → Bool +has_implied_end wanted (DocumentLog _ events) = + has_implied_end_event wanted events + +render_event : DocumentEvent → String +render_event (StartTag _ tag) = "<" ++ tag ++ ">" +render_event (EndTag name) = "" +render_event (ImpliedEndTag name) = "" +render_event (TextEvent text) = text + +render_events : List DocumentEvent → String +render_events [] = "" +render_events (event :: rest) = render_event event ++ render_events rest + +public export +information_from_log : String → String → RecoveredDocumentLog → InformationView +information_from_log requested_url resolved_url (DocumentLog _ events) = + extract_html requested_url resolved_url (render_events events) + +public export +extract_recovered_html : String → String → String → String → InformationView +extract_recovered_html source requested_url resolved_url html = + information_from_log requested_url resolved_url (recover_document_log source html) diff --git a/tests/hostile-ingestion/README.md b/tests/hostile-ingestion/README.md new file mode 100644 index 0000000..785df47 --- /dev/null +++ b/tests/hostile-ingestion/README.md @@ -0,0 +1,21 @@ +# IB hostile-ingestion consumer + +IB consumes the exact canonical corpus revision in `ai-ci.lock`; it does not +copy the fixtures or call the external oracle. + +The Idriç candidate path now inserts `IB.RecoveredInformation` between raw HTML +tokenization and `InformationView`. Its first declared recovery contract is the +pinned corpus's `document_log_subset_v0`: source fixture identity plus ordered +tag/text events, with only the implied-end recovery needed by the useful-document +subset. It is not a browser DOM and does not claim full WHATWG recovery. + +The malformed fixture exercises that contract directly. Before document +construction can pass, the recovered log must retain its source identity and +show implied closure for the malformed paragraph, table cell/row, and the +unterminated link at EOF. `InformationView` construction then has to retain the +recovered `A`/`B` table row, and downstream extraction has to recover the +`syllabus` link at `/syllabus`. + +Network remains `SKIP/not_applicable_local_fixture` here. The fixture bytes are +read locally from the pinned `ai-ci` corpus, the external oracle is never called +or used as fallback, and network ownership remains in ICU rather than IB. diff --git a/tests/hostile-ingestion/ai-ci.lock b/tests/hostile-ingestion/ai-ci.lock new file mode 100644 index 0000000..eaddf91 --- /dev/null +++ b/tests/hostile-ingestion/ai-ci.lock @@ -0,0 +1,4 @@ +repository isomorphisms/ai-ci +repository_revision c1e423d506f6adfc8c1c9659a4faaf94ae84eaf0 +corpus_revision hostile-web-v1-2026-08-30 +consumer_role ib-document-extraction