From 511a436418a7abf66b6272d90a8793191ae2a86f Mon Sep 17 00:00:00 2001 From: i Date: Tue, 25 Aug 2026 09:57:38 -0400 Subject: [PATCH] Rebase prepaint network boundary onto current main --- .github/workflows/idric-core.yml | 3 + bin/ci_browser_foundation.grease | 26 ++++- bin/icu_search.grease | 18 ++++ bin/prepaint_predecision.grease | 118 +++++++++++++++++++++ docs/prepaint-policy-boundary.md | 107 +++++++++++++++++++ docs/url-recognition-strategy.txt | 114 ++++++++++++++++++++ src/IB/Information.idric | 140 +++++++++++++++++++++++-- src/InformationSmoke.idric | 30 ++++++ tests/test_icu_search.grease | 28 +++++ tests/test_prepaint_predecision.grease | 62 +++++++++++ 10 files changed, 638 insertions(+), 8 deletions(-) create mode 100755 bin/icu_search.grease create mode 100755 bin/prepaint_predecision.grease create mode 100644 docs/prepaint-policy-boundary.md create mode 100644 docs/url-recognition-strategy.txt create mode 100755 tests/test_icu_search.grease create mode 100755 tests/test_prepaint_predecision.grease diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 0d3bffd..ce0fbd5 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -29,6 +29,9 @@ jobs: - name: Install system dependencies through Grease run: sh bin/ci_browser_foundation.grease install-dependencies + - name: Exercise ICU search and bounded network pre-decision + run: sh bin/ci_browser_foundation.grease exercise-prepaint-network + - name: Restore Idric compiler id: idric-cache uses: actions/cache/restore@v4 diff --git a/bin/ci_browser_foundation.grease b/bin/ci_browser_foundation.grease index b3df9fe..f44975d 100755 --- a/bin/ci_browser_foundation.grease +++ b/bin/ci_browser_foundation.grease @@ -8,7 +8,7 @@ pdf_harvester_ref=${PDF_HARVESTER_REF:-77d85dc6f7e89d109cb9e06f42706bc34f00e4a3} install_dependencies() { sudo apt-get update - sudo apt-get install -y chezscheme curl xmlstarlet busybox poppler-utils + sudo apt-get install -y chezscheme curl xmlstarlet busybox poppler-utils jq } build_idric() { @@ -70,6 +70,16 @@ exercise_workbench() { grep -Fx 'resident-at-10=10' /tmp/ib-workbench.txt } +exercise_prepaint_network() { + cd "$repository_root" + sh -n bin/icu_search.grease + sh -n bin/prepaint_predecision.grease + sh -n tests/test_icu_search.grease + sh -n tests/test_prepaint_predecision.grease + sh tests/test_icu_search.grease + sh tests/test_prepaint_predecision.grease +} + build_information_programs() { test -x "$idric_prefix/bin/idris2" || build_idric cd "$repository_root/src" @@ -100,6 +110,17 @@ exercise_information() { grep -Fx 'phone-api-row=True' /tmp/ib-information-smoke.txt grep -Fx 'phone-lines=8' /tmp/ib-information-smoke.txt grep -Fx 'phone-width-bounded=True' /tmp/ib-information-smoke.txt + grep -Fx 'article-first-paragraph=True' /tmp/ib-information-smoke.txt + grep -Fx 'article-second-paragraph=True' /tmp/ib-information-smoke.txt + grep -Fx 'article-href=True' /tmp/ib-information-smoke.txt + grep -Fx 'article-inline-href=True' /tmp/ib-information-smoke.txt + grep -Fx 'article-inline-prefix=True' /tmp/ib-information-smoke.txt + grep -Fx 'article-inline-suffix=True' /tmp/ib-information-smoke.txt + grep -Fx 'article-linked-image=True' /tmp/ib-information-smoke.txt + grep -Fx 'article-image-empty-link=False' /tmp/ib-information-smoke.txt + grep -Fx 'plain-first-url=True' /tmp/ib-information-smoke.txt + grep -Fx 'plain-second-url=True' /tmp/ib-information-smoke.txt + grep -Fx 'cheap-url-rejects-relative=False' /tmp/ib-information-smoke.txt sh -n bin/prepaint_arxiv_progressively.grease sh -n tests/test_arxiv_progressive_prepaint.grease sh tests/test_arxiv_progressive_prepaint.grease @@ -149,11 +170,12 @@ case "${1:-}" in exercise-information) exercise_information ;; exercise-core) exercise_core ;; exercise-workbench) exercise_workbench ;; + exercise-prepaint-network) exercise_prepaint_network ;; exercise-scientific-media) exercise_scientific_media ;; exercise-live-arxiv) exercise_live_arxiv ;; exercise-live-arxiv-prepaint) exercise_live_arxiv_prepaint ;; *) - printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2 + printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-prepaint-network|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2 exit 2 ;; esac diff --git a/bin/icu_search.grease b/bin/icu_search.grease new file mode 100755 index 0000000..cd037cd --- /dev/null +++ b/bin/icu_search.grease @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu + +if test "$#" -eq 0; then + printf 'usage: %s SEARCH WORDS...\n' "$0" >&2 + exit 2 +fi + +icu_command=${IB_ICU:-icu} +jq_command=${IB_JQ:-jq} + +# With the ordinary space IFS, "$*" is the shell operation the original tiny +# search command was remembering. PS1 is only the interactive prompt. jq owns +# UTF-8 form encoding here; replacing %20 with + gives the familiar query form. +query=$* +encoded_query=$(printf '%s' "$query" | "$jq_command" -sRr @uri | sed 's/%20/+/g') + +exec "$icu_command" get "https://www.google.com/search?q=$encoded_query" diff --git a/bin/prepaint_predecision.grease b/bin/prepaint_predecision.grease new file mode 100755 index 0000000..6852304 --- /dev/null +++ b/bin/prepaint_predecision.grease @@ -0,0 +1,118 @@ +#!/bin/sh +set -eu + +if test "$#" -ne 1; then + printf 'usage: %s URL\n' "$0" >&2 + exit 2 +fi + +input_url=$1 +case $input_url in + http://*) authority_and_rest=${input_url#http://} ;; + https://*) authority_and_rest=${input_url#https://} ;; + *) + printf 'predecision: URL must begin with http:// or https://\n' >&2 + exit 2 + ;; +esac + +authority=${authority_and_rest%%/*} +authority=${authority%%\?*} +authority=${authority%%\#*} +case $authority in + ''|*@*|'['*) + printf 'predecision: unsupported or empty URL authority\n' >&2 + exit 2 + ;; +esac +host=${authority%%:*} + +dig_command=${IB_DIG:-dig} +traceroute_command=${IB_TRACEROUTE:-traceroute} +timeout_command=${IB_TIMEOUT:-timeout} + +work_directory=$(mktemp -d) +cleanup() { + rm -rf "$work_directory" +} +trap cleanup EXIT HUP INT TERM + +dig_status=unavailable +if command -v "$dig_command" >/dev/null 2>&1; then + if "$timeout_command" 3 "$dig_command" +tries=1 +time=1 +stats \ + "$host" A "$host" NS > "$work_directory/dig.txt" 2>&1 + then + dig_status=ok + else + dig_status=failed + fi +else + : > "$work_directory/dig.txt" +fi + +route_status=unavailable +if command -v "$traceroute_command" >/dev/null 2>&1; then + if "$timeout_command" 10 "$traceroute_command" -n -m 8 -w 1 -q 1 \ + "$host" > "$work_directory/traceroute.txt" 2>&1 + then + route_status=ok + else + route_status=failed + fi +else + : > "$work_directory/traceroute.txt" +fi + +dns_milliseconds=$(awk '/Query time:/ { print $(NF - 1); exit }' "$work_directory/dig.txt") +test -n "$dns_milliseconds" || dns_milliseconds=unknown +address_records=$(awk '$4 == "A" || $4 == "AAAA" { amount += 1 } END { print amount + 0 }' \ + "$work_directory/dig.txt") +namespace_records=$(awk '$4 == "NS" { amount += 1 } END { print amount + 0 }' \ + "$work_directory/dig.txt") +route_hops=$(awk '/^[[:space:]]*[0-9]+[[:space:]]/ { hop = $1 } END { if (hop == "") print 0; else print hop }' \ + "$work_directory/traceroute.txt") +route_last_milliseconds=$(awk ' + /^[[:space:]]*[0-9]+[[:space:]]/ { + for (field = 1; field <= NF; field += 1) { + if ($field == "ms" && field > 1) last = $(field - 1) + } + } + END { if (last == "") print "unknown"; else print last } +' "$work_directory/traceroute.txt") + +known_asset_count=${IB_KNOWN_ASSET_COUNT:-unknown} +known_asset_bytes=${IB_KNOWN_ASSET_BYTES:-unknown} +assumed_bytes_per_second=${IB_ASSUMED_BYTES_PER_SECOND:-unknown} +estimated_transfer_seconds=unknown +decision=measure-assets-before-heavy-renderer + +case $known_asset_bytes:$assumed_bytes_per_second in + *[!0-9:]*|:*|*:0) ;; + *) + estimated_transfer_seconds=$(( + (known_asset_bytes + assumed_bytes_per_second - 1) / assumed_bytes_per_second + )) + if test "$estimated_transfer_seconds" -ge 300; then + decision=keep-prepaint-unless-user-escalates + else + decision=prepaint-then-bounded-escalation + fi + ;; +esac + +printf 'predecision\t1\n' +printf 'url\t%s\n' "$input_url" +printf 'host\t%s\n' "$host" +printf 'dig-status\t%s\n' "$dig_status" +printf 'dns-ms\t%s\n' "$dns_milliseconds" +printf 'address-records\t%s\n' "$address_records" +printf 'namespace-records\t%s\n' "$namespace_records" +printf 'traceroute-status\t%s\n' "$route_status" +printf 'route-hops\t%s\n' "$route_hops" +printf 'route-last-ms\t%s\n' "$route_last_milliseconds" +printf 'known-asset-count\t%s\n' "$known_asset_count" +printf 'known-asset-bytes\t%s\n' "$known_asset_bytes" +printf 'assumed-bytes-per-second\t%s\n' "$assumed_bytes_per_second" +printf 'estimated-transfer-seconds\t%s\n' "$estimated_transfer_seconds" +printf 'decision\t%s\n' "$decision" +printf 'route-warning\t%s\n' 'DNS and traceroute are diagnostics, not page-load predictions.' diff --git a/docs/prepaint-policy-boundary.md b/docs/prepaint-policy-boundary.md new file mode 100644 index 0000000..f800643 --- /dev/null +++ b/docs/prepaint-policy-boundary.md @@ -0,0 +1,107 @@ +# Page pre-paint, browser policy, and workbench diagnostics + +The page pre-painter and the developer workbench answer different questions. +They must not be merged into one screen merely because both can be rendered as +dark text and rows. + +## Page pre-paint + +The page surface contains information that came from the selected resource: + +- article paragraphs and headings; +- links and their resolved targets; +- page forms and controls when the browser can represent their action honestly; +- table/data rows; +- already-fetched images, unchanged; +- a small browser-owned search/navigation control outside page content. + +The page surface does not contain fixture-universe counts, hot/warm/cold tab +graphs, deployment topology invented by a test, DNS traces, route hops, cache +classification, or storage totals. A real page may itself contain a topology +image; that is ordinary page content. The bundled APK sample should not use such +an image because it falsely suggests that IB adds the diagnostic to every page. + +Opening a plain UTF-8 file is a real pre-paint path. Ordinary prose is paintable +without an `ib-prepaint` envelope. A file containing one absolute HTTP(S) URL per +line becomes a link list. A saved HTML file still belongs to the Idriç HTML +extractor; the Android display harness must not grow a second HTML parser. + +The HTML extractor retains links nested inside article paragraphs and retains +an enclosing anchor on an image as a linked-image item. A remote `src` is only a +fetch candidate: the background ICU path must validate the response and replace +it with an already-fetched source before the Android harness paints the image. + +## Browser policy and the pre-decision probe + +Before starting a heavyweight renderer, browser policy can inspect bounded +evidence: + +- whether a cached pre-paint is already useful; +- known resource count and known/declared bytes; +- prior measurements for the site and device; +- DNS lookup state; +- a deliberately sampled route diagnostic; +- whether the page clearly requires JavaScript or another missing capability. + +`bin/prepaint_predecision.grease` is a first shell boundary for this evidence. +It gives `dig` three seconds and asks for A plus NS data. It gives `traceroute` +ten seconds, eight hops, and one probe per hop. Missing, filtered, or failed +diagnostics remain data; they do not block the cheap paint. + +DNS time and traceroute hop latency are not estimates of total page-load time. +They do not reveal asset count, transfer bandwidth, server computation, browser +main-thread work, JavaScript work, decoding, or layout. The probe therefore +labels them as diagnostics. Its only initial duration estimate is the explicit +lower bound: + +```text +ceil(known asset bytes / assumed measured bytes per second) +``` + +At five minutes or more, the stub keeps the useful pre-paint and requires a user +or later policy decision before heavyweight escalation. A future policy should +use observed per-device/site throughput instead of inventing a network rate. + +This probe is opt-in or sampled. `dig` and especially `traceroute` must not run on +every navigation. + +## Developer workbench / inspector + +The workbench is where browser-owned facts and diagnostic controls belong: + +- logical-page universe and hot/warm/cold counts; +- resident renderer working set and memory estimates; +- cache/storage classification and bounded inspection; +- resource graph size and known byte totals; +- DNS and route probe results; +- the pre-decision evidence and reason for allowing or deferring escalation; +- controls for eviction, cache clearing, simulated pressure, and replay. + +These facts may explain why a page stayed in pre-paint mode, but they are not the +page. A small status affordance can link from the page to the inspector without +injecting the diagnostic graph into page content. + +## Search handoff + +Search is browser chrome, not a fake form copied out of a workbench fixture: + +```text +query words + -> application/x-www-form-urlencoded query (spaces become +) + -> https://www.google.com/search?q=... + -> icu get URL + -> Idriç text/link/image extraction + -> disposable pre-paint + -> Android display +``` + +`bin/icu_search.grease` records the shell boundary. In the old shell idiom, +`$*` is the expansion that joins positional parameters using the first `IFS` +character; `PS1` is only the interactive prompt. The current stub uses `jq` for +UTF-8 percent encoding and then changes encoded spaces to `+`. ICU, not curl or +WebView, performs the GET. + +The standalone display APK has no Internet permission and cannot execute the +host ICU binary. It may show and copy the exact request so the interaction is +testable, but it must not pretend that local request construction is a fetched +result. The integrated browser shell will own the executable ICU/Android bridge. diff --git a/docs/url-recognition-strategy.txt b/docs/url-recognition-strategy.txt new file mode 100644 index 0000000..a8b19c6 --- /dev/null +++ b/docs/url-recognition-strategy.txt @@ -0,0 +1,114 @@ +IB URL recognition strategy — staged, readable, and replaceable +================================================================ + +This file deliberately keeps two recognizers instead of one enormous regular +expression. + + +1. Cheap foreground recognizer +------------------------------ + +Purpose: turn an ordinary text file containing one URL per line into useful +links without delaying first paint. + +Current rule, written as words: + +* begin at the start of the trimmed line; +* find exactly "http://" or "https://"; +* require a nonempty authority before slash, question mark, or fragment; +* reject whitespace, controls, non-ASCII bytes, and user information; +* accept the remainder as a candidate, not as proof that the resource exists. + +This intentionally mirrors the current ICU input boundary. It will miss valid +internationalized or unusual URLs. False negatives are acceptable in the first +paint because the background recognizer can add links later. + + +2. Authoritative recognizer +--------------------------- + +Do not validate the full web with a copied mega-regex. Parse components. + +The complete generic syntax is the collected ABNF in RFC 3986 Appendix A: + +https://www.rfc-editor.org/rfc/rfc3986.html#appendix-A + +That grammar covers scheme, authority, user information, host forms including +IP literals, port, all path variants, query, fragment, percent encoding, +reserved characters, and unreserved characters. Scheme-specific rules still +apply after generic parsing. RFC 3986 itself warns that generic syntax is a +superset; parsing a shape is not the same as proving it is dereferenceable. + +IB's current authoritative HTTP subset should call the same Idriç URL parser as +ICU rather than independently drifting. Today ICU intentionally supports only +http/https, visible ASCII targets, no user information, and no IPv6 literal. +When ICU grows IDNA/IPv6/IRI handling, IB should consume that typed result. + + +3. Background enrichment +------------------------ + +After plain text is already visible, a low-priority pass may: + +* scan URLs embedded inside prose rather than occupying a whole line; +* trim sentence punctuation using surrounding-text rules; +* resolve HTML href and src values against the fetched base URL; +* preserve an anchor's label and absolute target; +* preserve an image's alternate text, caption, source URL, and link target; +* ask ICU to fetch the candidate; +* classify an image from response Content-Type plus decoded magic, not merely a + filename extension; +* replace a remote image reference with an already-fetched local/content source + in a later pre-paint revision. + +An `` is a link even when it wraps text. An `` wrapping an image +must retain both behaviors: paint the fetched image as an image and keep its +navigation target. `InformationItem.Image` now carries source, alternate text, +caption/title, and the enclosing link target. The sibling Android interchange +accepts the same optional linked-image field. Until ICU has fetched and checked +the bytes, the source remains a remote reference; only the later revision may +replace it with a local/content source and claim that it is paintable. + + +4. Readable-regex references +---------------------------- + +The remembered Ruby project is VerbalExpressions. Its URL example literally +uses names such as start_of_line, find, maybe, anything_but, and end_of_line: + +https://github.com/ryan-endacott/verbal_expressions + +It is MIT licensed and useful as a readable sketching vocabulary. Its small URL +example is not a complete URL validator; it essentially accepts http(s), an +optional www, and then non-space text. Keep it as a strategy/prototyping +reference, not as IB's source of truth. + +Ruby's maintained URI module is a better executable reference for component +parsing and exposes parser/regular-expression facilities: + +https://github.com/ruby/uri + +Neither Ruby dependency belongs on the foreground phone path. A slow comparison +tool or fixture oracle is acceptable while the Idriç/ICU parser is developed. + + +5. Test ladder +-------------- + +Keep separate fixtures for: + +* cheap obvious http and https lines; +* relative href resolution against a base URL; +* punctuation around a URL in prose; +* query, fragment, percent encoding, and an explicit port; +* IPv4, IPv6 literals, IDNA, and Unicode IRI input; +* malformed percent escapes, spaces, controls, and empty hosts; +* user information that must not leak into painted text; +* a text URL that resolves to HTML; +* a text URL that resolves to an image despite having no image extension; +* an image-looking extension whose response is not an image; +* an anchor wrapping an image. + +The cheap recognizer may deliberately fail advanced positive cases. The +authoritative/background path may not silently convert malformed or dangerous +input into a clickable target. diff --git a/src/IB/Information.idric b/src/IB/Information.idric index 337d535..ad0cf0e 100644 --- a/src/IB/Information.idric +++ b/src/IB/Information.idric @@ -7,6 +7,7 @@ data InformationItem = Heading String | TextBlock String | Link String String + | Image String String String String | TableRow (List String) | Form String String @@ -36,7 +37,7 @@ data Capture | CaptureTitle | CaptureHeading String | CaptureText String - | CaptureLink String + | CaptureLink String Bool Capture | CaptureForm String | CaptureCell String @@ -158,6 +159,47 @@ starts_with_chars (_ :: _) [] = False starts_with_chars (wanted :: wanted_rest) (value :: rest) = wanted == value && starts_with_chars wanted_rest rest +strip_prefix_chars : List Char → List Char → Maybe (List Char) +strip_prefix_chars [] values = Just values +strip_prefix_chars (_ :: _) [] = Nothing +strip_prefix_chars (wanted :: wanted_rest) (value :: rest) = + if wanted == value + then strip_prefix_chars wanted_rest rest + else Nothing + +visible_http_character : Char → Bool +visible_http_character value = ord value > 32 && ord value < 127 + +authority_end : Char → Bool +authority_end value = value == '/' || value == '?' || value == '#' + +safe_authority_tail : List Char → Bool +safe_authority_tail [] = True +safe_authority_tail (value :: rest) = + if authority_end value + then True + else visible_http_character value && value /= '@' && safe_authority_tail rest + +safe_authority : List Char → Bool +safe_authority [] = False +safe_authority ('[' :: _) = False +safe_authority (value :: rest) = + not (authority_end value) && visible_http_character value && value /= '@' && + safe_authority_tail rest + +http_tail : String → Maybe (List Char) +http_tail text = + case strip_prefix_chars (unpack "http://") (unpack text) of + Just rest ⇒ Just rest + Nothing ⇒ strip_prefix_chars (unpack "https://") (unpack text) + +public export +cheap_http_url : String → Bool +cheap_http_url text = + case http_tail text of + Nothing ⇒ False + Just rest ⇒ safe_authority rest + drop_chars : Nat → List Char → List Char drop_chars Z values = values drop_chars (S amount) [] = [] @@ -216,7 +258,7 @@ capture_target NoCapture = "" capture_target CaptureTitle = "title" capture_target (CaptureHeading name) = name capture_target (CaptureText name) = name -capture_target (CaptureLink _) = "a" +capture_target (CaptureLink _ _ _) = "a" capture_target (CaptureForm _) = "form" capture_target (CaptureCell name) = name @@ -231,8 +273,10 @@ finish_capture (CaptureHeading _) text title items row = if text == "" then (title, items, row) else (title, Heading text :: items, row) finish_capture (CaptureText _) text title items row = if text == "" then (title, items, row) else (title, TextBlock text :: items, row) -finish_capture (CaptureLink href) text title items row = - if text == "" && href == "" then (title, items, row) else (title, Link text href :: items, row) +finish_capture (CaptureLink href has_image parent) text title items row = + if text == "" && (href == "" || has_image) + then (title, items, row) + else (title, Link text href :: items, row) finish_capture (CaptureForm action) text title items row = if text == "" && action == "" then (title, items, row) else (title, Form text action :: items, row) finish_capture (CaptureCell _) text title items Nothing = (title, items, Nothing) @@ -250,6 +294,32 @@ add_standalone_text text Nothing items = let cleaned = clean_text text in if cleaned == "" then items else TextBlock cleaned :: items +capture_parent : Capture → Capture +capture_parent (CaptureLink _ _ parent) = parent +capture_parent _ = NoCapture + +capture_link : Capture → String +capture_link (CaptureLink href _ _) = href +capture_link _ = "" + +mark_link_image : Capture → Capture +mark_link_image (CaptureLink href _ parent) = CaptureLink href True parent +mark_link_image capture = capture + +image_from_tag : String → String → Maybe InformationItem +image_from_tag href tag = + let source = attribute_or_empty "src" tag in + if source == "" + then Nothing + else Just (Image source (attribute_or_empty "alt" tag) + (attribute_or_empty "title" tag) href) + +add_image : String → String → List InformationItem → List InformationItem +add_image href tag items = + case image_from_tag href tag of + Nothing ⇒ items + Just image ⇒ image :: items + update_skip : String → Bool → SkipState → SkipState update_skip name closing NotSkipping = NotSkipping update_skip name closing (Skipping target depth) = @@ -291,7 +361,11 @@ collect_information (HtmlTag tag :: rest) skip capture chars row title items = else if (name == "p" || name == "li") && not closing then collect_information rest NotSkipping (CaptureText name) [] row title items else if name == "a" && not closing - then collect_information rest NotSkipping (CaptureLink (attribute_or_empty "href" tag)) [] row title items + then collect_information rest NotSkipping + (CaptureLink (attribute_or_empty "href" tag) False NoCapture) [] row title items + else if name == "img" && not closing + then collect_information rest NotSkipping NoCapture [] row title + (add_image "" tag items) else if name == "form" && not closing then collect_information rest NotSkipping (CaptureForm (attribute_or_empty "action" tag)) [] row title items else if name == "tr" && not closing @@ -307,9 +381,24 @@ collect_information (HtmlTag tag :: rest) skip capture chars row title items = if closing && name == capture_target capture then let text = clean_text (pack (reverse chars)) in let (next_title, next_items, next_row) = finish_capture capture text title items row in - collect_information rest NotSkipping NoCapture [] next_row next_title next_items + collect_information rest NotSkipping (capture_parent capture) [] next_row next_title next_items else if not closing && skipped_tag name then collect_information rest (Skipping name Z) capture chars row title items + else if name == "a" && not closing + then case capture of + CaptureLink _ _ _ ⇒ collect_information rest NotSkipping capture chars row title items + _ ⇒ + let text = clean_text (pack (reverse chars)) in + let (next_title, next_items, next_row) = finish_capture capture text title items row in + collect_information rest NotSkipping + (CaptureLink (attribute_or_empty "href" tag) False capture) + [] next_row next_title next_items + else if name == "img" && not closing + then let next_capture = mark_link_image capture in + let text = clean_text (pack (reverse chars)) in + let (next_title, next_items, next_row) = finish_capture next_capture text title items row in + collect_information rest NotSkipping next_capture [] next_row next_title + (add_image (capture_link capture) tag next_items) else collect_information rest NotSkipping capture chars row title items public export @@ -318,6 +407,31 @@ extract_html requested_url resolved_url html = let (title, items) = collect_information (tokenize html) NotSkipping NoCapture [] Nothing "" [] in View requested_url resolved_url title items +finish_plain_line : List Char → List InformationItem → List InformationItem +finish_plain_line chars items = + let line = clean_text (pack (reverse chars)) in + if line == "" + then items + else if cheap_http_url line + then Link line line :: items + else TextBlock line :: items + +plain_text_items : List Char → List Char → List InformationItem → List InformationItem +plain_text_items [] chars items = reverse (finish_plain_line chars items) +plain_text_items ('\n' :: rest) chars items = + plain_text_items rest [] (finish_plain_line chars items) +plain_text_items (value :: rest) chars items = + plain_text_items rest (value :: chars) items + +public export +prepaint_text : String → String → InformationView +prepaint_text title body = View "" "" title (plain_text_items (unpack body) [] []) + +public export +prepaint_full_text : String → String → String → InformationView +prepaint_full_text requested_url resolved_url html = + extract_html requested_url resolved_url html + public export has_table_row : List String → InformationView → Bool has_table_row wanted (View _ _ _ items) = go items @@ -336,6 +450,16 @@ has_link wanted_text wanted_href (View _ _ _ items) = go items go (Link text href :: rest) = (text == wanted_text && href == wanted_href) || go rest go (_ :: rest) = go rest +public export +has_image : String → String → String → InformationView → Bool +has_image wanted_alt wanted_source wanted_href (View _ _ _ items) = go items + where + go : List InformationItem → Bool + go [] = False + go (Image source alt caption href :: rest) = + (alt == wanted_alt && source == wanted_source && href == wanted_href) || go rest + go (_ :: rest) = go rest + public export has_form : String → InformationView → Bool has_form wanted_action (View _ _ _ items) = go items @@ -354,6 +478,7 @@ item_text : InformationItem → String item_text (Heading text) = text item_text (TextBlock text) = text item_text (Link text href) = text ++ " " ++ href +item_text (Image source alt caption href) = alt ++ " " ++ source ++ " " ++ caption ++ " " ++ href item_text (TableRow cells) = join_cells cells item_text (Form text action) = text ++ " " ++ action @@ -387,6 +512,9 @@ render_item : Nat → InformationItem → String render_item width (Heading text) = clip_text width ("# " ++ text) render_item width (TextBlock text) = clip_text width text render_item width (Link text href) = clip_text width ("link: " ++ text ++ " -> " ++ href) +render_item width (Image source alt caption href) = + clip_text width ("image: " ++ alt ++ " -> " ++ source ++ + if href == "" then "" else " [link " ++ href ++ "]") render_item width (TableRow cells) = clip_text width ("| " ++ join_with " | " cells ++ " |") render_item width (Form text action) = clip_text width ("form: " ++ text ++ " -> " ++ action) diff --git a/src/InformationSmoke.idric b/src/InformationSmoke.idric index 8d04202..5565112 100644 --- a/src/InformationSmoke.idric +++ b/src/InformationSmoke.idric @@ -35,6 +35,23 @@ shell_only = "" ++ "" +article_body : String +article_body = + "Field notes
" ++ + "

Small pages first

" ++ + "

The first paragraph is the useful first paint.

" ++ + "

The second paragraph remains part of the full text.

" ++ + "

Text before an inline article link text after.

" ++ + "Read the source" ++ + "Figure one" ++ + "
" + +plain_url_list : String +plain_url_list = + "https://example.test/one\n" ++ + "ordinary text\n" ++ + "http://example.test/two?q=three\n" + all_lines_within : Nat → List String → Bool all_lines_within width [] = True all_lines_within width (line :: rest) = @@ -49,6 +66,8 @@ main = do let early = extract_html "https://console.test/deployments" "https://console.test/deployments" heavy_prefix let full = extract_html "https://console.test/deployments" "https://console.test/deployments" (heavy_prefix ++ heavy_tail) let shell = extract_html "https://console.test/app" "https://console.test/app" shell_only + let article = prepaint_full_text "https://example.test/notes" "https://example.test/notes" article_body + let plain = prepaint_text "URL list" plain_url_list let phone = phone_lines 6 48 full putStrLn ("information-title=" ++ view_title full) putStrLn ("early-table-header=" ++ show (has_table_row ["Service", "Status"] early)) @@ -63,3 +82,14 @@ main = do putStrLn ("phone-api-row=" ++ show (has_line "| api | healthy |" phone)) putStrLn ("phone-lines=" ++ show (length phone)) putStrLn ("phone-width-bounded=" ++ show (all_lines_within 48 phone)) + putStrLn ("article-first-paragraph=" ++ show (has_exact_item_text "The first paragraph is the useful first paint." article)) + putStrLn ("article-second-paragraph=" ++ show (has_exact_item_text "The second paragraph remains part of the full text." article)) + putStrLn ("article-href=" ++ show (has_link "Read the source" "/source" article)) + putStrLn ("article-inline-href=" ++ show (has_link "an inline article link" "/inside" article)) + putStrLn ("article-inline-prefix=" ++ show (has_exact_item_text "Text before" article)) + putStrLn ("article-inline-suffix=" ++ show (has_exact_item_text "text after." article)) + putStrLn ("article-linked-image=" ++ show (has_image "Figure one" "/figure.png" "/full-figure" article)) + putStrLn ("article-image-empty-link=" ++ show (has_link "" "/full-figure" article)) + putStrLn ("plain-first-url=" ++ show (has_link "https://example.test/one" "https://example.test/one" plain)) + putStrLn ("plain-second-url=" ++ show (has_link "http://example.test/two?q=three" "http://example.test/two?q=three" plain)) + putStrLn ("cheap-url-rejects-relative=" ++ show (cheap_http_url "/relative")) diff --git a/tests/test_icu_search.grease b/tests/test_icu_search.grease new file mode 100755 index 0000000..b53238d --- /dev/null +++ b/tests/test_icu_search.grease @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +work_directory=$(mktemp -d) +cleanup() { + rm -rf "$work_directory" +} +trap cleanup EXIT HUP INT TERM + +fake_icu="$work_directory/icu" +apply_log="$work_directory/icu-arguments.txt" +{ + printf '%s\n' '#!/bin/sh' + printf '%s\n' 'set -eu' + printf '%s\n' 'printf "%s\\n" "$@" > "$IB_TEST_ICU_LOG"' +} > "$fake_icu" +chmod +x "$fake_icu" + +IB_ICU="$fake_icu" IB_TEST_ICU_LOG="$apply_log" \ + sh "$repository_root/bin/icu_search.grease" small fast browser + +test "$(sed -n '1p' "$apply_log")" = get +test "$(sed -n '2p' "$apply_log")" = \ + 'https://www.google.com/search?q=small+fast+browser' +test "$(wc -l < "$apply_log" | tr -d ' ')" = 2 + +printf 'icu search tests: ok\n' diff --git a/tests/test_prepaint_predecision.grease b/tests/test_prepaint_predecision.grease new file mode 100755 index 0000000..053d9db --- /dev/null +++ b/tests/test_prepaint_predecision.grease @@ -0,0 +1,62 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +work_directory=$(mktemp -d) +cleanup() { + rm -rf "$work_directory" +} +trap cleanup EXIT HUP INT TERM + +fake_commands="$work_directory/bin" +mkdir -p "$fake_commands" + +{ + printf '%s\n' '#!/bin/sh' + printf '%s\n' 'set -eu' + printf '%s\n' 'printf "%s\\n" "$@" > "$IB_TEST_DIG_LOG"' + printf '%s\n' 'printf "%s\\n" "example.com. 300 IN A 192.0.2.8"' + printf '%s\n' 'printf "%s\\n" "example.com. 300 IN NS ns1.example.com."' + printf '%s\n' 'printf "%s\\n" ";; Query time: 7 msec"' +} > "$fake_commands/dig" + +{ + printf '%s\n' '#!/bin/sh' + printf '%s\n' 'set -eu' + printf '%s\n' 'printf "%s\\n" "$@" > "$IB_TEST_TRACEROUTE_LOG"' + printf '%s\n' 'printf "%s\\n" "traceroute to example.com, 8 hops max"' + printf '%s\n' 'printf "%s\\n" " 1 192.0.2.1 3.1 ms"' + printf '%s\n' 'printf "%s\\n" " 2 192.0.2.8 17.2 ms"' +} > "$fake_commands/traceroute" + +chmod +x "$fake_commands/dig" "$fake_commands/traceroute" + +output="$work_directory/predecision.tsv" +PATH="$fake_commands:$PATH" \ +IB_TEST_DIG_LOG="$work_directory/dig-arguments.txt" \ +IB_TEST_TRACEROUTE_LOG="$work_directory/traceroute-arguments.txt" \ +IB_KNOWN_ASSET_COUNT=900 \ +IB_KNOWN_ASSET_BYTES=39321600 \ +IB_ASSUMED_BYTES_PER_SECOND=131072 \ + sh "$repository_root/bin/prepaint_predecision.grease" \ + 'https://example.com/large/page?q=one' > "$output" + +tab=$(printf '\t') +grep -Fx "host${tab}example.com" "$output" >/dev/null +grep -Fx "dig-status${tab}ok" "$output" >/dev/null +grep -Fx "dns-ms${tab}7" "$output" >/dev/null +grep -Fx "address-records${tab}1" "$output" >/dev/null +grep -Fx "namespace-records${tab}1" "$output" >/dev/null +grep -Fx "traceroute-status${tab}ok" "$output" >/dev/null +grep -Fx "route-hops${tab}2" "$output" >/dev/null +grep -Fx "route-last-ms${tab}17.2" "$output" >/dev/null +grep -Fx "estimated-transfer-seconds${tab}300" "$output" >/dev/null +grep -Fx "decision${tab}keep-prepaint-unless-user-escalates" "$output" >/dev/null + +grep -Fx '+tries=1' "$work_directory/dig-arguments.txt" >/dev/null +grep -Fx '+time=1' "$work_directory/dig-arguments.txt" >/dev/null +grep -Fx 'NS' "$work_directory/dig-arguments.txt" >/dev/null +grep -Fx -- '-m' "$work_directory/traceroute-arguments.txt" >/dev/null +grep -Fx '8' "$work_directory/traceroute-arguments.txt" >/dev/null + +printf 'prepaint predecision tests: ok\n'