Add ICU search and prepaint pre-decision boundary

.github/workflows/idric-core.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Install system dependencies through Grease
         run: sh bin/ci_browser_foundation.grease install-dependencies
 
+      - name: Exercise ICU search and bounded network pre-decision
+        run: sh bin/ci_browser_foundation.grease exercise-prepaint-network
+
       - name: Restore Idric compiler
         id: idric-cache
         uses: actions/cache/restore@v4

bin/ci_browser_foundation.grease

@@ -8,7 +8,7 @@ pdf_harvester_ref=${PDF_HARVESTER_REF:-77d85dc6f7e89d109cb9e06f42706bc34f00e4a3}
 
 install_dependencies() {
   sudo apt-get update
-  sudo apt-get install -y chezscheme curl xmlstarlet busybox poppler-utils
+  sudo apt-get install -y chezscheme curl xmlstarlet busybox poppler-utils jq
 }
 
 build_idric() {
@@ -70,6 +70,16 @@ exercise_workbench() {
   grep -Fx 'resident-at-10=10' /tmp/ib-workbench.txt
 }
 
+exercise_prepaint_network() {
+  cd "$repository_root"
+  sh -n bin/icu_search.grease
+  sh -n bin/prepaint_predecision.grease
+  sh -n tests/test_icu_search.grease
+  sh -n tests/test_prepaint_predecision.grease
+  sh tests/test_icu_search.grease
+  sh tests/test_prepaint_predecision.grease
+}
+
 build_information_programs() {
   test -x "$idric_prefix/bin/idris2" || build_idric
   cd "$repository_root/src"
@@ -100,6 +110,17 @@ exercise_information() {
   grep -Fx 'phone-api-row=True' /tmp/ib-information-smoke.txt
   grep -Fx 'phone-lines=8' /tmp/ib-information-smoke.txt
   grep -Fx 'phone-width-bounded=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'article-first-paragraph=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'article-second-paragraph=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'article-href=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'article-inline-href=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'article-inline-prefix=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'article-inline-suffix=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'article-linked-image=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'article-image-empty-link=False' /tmp/ib-information-smoke.txt
+  grep -Fx 'plain-first-url=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'plain-second-url=True' /tmp/ib-information-smoke.txt
+  grep -Fx 'cheap-url-rejects-relative=False' /tmp/ib-information-smoke.txt
   sh -n bin/prepaint_arxiv_progressively.grease
   sh -n tests/test_arxiv_progressive_prepaint.grease
   sh tests/test_arxiv_progressive_prepaint.grease
@@ -149,11 +170,12 @@ case "${1:-}" in
   exercise-information) exercise_information ;;
   exercise-core) exercise_core ;;
   exercise-workbench) exercise_workbench ;;
+  exercise-prepaint-network) exercise_prepaint_network ;;
   exercise-scientific-media) exercise_scientific_media ;;
   exercise-live-arxiv) exercise_live_arxiv ;;
   exercise-live-arxiv-prepaint) exercise_live_arxiv_prepaint ;;
   *)
-    printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2
+    printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-prepaint-network|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2
     exit 2
     ;;
 esac

bin/icu_search.grease

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+if test "$#" -eq 0; then
+  printf 'usage: %s SEARCH WORDS...\n' "$0" >&2
+  exit 2
+fi
+
+icu_command=${IB_ICU:-icu}
+jq_command=${IB_JQ:-jq}
+
+# With the ordinary space IFS, "$*" is the shell operation the original tiny
+# search command was remembering. PS1 is only the interactive prompt. jq owns
+# UTF-8 form encoding here; replacing %20 with + gives the familiar query form.
+query=$*
+encoded_query=$(printf '%s' "$query" | "$jq_command" -sRr @uri | sed 's/%20/+/g')
+
+exec "$icu_command" get "https://www.google.com/search?q=$encoded_query"

bin/prepaint_predecision.grease

@@ -0,0 +1,118 @@
+#!/bin/sh
+set -eu
+
+if test "$#" -ne 1; then
+  printf 'usage: %s URL\n' "$0" >&2
+  exit 2
+fi
+
+input_url=$1
+case $input_url in
+  http://*) authority_and_rest=${input_url#http://} ;;
+  https://*) authority_and_rest=${input_url#https://} ;;
+  *)
+    printf 'predecision: URL must begin with http:// or https://\n' >&2
+    exit 2
+    ;;
+esac
+
+authority=${authority_and_rest%%/*}
+authority=${authority%%\?*}
+authority=${authority%%\#*}
+case $authority in
+  ''|*@*|'['*)
+    printf 'predecision: unsupported or empty URL authority\n' >&2
+    exit 2
+    ;;
+esac
+host=${authority%%:*}
+
+dig_command=${IB_DIG:-dig}
+traceroute_command=${IB_TRACEROUTE:-traceroute}
+timeout_command=${IB_TIMEOUT:-timeout}
+
+work_directory=$(mktemp -d)
+cleanup() {
+  rm -rf "$work_directory"
+}
+trap cleanup EXIT HUP INT TERM
+
+dig_status=unavailable
+if command -v "$dig_command" >/dev/null 2>&1; then
+  if "$timeout_command" 3 "$dig_command" +tries=1 +time=1 +stats \
+      "$host" A "$host" NS > "$work_directory/dig.txt" 2>&1
+  then
+    dig_status=ok
+  else
+    dig_status=failed
+  fi
+else
+  : > "$work_directory/dig.txt"
+fi
+
+route_status=unavailable
+if command -v "$traceroute_command" >/dev/null 2>&1; then
+  if "$timeout_command" 10 "$traceroute_command" -n -m 8 -w 1 -q 1 \
+      "$host" > "$work_directory/traceroute.txt" 2>&1
+  then
+    route_status=ok
+  else
+    route_status=failed
+  fi
+else
+  : > "$work_directory/traceroute.txt"
+fi
+
+dns_milliseconds=$(awk '/Query time:/ { print $(NF - 1); exit }' "$work_directory/dig.txt")
+test -n "$dns_milliseconds" || dns_milliseconds=unknown
+address_records=$(awk '$4 == "A" || $4 == "AAAA" { amount += 1 } END { print amount + 0 }' \
+  "$work_directory/dig.txt")
+namespace_records=$(awk '$4 == "NS" { amount += 1 } END { print amount + 0 }' \
+  "$work_directory/dig.txt")
+route_hops=$(awk '/^[[:space:]]*[0-9]+[[:space:]]/ { hop = $1 } END { if (hop == "") print 0; else print hop }' \
+  "$work_directory/traceroute.txt")
+route_last_milliseconds=$(awk '
+  /^[[:space:]]*[0-9]+[[:space:]]/ {
+    for (field = 1; field <= NF; field += 1) {
+      if ($field == "ms" && field > 1) last = $(field - 1)
+    }
+  }
+  END { if (last == "") print "unknown"; else print last }
+' "$work_directory/traceroute.txt")
+
+known_asset_count=${IB_KNOWN_ASSET_COUNT:-unknown}
+known_asset_bytes=${IB_KNOWN_ASSET_BYTES:-unknown}
+assumed_bytes_per_second=${IB_ASSUMED_BYTES_PER_SECOND:-unknown}
+estimated_transfer_seconds=unknown
+decision=measure-assets-before-heavy-renderer
+
+case $known_asset_bytes:$assumed_bytes_per_second in
+  *[!0-9:]*|:*|*:0) ;;
+  *)
+    estimated_transfer_seconds=$((
+      (known_asset_bytes + assumed_bytes_per_second - 1) / assumed_bytes_per_second
+    ))
+    if test "$estimated_transfer_seconds" -ge 300; then
+      decision=keep-prepaint-unless-user-escalates
+    else
+      decision=prepaint-then-bounded-escalation
+    fi
+    ;;
+esac
+
+printf 'predecision\t1\n'
+printf 'url\t%s\n' "$input_url"
+printf 'host\t%s\n' "$host"
+printf 'dig-status\t%s\n' "$dig_status"
+printf 'dns-ms\t%s\n' "$dns_milliseconds"
+printf 'address-records\t%s\n' "$address_records"
+printf 'namespace-records\t%s\n' "$namespace_records"
+printf 'traceroute-status\t%s\n' "$route_status"
+printf 'route-hops\t%s\n' "$route_hops"
+printf 'route-last-ms\t%s\n' "$route_last_milliseconds"
+printf 'known-asset-count\t%s\n' "$known_asset_count"
+printf 'known-asset-bytes\t%s\n' "$known_asset_bytes"
+printf 'assumed-bytes-per-second\t%s\n' "$assumed_bytes_per_second"
+printf 'estimated-transfer-seconds\t%s\n' "$estimated_transfer_seconds"
+printf 'decision\t%s\n' "$decision"
+printf 'route-warning\t%s\n' 'DNS and traceroute are diagnostics, not page-load predictions.'

docs/prepaint-policy-boundary.md

@@ -0,0 +1,107 @@
+# Page pre-paint, browser policy, and workbench diagnostics
+
+The page pre-painter and the developer workbench answer different questions.
+They must not be merged into one screen merely because both can be rendered as
+dark text and rows.
+
+## Page pre-paint
+
+The page surface contains information that came from the selected resource:
+
+- article paragraphs and headings;
+- links and their resolved targets;
+- page forms and controls when the browser can represent their action honestly;
+- table/data rows;
+- already-fetched images, unchanged;
+- a small browser-owned search/navigation control outside page content.
+
+The page surface does not contain fixture-universe counts, hot/warm/cold tab
+graphs, deployment topology invented by a test, DNS traces, route hops, cache
+classification, or storage totals. A real page may itself contain a topology
+image; that is ordinary page content. The bundled APK sample should not use such
+an image because it falsely suggests that IB adds the diagnostic to every page.
+
+Opening a plain UTF-8 file is a real pre-paint path. Ordinary prose is paintable
+without an `ib-prepaint` envelope. A file containing one absolute HTTP(S) URL per
+line becomes a link list. A saved HTML file still belongs to the Idriç HTML
+extractor; the Android display harness must not grow a second HTML parser.
+
+The HTML extractor retains links nested inside article paragraphs and retains
+an enclosing anchor on an image as a linked-image item. A remote `src` is only a
+fetch candidate: the background ICU path must validate the response and replace
+it with an already-fetched source before the Android harness paints the image.
+
+## Browser policy and the pre-decision probe
+
+Before starting a heavyweight renderer, browser policy can inspect bounded
+evidence:
+
+- whether a cached pre-paint is already useful;
+- known resource count and known/declared bytes;
+- prior measurements for the site and device;
+- DNS lookup state;
+- a deliberately sampled route diagnostic;
+- whether the page clearly requires JavaScript or another missing capability.
+
+`bin/prepaint_predecision.grease` is a first shell boundary for this evidence.
+It gives `dig` three seconds and asks for A plus NS data. It gives `traceroute`
+ten seconds, eight hops, and one probe per hop. Missing, filtered, or failed
+diagnostics remain data; they do not block the cheap paint.
+
+DNS time and traceroute hop latency are not estimates of total page-load time.
+They do not reveal asset count, transfer bandwidth, server computation, browser
+main-thread work, JavaScript work, decoding, or layout. The probe therefore
+labels them as diagnostics. Its only initial duration estimate is the explicit
+lower bound:
+
+```text
+ceil(known asset bytes / assumed measured bytes per second)
+```
+
+At five minutes or more, the stub keeps the useful pre-paint and requires a user
+or later policy decision before heavyweight escalation. A future policy should
+use observed per-device/site throughput instead of inventing a network rate.
+
+This probe is opt-in or sampled. `dig` and especially `traceroute` must not run on
+every navigation.
+
+## Developer workbench / inspector
+
+The workbench is where browser-owned facts and diagnostic controls belong:
+
+- logical-page universe and hot/warm/cold counts;
+- resident renderer working set and memory estimates;
+- cache/storage classification and bounded inspection;
+- resource graph size and known byte totals;
+- DNS and route probe results;
+- the pre-decision evidence and reason for allowing or deferring escalation;
+- controls for eviction, cache clearing, simulated pressure, and replay.
+
+These facts may explain why a page stayed in pre-paint mode, but they are not the
+page. A small status affordance can link from the page to the inspector without
+injecting the diagnostic graph into page content.
+
+## Search handoff
+
+Search is browser chrome, not a fake form copied out of a workbench fixture:
+
+```text
+query words
+  -> application/x-www-form-urlencoded query (spaces become +)
+  -> https://www.google.com/search?q=...
+  -> icu get URL
+  -> Idriç text/link/image extraction
+  -> disposable pre-paint
+  -> Android display
+```
+
+`bin/icu_search.grease` records the shell boundary. In the old shell idiom,
+`$*` is the expansion that joins positional parameters using the first `IFS`
+character; `PS1` is only the interactive prompt. The current stub uses `jq` for
+UTF-8 percent encoding and then changes encoded spaces to `+`. ICU, not curl or
+WebView, performs the GET.
+
+The standalone display APK has no Internet permission and cannot execute the
+host ICU binary. It may show and copy the exact request so the interaction is
+testable, but it must not pretend that local request construction is a fetched
+result. The integrated browser shell will own the executable ICU/Android bridge.