From 946ad8e6f25636cc1be11a742ad976206101d523 Mon Sep 17 00:00:00 2001 From: i Date: Sun, 23 Aug 2026 13:45:01 -0400 Subject: [PATCH 01/10] Add provider-neutral product and price records --- src/IB/Product.idric | 84 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/IB/Product.idric diff --git a/src/IB/Product.idric b/src/IB/Product.idric new file mode 100644 index 0000000..cf4c7cb --- /dev/null +++ b/src/IB/Product.idric @@ -0,0 +1,84 @@ +module IB.Product + +%default total + +public export +data ProductKey + = ISBN String + | UPC String + | GTIN String + | ASIN String + | SourceKey String String + +public export +key_text : ProductKey → String +key_text (ISBN value) = "isbn:" ++ value +key_text (UPC value) = "upc:" ++ value +key_text (GTIN value) = "gtin:" ++ value +key_text (ASIN value) = "asin:" ++ value +key_text (SourceKey source value) = source ++ ":" ++ value + +public export +data Product = Item String (List ProductKey) (List String) + +public export +product_id : Product → String +product_id (Item value _ _) = value + +public export +product_keys : Product → List ProductKey +product_keys (Item _ keys _) = keys + +public export +product_terms : Product → List String +product_terms (Item _ _ terms) = terms + +public export +data PriceObservation = Observed String String Nat String String + +public export +price_source : PriceObservation → String +price_source (Observed source _ _ _ _) = source + +public export +price_product : PriceObservation → String +price_product (Observed _ product _ _ _) = product + +public export +price_minor_units : PriceObservation → Nat +price_minor_units (Observed _ _ amount _ _) = amount + +public export +price_currency : PriceObservation → String +price_currency (Observed _ _ _ currency _) = currency + +public export +price_observed_at : PriceObservation → String +price_observed_at (Observed _ _ _ _ observed_at) = observed_at + +public export +data CurrentOffer = Offer String ProductKey Nat String String String + +public export +offer_source : CurrentOffer → String +offer_source (Offer source _ _ _ _ _) = source + +public export +offer_key : CurrentOffer → ProductKey +offer_key (Offer _ key _ _ _ _) = key + +public export +offer_minor_units : CurrentOffer → Nat +offer_minor_units (Offer _ _ amount _ _ _) = amount + +public export +offer_currency : CurrentOffer → String +offer_currency (Offer _ _ _ currency _ _) = currency + +public export +offer_observed_at : CurrentOffer → String +offer_observed_at (Offer _ _ _ _ observed_at _) = observed_at + +public export +offer_expires_at : CurrentOffer → String +offer_expires_at (Offer _ _ _ _ _ expires_at) = expires_at From c2bee9345b8a145a05d4dd8c8acef8c6280ef963 Mon Sep 17 00:00:00 2001 From: i Date: Sun, 23 Aug 2026 13:45:09 -0400 Subject: [PATCH 02/10] Add rebuildable float vector index core --- src/IB/VectorIndex.idric | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/IB/VectorIndex.idric diff --git a/src/IB/VectorIndex.idric b/src/IB/VectorIndex.idric new file mode 100644 index 0000000..4cf07b2 --- /dev/null +++ b/src/IB/VectorIndex.idric @@ -0,0 +1,39 @@ +module IB.VectorIndex + +%default total + +public export +data VectorRow = VectorRow String (List Float) + +public export +vector_id : VectorRow → String +vector_id (VectorRow value _) = value + +public export +vector_values : VectorRow → List Float +vector_values (VectorRow _ values) = values + +public export +dot : List Float → List Float → Float +dot [] _ = 0.0 +dot _ [] = 0.0 +dot (left :: more_left) (right :: more_right) = + left * right + dot more_left more_right + +public export +score : List Float → VectorRow → Float +score query row = dot query (vector_values row) + +public export +best_match : List Float → List VectorRow → Maybe (String, Float) +best_match query [] = Nothing +best_match query (first :: rest) = + go first (score query first) rest + where + go : VectorRow → Float → List VectorRow → Maybe (String, Float) + go best best_score [] = Just (vector_id best, best_score) + go best best_score (candidate :: more) = + let candidate_score = score query candidate in + if candidate_score > best_score + then go candidate candidate_score more + else go best best_score more From aac713dc4e8055fe78e82ba2c787b7d901ba184d Mon Sep 17 00:00:00 2001 From: i Date: Sun, 23 Aug 2026 13:45:28 -0400 Subject: [PATCH 03/10] Document product prices Amazon boundary and vector index --- docs/product-price-index.md | 101 ++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docs/product-price-index.md diff --git a/docs/product-price-index.md b/docs/product-price-index.md new file mode 100644 index 0000000..9ef1f7a --- /dev/null +++ b/docs/product-price-index.md @@ -0,0 +1,101 @@ +# Product, price, and vector index + +IB should treat products and prices as durable browser-owned data rather than as state owned by a merchant renderer. + +The merchant is a source, not the object model. + +## Product identity + +A product has an IB identity and zero or more external keys: + +- ISBN +- UPC +- GTIN +- ASIN +- source-specific keys + +An ASIN is therefore useful without making Amazon the canonical namespace. + +```text +product p-0001 +key isbn:978... +key asin:B0... +term category-theory +term paperback +``` + +`src/IB/Product.idric` contains the first typed representation. + +## Price observations + +Prices are observations, not properties of a product. + +```text +observed 2026-08-23T13:42:00-04:00 +product p-0001 +source merchant-x +currency USD +minor_units 1799 +``` + +Store money in integer minor units. A price observation belongs in canonical history only when its source permits durable storage. + +Provider data with a short legal or technical lifetime is a `CurrentOffer`, not a historical observation. It belongs in disposable cache and carries an expiry time. + +This distinction lets IB keep a real price history without pretending that every provider grants a license to archive its API responses. + +## Amazon + +Amazon is one provider. + +The Creators API is useful for live catalog lookup and current offers, but its Program Content has unusually tight use and caching restrictions. The durable model should therefore keep Amazon identifiers such as ASINs while treating Creators API offer data as volatile provider cache. + +Do not make the IB canonical price ledger depend on Amazon Creators API response retention. + +The Amazon adapter should eventually expose roughly: + +```text +search(query) -> product candidates +item(asin) -> current metadata +current_offer(asin) -> expiring offer +outbound_link(asin, partner_tag) -> Amazon link +``` + +The core does not need to know how Amazon authentication works. + +Affiliate identity is distribution configuration, not product identity. A downstream build can supply its own partner tag or no tag. A distribution operated by one Associate can supply that Associate's tag where Amazon permits that distribution surface. + +## Vector index + +Vector search is a normal rebuildable index. + +Canonical product text and identifiers remain inspectable records. Embeddings derived from canonical data live under an index namespace, for example: + +```text +state/ + products/ + prices/ + indexes/ + products/ + / + vectors + metadata +``` + +The first implementation in `src/IB/VectorIndex.idric` uses `Float` vectors and exact dot-product search. Normalize vectors before insertion when cosine-like ranking is desired. Exact scanning is intentionally simple and gives us a correct baseline. If corpus size makes it necessary, the same rebuildable index can later gain an ANN representation such as HNSW without changing canonical product records. + +Do not derive the persistent embedding corpus from provider content whose license forbids repurposing, analysis, or long-term storage. Vectorize IB-owned/canonical product text and independently usable metadata instead. + +## Browser integration + +This follows the existing IB storage rule: canonical state is separate from rebuildable indexes and disposable caches. + +A visit to a product page can therefore connect: + +```text +visit -> URL -> product identity -> current offers + |-> durable permitted price observations + |-> rebuildable semantic vector +``` + +The UI can show a current Amazon price next to a book, tool, replacement part, or other object without making Amazon the place where the user must buy it. From bacd9e1af13a1429ed130dd3d3a624376c129ce2 Mon Sep 17 00:00:00 2001 From: i Date: Sun, 23 Aug 2026 13:45:43 -0400 Subject: [PATCH 04/10] Smoke test product prices and float vector search --- src/Smoke.idric | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/Smoke.idric b/src/Smoke.idric index fdb5e4e..25763c2 100644 --- a/src/Smoke.idric +++ b/src/Smoke.idric @@ -4,6 +4,8 @@ import IB.History import IB.Index import IB.Storage import IB.Inspect +import IB.Product +import IB.VectorIndex %default total @@ -15,14 +17,26 @@ sample_entries = [ Entry "https://missing-time.test/c" Nothing Nothing raw_urls visit Nothing 93 ] +sample_vectors : List VectorRow +sample_vectors = [ + VectorRow "p-book" [1.0, 0.0, 0.0], + VectorRow "p-tool" [0.0, 1.0, 0.0], + VectorRow "p-part" [0.0, 0.0, 1.0] + ] + first_or : Nat → List Nat → Nat first_or fallback [] = fallback first_or fallback (value :: _) = value +match_id : Maybe (String, Float) → String +match_id Nothing = "none" +match_id (Just (value, _)) = value + main : IO () main = do let raw = ingest_raw_lines ["# ignored", " https://example.test/a ", "https://example.test/a", "https://other.test/b"] let indices = build_indices sample_entries + let price = Observed "permitted-history-source" "p-book" 1799 "USD" "2026-08-23T13:42:00-04:00" let files = [ FileRow "visits.jsonl" 20 regular_file, FileRow "indexes/summary.json" 10 regular_file, @@ -39,3 +53,6 @@ main = do putStrLn ("secret=" ++ storage_kind_text (classify_path "renderer/passwords")) putStrLn ("readable-files=" ++ show (readable_file_count files)) putStrLn ("canonical-files=" ++ show (files_of_kind canonical files)) + putStrLn ("asin-key=" ++ key_text (ASIN "B000TEST")) + putStrLn ("price-minor-units=" ++ show (price_minor_units price)) + putStrLn ("vector-match=" ++ match_id (best_match [0.9, 0.1, 0.0] sample_vectors)) From a05c9b23b35d9b5191c35c59d8fe143a8a1dc65a Mon Sep 17 00:00:00 2001 From: i Date: Mon, 24 Aug 2026 01:40:33 -0400 Subject: [PATCH 05/10] Fix Idric smoke working directory --- .github/workflows/idric-core.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 70ef1da..0ba1969 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -50,12 +50,14 @@ jobs: - name: Compile IB Idric smoke program run: | set -o pipefail - /tmp/idric/bin/idris2 src/Smoke.idric -o ib-smoke 2>&1 | tee /tmp/idric-compile.txt + cd src + /tmp/idric/bin/idris2 Smoke.idric -o ib-smoke 2>&1 | tee /tmp/idric-compile.txt test -x ./build/exec/ib-smoke ! grep -q '^Error:' /tmp/idric-compile.txt - name: Exercise browser-owned core run: | + cd src ./build/exec/ib-smoke | tee /tmp/ib-smoke.txt grep -Fx 'ingest=3' /tmp/ib-smoke.txt grep -Fx 'duplicate-url-count=2' /tmp/ib-smoke.txt @@ -65,4 +67,4 @@ jobs: grep -Fx 'cache-fake=cache' /tmp/ib-smoke.txt grep -Fx 'secret=secret' /tmp/ib-smoke.txt grep -Fx 'readable-files=2' /tmp/ib-smoke.txt - grep -Fx 'canonical-files=2' /tmp/ib-smoke.txt + grep -Fx 'canonical-files=2' /tmp/ib-smoke.txt \ No newline at end of file From 5a86127fd8a4c853bfcf2399f3386894193aa53a Mon Sep 17 00:00:00 2001 From: i Date: Mon, 24 Aug 2026 01:40:55 -0400 Subject: [PATCH 06/10] Prefer ICU and Grease for price retrieval --- docs/product-price-index.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/product-price-index.md b/docs/product-price-index.md index 9ef1f7a..6c9d4a2 100644 --- a/docs/product-price-index.md +++ b/docs/product-price-index.md @@ -44,20 +44,27 @@ Provider data with a short legal or technical lifetime is a `CurrentOffer`, not This distinction lets IB keep a real price history without pretending that every provider grants a license to archive its API responses. +## Price retrieval + +For price data, prefer the ordinary web path over a restricted merchant API: ICU fetches the page and Grease handles the retrieval/extraction workflow. If the useful price value is easy to isolate, a small grep-like extraction is enough. If the page needs structured parsing, feed the ICU-fetched text to a small Idriç parser rather than moving browser policy into shell code. + +This rule is specifically about price data. Product metadata, identifiers, outbound links, and other provider integrations may use different adapters when appropriate. + +Fetched page bodies are not the durable price ledger. Reduce them to the minimal price fact needed by the core, then apply the existing storage boundary: a permitted durable fact becomes a `PriceObservation`; data that must remain short-lived stays a `CurrentOffer` in disposable cache. + ## Amazon Amazon is one provider. -The Creators API is useful for live catalog lookup and current offers, but its Program Content has unusually tight use and caching restrictions. The durable model should therefore keep Amazon identifiers such as ASINs while treating Creators API offer data as volatile provider cache. +Do not make the IB price path depend on Amazon Creators API offer retention. For current Amazon prices, use the ICU + Grease retrieval path above and retain only the minimal parsed price fact according to the source's storage policy. -Do not make the IB canonical price ledger depend on Amazon Creators API response retention. +Amazon identifiers such as ASINs remain useful durable product keys. An Amazon adapter can still provide non-price catalog metadata or outbound-link behavior without making Amazon the canonical product namespace. The Amazon adapter should eventually expose roughly: ```text search(query) -> product candidates item(asin) -> current metadata -current_offer(asin) -> expiring offer outbound_link(asin, partner_tag) -> Amazon link ``` @@ -98,4 +105,4 @@ visit -> URL -> product identity -> current offers |-> rebuildable semantic vector ``` -The UI can show a current Amazon price next to a book, tool, replacement part, or other object without making Amazon the place where the user must buy it. +The UI can show a current Amazon price next to a book, tool, replacement part, or other object without making Amazon the place where the user must buy it. \ No newline at end of file From 83ef6adb0679bdefb706fb1963ce8d45080eb502 Mon Sep 17 00:00:00 2001 From: i Date: Mon, 24 Aug 2026 01:43:17 -0400 Subject: [PATCH 07/10] Use supported scalar in rebuildable vector index --- src/IB/VectorIndex.idric | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/IB/VectorIndex.idric b/src/IB/VectorIndex.idric index 4cf07b2..5da622f 100644 --- a/src/IB/VectorIndex.idric +++ b/src/IB/VectorIndex.idric @@ -2,38 +2,41 @@ module IB.VectorIndex %default total +-- The pinned Idriç compiler does not yet expose a primitive 32-bit Float. +-- This index is rebuildable, so using Double here does not affect canonical +-- browser state and can be replaced by Float without a state migration. public export -data VectorRow = VectorRow String (List Float) +data VectorRow = Row String (List Double) public export vector_id : VectorRow → String -vector_id (VectorRow value _) = value +vector_id (Row value _) = value public export -vector_values : VectorRow → List Float -vector_values (VectorRow _ values) = values +vector_values : VectorRow → List Double +vector_values (Row _ values) = values public export -dot : List Float → List Float → Float +dot : List Double → List Double → Double dot [] _ = 0.0 dot _ [] = 0.0 dot (left :: more_left) (right :: more_right) = left * right + dot more_left more_right public export -score : List Float → VectorRow → Float +score : List Double → VectorRow → Double score query row = dot query (vector_values row) public export -best_match : List Float → List VectorRow → Maybe (String, Float) +best_match : List Double → List VectorRow → Maybe (String, Double) best_match query [] = Nothing best_match query (first :: rest) = go first (score query first) rest where - go : VectorRow → Float → List VectorRow → Maybe (String, Float) + go : VectorRow → Double → List VectorRow → Maybe (String, Double) go best best_score [] = Just (vector_id best, best_score) go best best_score (candidate :: more) = let candidate_score = score query candidate in if candidate_score > best_score then go candidate candidate_score more - else go best best_score more + else go best best_score more \ No newline at end of file From 8df6a7be7023452321a5983ea3a3e9f76169c892 Mon Sep 17 00:00:00 2001 From: i Date: Mon, 24 Aug 2026 01:43:29 -0400 Subject: [PATCH 08/10] Adjust vector smoke test for supported scalar --- src/Smoke.idric | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Smoke.idric b/src/Smoke.idric index 25763c2..3c07578 100644 --- a/src/Smoke.idric +++ b/src/Smoke.idric @@ -19,16 +19,16 @@ sample_entries = [ sample_vectors : List VectorRow sample_vectors = [ - VectorRow "p-book" [1.0, 0.0, 0.0], - VectorRow "p-tool" [0.0, 1.0, 0.0], - VectorRow "p-part" [0.0, 0.0, 1.0] + Row "p-book" [1.0, 0.0, 0.0], + Row "p-tool" [0.0, 1.0, 0.0], + Row "p-part" [0.0, 0.0, 1.0] ] first_or : Nat → List Nat → Nat first_or fallback [] = fallback first_or fallback (value :: _) = value -match_id : Maybe (String, Float) → String +match_id : Maybe (String, Double) → String match_id Nothing = "none" match_id (Just (value, _)) = value @@ -55,4 +55,4 @@ main = do putStrLn ("canonical-files=" ++ show (files_of_kind canonical files)) putStrLn ("asin-key=" ++ key_text (ASIN "B000TEST")) putStrLn ("price-minor-units=" ++ show (price_minor_units price)) - putStrLn ("vector-match=" ++ match_id (best_match [0.9, 0.1, 0.0] sample_vectors)) + putStrLn ("vector-match=" ++ match_id (best_match [0.9, 0.1, 0.0] sample_vectors)) \ No newline at end of file From 653f5c505e92dc474bc394064ca1fd50056bb6e1 Mon Sep 17 00:00:00 2001 From: i Date: Mon, 24 Aug 2026 01:43:44 -0400 Subject: [PATCH 09/10] Document rebuildable vector scalar fallback --- docs/product-price-index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/product-price-index.md b/docs/product-price-index.md index 6c9d4a2..2f6f8ba 100644 --- a/docs/product-price-index.md +++ b/docs/product-price-index.md @@ -89,7 +89,9 @@ state/ metadata ``` -The first implementation in `src/IB/VectorIndex.idric` uses `Float` vectors and exact dot-product search. Normalize vectors before insertion when cosine-like ranking is desired. Exact scanning is intentionally simple and gives us a correct baseline. If corpus size makes it necessary, the same rebuildable index can later gain an ANN representation such as HNSW without changing canonical product records. +The intended scalar for this index is 32-bit `Float`. The Idriç compiler revision currently pinned by IB exposes `Double` but no primitive `Float`, so the first executable baseline uses `Double` inside the rebuildable index. This is deliberately not canonical state: when Idriç gains `Float`, the index can be rebuilt with 32-bit vectors without migrating product or price records. + +The baseline uses exact dot-product search. Normalize vectors before insertion when cosine-like ranking is desired. Exact scanning is intentionally simple and gives us a correct reference implementation. If corpus size makes it necessary, the same rebuildable index can later gain an ANN representation such as HNSW without changing canonical product records. Do not derive the persistent embedding corpus from provider content whose license forbids repurposing, analysis, or long-term storage. Vectorize IB-owned/canonical product text and independently usable metadata instead. From 2eb89ee602334fc5eed9153829bc7e07fba275ca Mon Sep 17 00:00:00 2001 From: i Date: Mon, 24 Aug 2026 01:43:54 -0400 Subject: [PATCH 10/10] Assert product price and vector smoke output --- .github/workflows/idric-core.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 0ba1969..b86c923 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -67,4 +67,7 @@ jobs: grep -Fx 'cache-fake=cache' /tmp/ib-smoke.txt grep -Fx 'secret=secret' /tmp/ib-smoke.txt grep -Fx 'readable-files=2' /tmp/ib-smoke.txt - grep -Fx 'canonical-files=2' /tmp/ib-smoke.txt \ No newline at end of file + grep -Fx 'canonical-files=2' /tmp/ib-smoke.txt + grep -Fx 'asin-key=asin:B000TEST' /tmp/ib-smoke.txt + grep -Fx 'price-minor-units=1799' /tmp/ib-smoke.txt + grep -Fx 'vector-match=p-book' /tmp/ib-smoke.txt \ No newline at end of file