diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 70ef1da..b86c923 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -50,12 +50,14 @@ jobs: - name: Compile IB Idric smoke program run: | set -o pipefail - /tmp/idric/bin/idris2 src/Smoke.idric -o ib-smoke 2>&1 | tee /tmp/idric-compile.txt + cd src + /tmp/idric/bin/idris2 Smoke.idric -o ib-smoke 2>&1 | tee /tmp/idric-compile.txt test -x ./build/exec/ib-smoke ! grep -q '^Error:' /tmp/idric-compile.txt - name: Exercise browser-owned core run: | + cd src ./build/exec/ib-smoke | tee /tmp/ib-smoke.txt grep -Fx 'ingest=3' /tmp/ib-smoke.txt grep -Fx 'duplicate-url-count=2' /tmp/ib-smoke.txt @@ -66,3 +68,6 @@ jobs: grep -Fx 'secret=secret' /tmp/ib-smoke.txt grep -Fx 'readable-files=2' /tmp/ib-smoke.txt grep -Fx 'canonical-files=2' /tmp/ib-smoke.txt + grep -Fx 'asin-key=asin:B000TEST' /tmp/ib-smoke.txt + grep -Fx 'price-minor-units=1799' /tmp/ib-smoke.txt + grep -Fx 'vector-match=p-book' /tmp/ib-smoke.txt \ No newline at end of file diff --git a/docs/product-price-index.md b/docs/product-price-index.md new file mode 100644 index 0000000..2f6f8ba --- /dev/null +++ b/docs/product-price-index.md @@ -0,0 +1,110 @@ +# Product, price, and vector index + +IB should treat products and prices as durable browser-owned data rather than as state owned by a merchant renderer. + +The merchant is a source, not the object model. + +## Product identity + +A product has an IB identity and zero or more external keys: + +- ISBN +- UPC +- GTIN +- ASIN +- source-specific keys + +An ASIN is therefore useful without making Amazon the canonical namespace. + +```text +product p-0001 +key isbn:978... +key asin:B0... +term category-theory +term paperback +``` + +`src/IB/Product.idric` contains the first typed representation. + +## Price observations + +Prices are observations, not properties of a product. + +```text +observed 2026-08-23T13:42:00-04:00 +product p-0001 +source merchant-x +currency USD +minor_units 1799 +``` + +Store money in integer minor units. A price observation belongs in canonical history only when its source permits durable storage. + +Provider data with a short legal or technical lifetime is a `CurrentOffer`, not a historical observation. It belongs in disposable cache and carries an expiry time. + +This distinction lets IB keep a real price history without pretending that every provider grants a license to archive its API responses. + +## Price retrieval + +For price data, prefer the ordinary web path over a restricted merchant API: ICU fetches the page and Grease handles the retrieval/extraction workflow. If the useful price value is easy to isolate, a small grep-like extraction is enough. If the page needs structured parsing, feed the ICU-fetched text to a small Idriç parser rather than moving browser policy into shell code. + +This rule is specifically about price data. Product metadata, identifiers, outbound links, and other provider integrations may use different adapters when appropriate. + +Fetched page bodies are not the durable price ledger. Reduce them to the minimal price fact needed by the core, then apply the existing storage boundary: a permitted durable fact becomes a `PriceObservation`; data that must remain short-lived stays a `CurrentOffer` in disposable cache. + +## Amazon + +Amazon is one provider. + +Do not make the IB price path depend on Amazon Creators API offer retention. For current Amazon prices, use the ICU + Grease retrieval path above and retain only the minimal parsed price fact according to the source's storage policy. + +Amazon identifiers such as ASINs remain useful durable product keys. An Amazon adapter can still provide non-price catalog metadata or outbound-link behavior without making Amazon the canonical product namespace. + +The Amazon adapter should eventually expose roughly: + +```text +search(query) -> product candidates +item(asin) -> current metadata +outbound_link(asin, partner_tag) -> Amazon link +``` + +The core does not need to know how Amazon authentication works. + +Affiliate identity is distribution configuration, not product identity. A downstream build can supply its own partner tag or no tag. A distribution operated by one Associate can supply that Associate's tag where Amazon permits that distribution surface. + +## Vector index + +Vector search is a normal rebuildable index. + +Canonical product text and identifiers remain inspectable records. Embeddings derived from canonical data live under an index namespace, for example: + +```text +state/ + products/ + prices/ + indexes/ + products/ + / + vectors + metadata +``` + +The intended scalar for this index is 32-bit `Float`. The Idriç compiler revision currently pinned by IB exposes `Double` but no primitive `Float`, so the first executable baseline uses `Double` inside the rebuildable index. This is deliberately not canonical state: when Idriç gains `Float`, the index can be rebuilt with 32-bit vectors without migrating product or price records. + +The baseline uses exact dot-product search. Normalize vectors before insertion when cosine-like ranking is desired. Exact scanning is intentionally simple and gives us a correct reference implementation. If corpus size makes it necessary, the same rebuildable index can later gain an ANN representation such as HNSW without changing canonical product records. + +Do not derive the persistent embedding corpus from provider content whose license forbids repurposing, analysis, or long-term storage. Vectorize IB-owned/canonical product text and independently usable metadata instead. + +## Browser integration + +This follows the existing IB storage rule: canonical state is separate from rebuildable indexes and disposable caches. + +A visit to a product page can therefore connect: + +```text +visit -> URL -> product identity -> current offers + |-> durable permitted price observations + |-> rebuildable semantic vector +``` + +The UI can show a current Amazon price next to a book, tool, replacement part, or other object without making Amazon the place where the user must buy it. \ No newline at end of file diff --git a/src/IB/Product.idric b/src/IB/Product.idric new file mode 100644 index 0000000..cf4c7cb --- /dev/null +++ b/src/IB/Product.idric @@ -0,0 +1,84 @@ +module IB.Product + +%default total + +public export +data ProductKey + = ISBN String + | UPC String + | GTIN String + | ASIN String + | SourceKey String String + +public export +key_text : ProductKey → String +key_text (ISBN value) = "isbn:" ++ value +key_text (UPC value) = "upc:" ++ value +key_text (GTIN value) = "gtin:" ++ value +key_text (ASIN value) = "asin:" ++ value +key_text (SourceKey source value) = source ++ ":" ++ value + +public export +data Product = Item String (List ProductKey) (List String) + +public export +product_id : Product → String +product_id (Item value _ _) = value + +public export +product_keys : Product → List ProductKey +product_keys (Item _ keys _) = keys + +public export +product_terms : Product → List String +product_terms (Item _ _ terms) = terms + +public export +data PriceObservation = Observed String String Nat String String + +public export +price_source : PriceObservation → String +price_source (Observed source _ _ _ _) = source + +public export +price_product : PriceObservation → String +price_product (Observed _ product _ _ _) = product + +public export +price_minor_units : PriceObservation → Nat +price_minor_units (Observed _ _ amount _ _) = amount + +public export +price_currency : PriceObservation → String +price_currency (Observed _ _ _ currency _) = currency + +public export +price_observed_at : PriceObservation → String +price_observed_at (Observed _ _ _ _ observed_at) = observed_at + +public export +data CurrentOffer = Offer String ProductKey Nat String String String + +public export +offer_source : CurrentOffer → String +offer_source (Offer source _ _ _ _ _) = source + +public export +offer_key : CurrentOffer → ProductKey +offer_key (Offer _ key _ _ _ _) = key + +public export +offer_minor_units : CurrentOffer → Nat +offer_minor_units (Offer _ _ amount _ _ _) = amount + +public export +offer_currency : CurrentOffer → String +offer_currency (Offer _ _ _ currency _ _) = currency + +public export +offer_observed_at : CurrentOffer → String +offer_observed_at (Offer _ _ _ _ observed_at _) = observed_at + +public export +offer_expires_at : CurrentOffer → String +offer_expires_at (Offer _ _ _ _ _ expires_at) = expires_at diff --git a/src/IB/VectorIndex.idric b/src/IB/VectorIndex.idric new file mode 100644 index 0000000..5da622f --- /dev/null +++ b/src/IB/VectorIndex.idric @@ -0,0 +1,42 @@ +module IB.VectorIndex + +%default total + +-- The pinned Idriç compiler does not yet expose a primitive 32-bit Float. +-- This index is rebuildable, so using Double here does not affect canonical +-- browser state and can be replaced by Float without a state migration. +public export +data VectorRow = Row String (List Double) + +public export +vector_id : VectorRow → String +vector_id (Row value _) = value + +public export +vector_values : VectorRow → List Double +vector_values (Row _ values) = values + +public export +dot : List Double → List Double → Double +dot [] _ = 0.0 +dot _ [] = 0.0 +dot (left :: more_left) (right :: more_right) = + left * right + dot more_left more_right + +public export +score : List Double → VectorRow → Double +score query row = dot query (vector_values row) + +public export +best_match : List Double → List VectorRow → Maybe (String, Double) +best_match query [] = Nothing +best_match query (first :: rest) = + go first (score query first) rest + where + go : VectorRow → Double → List VectorRow → Maybe (String, Double) + go best best_score [] = Just (vector_id best, best_score) + go best best_score (candidate :: more) = + let candidate_score = score query candidate in + if candidate_score > best_score + then go candidate candidate_score more + else go best best_score more \ No newline at end of file diff --git a/src/Smoke.idric b/src/Smoke.idric index fdb5e4e..3c07578 100644 --- a/src/Smoke.idric +++ b/src/Smoke.idric @@ -4,6 +4,8 @@ import IB.History import IB.Index import IB.Storage import IB.Inspect +import IB.Product +import IB.VectorIndex %default total @@ -15,14 +17,26 @@ sample_entries = [ Entry "https://missing-time.test/c" Nothing Nothing raw_urls visit Nothing 93 ] +sample_vectors : List VectorRow +sample_vectors = [ + Row "p-book" [1.0, 0.0, 0.0], + Row "p-tool" [0.0, 1.0, 0.0], + Row "p-part" [0.0, 0.0, 1.0] + ] + first_or : Nat → List Nat → Nat first_or fallback [] = fallback first_or fallback (value :: _) = value +match_id : Maybe (String, Double) → String +match_id Nothing = "none" +match_id (Just (value, _)) = value + main : IO () main = do let raw = ingest_raw_lines ["# ignored", " https://example.test/a ", "https://example.test/a", "https://other.test/b"] let indices = build_indices sample_entries + let price = Observed "permitted-history-source" "p-book" 1799 "USD" "2026-08-23T13:42:00-04:00" let files = [ FileRow "visits.jsonl" 20 regular_file, FileRow "indexes/summary.json" 10 regular_file, @@ -39,3 +53,6 @@ main = do putStrLn ("secret=" ++ storage_kind_text (classify_path "renderer/passwords")) putStrLn ("readable-files=" ++ show (readable_file_count files)) putStrLn ("canonical-files=" ++ show (files_of_kind canonical files)) + putStrLn ("asin-key=" ++ key_text (ASIN "B000TEST")) + putStrLn ("price-minor-units=" ++ show (price_minor_units price)) + putStrLn ("vector-match=" ++ match_id (best_match [0.9, 0.1, 0.0] sample_vectors)) \ No newline at end of file