diff --git a/baml_language/crates/baml_builtins2/keyword_docs/baml_keywords.yaml b/baml_language/crates/baml_builtins2/keyword_docs/baml_keywords.yaml index c76d65878da..dfff5ab9a3f 100644 --- a/baml_language/crates/baml_builtins2/keyword_docs/baml_keywords.yaml +++ b/baml_language/crates/baml_builtins2/keyword_docs/baml_keywords.yaml @@ -1262,7 +1262,10 @@ query: ended process: invocations, outcomes, total/self/I/O time). Value columns support ordinary subscripts and comparisons — `input_args['customer']['age'] >= 30`, - `input_args = baml_value_json('{"x":1}')` — decoded on demand. `WITH + `input_args = baml_value_json('{"x":1}')` — decoded on demand. An image, + audio, pdf or video value is described without its content + (`{"$media", "mime", "base64_len"}`); `['base64']` on it reads the content + as base64 text. `WITH RECURSIVE` walks span trees through `parent_span_id`. Discovery: `--schema`, `--explain`. Output: `--format table|json|jsonl` (jsonl ends with an outcome frame). Exit codes: 0 complete, 1 evidence-incomplete, diff --git a/baml_language/crates/baml_query_btel/tests/values.rs b/baml_language/crates/baml_query_btel/tests/values.rs index eca6a5185a0..0c78a4a382a 100644 --- a/baml_language/crates/baml_query_btel/tests/values.rs +++ b/baml_language/crates/baml_query_btel/tests/values.rs @@ -410,3 +410,80 @@ async fn captured_images_store_each_content_once_and_render_as_descriptors() { } assert_eq!(result.rows.len(), 2); } + +/// A media value renders as a descriptor without its content; a path that +/// ends in `base64` on it reads the content, from the value's own blob or +/// from the child blob a large content lives in. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn base64_on_a_media_value_reads_its_content() { + let project = tempfile::tempdir().unwrap(); + let source = r#" + function Show(small: image, large: image, linked: image) -> int { 0 } + function main(n: int) -> int { + let chunk = "iVBORw0K"; + let i = 0; + while (i < 12) { + chunk = chunk + chunk; + i = i + 1; + } + Show( + baml.media.Image.from_base64("iVBORw0KGgo=", "image/png"), + baml.media.Image.from_base64(chunk + "a", "image/png"), + baml.media.Image.from_url("https://example.com/cat.png", "image/png"), + ); + n + } + "#; + let results = record_program(project.path(), source, &["Show"], &[("main", 0)]).await; + assert_eq!(results, vec![Ok(bex_engine::BexExternalValue::Int(0))]); + // The large content is past the leaf size: it is a blob of its own. + let sizes = blob_sizes(project.path()); + assert_eq!( + sizes.iter().filter(|size| **size >= 32 << 10).count(), + 1, + "{sizes:?}" + ); + + let mut index = index(project.path()); + let result = sql( + &mut index, + "SELECT input_args['small']['_data']['base64'] AS small, + input_args['large']['_data']['base64'] AS large, + input_args['linked']['_data']['base64'] AS linked, + baml_value_state(input_args['linked']['_data']['base64']) AS linked_state, + baml_value_state(input_args['small']['_data']['mime']) AS other_key, + input_args['small']['_data'] AS described, + input_args['linked']['_data'] AS described_link + FROM spans WHERE span_name = 'user.Show'", + ); + assert_eq!( + result.outcome.status, + Status::Complete, + "{:?}", + result.outcome.diagnostics + ); + assert_eq!(result.rows.len(), 1); + let row = &result.rows[0]; + assert_eq!(row[0], json!("iVBORw0KGgo=")); + assert_eq!(row[1], json!("iVBORw0K".repeat(1 << 12) + "a")); + // Media that names a URL holds no content, and no other key reads media. + assert_eq!( + row[2..5], + [serde_json::Value::Null, json!("missing"), json!("missing")] + ); + // The value itself is still described without its content. + assert_eq!( + row[5], + json!({"$media": "image", "mime": "image/png", "base64_len": 12}) + ); + assert_eq!(row[6]["url"], json!("https://example.com/cat.png")); + assert_eq!(row[6].get("base64_len"), None); + + // The content is a value like any other text: a predicate reads it. + let found = sql( + &mut index, + "SELECT COUNT(*) FROM spans + WHERE span_name = 'user.Show' AND input_args['small']['_data']['base64'] = 'iVBORw0KGgo='", + ); + assert_eq!(found.rows, vec![vec![json!(1)]]); +} diff --git a/baml_language/crates/btel_reader/src/value.rs b/baml_language/crates/btel_reader/src/value.rs index 5259205b3e1..4afa3f05544 100644 --- a/baml_language/crates/btel_reader/src/value.rs +++ b/baml_language/crates/btel_reader/src/value.rs @@ -40,6 +40,11 @@ pub enum Segment { Index(i64), } +/// The key that reads a media value's content: `value['photo']['base64']` is the content of +/// the image at `photo` as base64 text, or absent when the value names a URL or a file and +/// holds no content. +pub const MEDIA_CONTENT: &str = "base64"; + /// Which root of a snapshot a column refers to. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Root { @@ -250,6 +255,12 @@ fn step<'a>( (DecodedObject::Truncated(limit), _) => { Err(Nav::Unavailable(Unavailable::Truncated(*limit))) } + // A value's rendering describes its media and leaves the content out, so that a row + // of a thousand runs is not a thousand pictures. A path asks for one picture's + // content by name. + (DecodedObject::Media(media), Segment::Key(key)) if key == MEDIA_CONTENT => Err( + span::media_text(source, current, media.source.data(), blobs_left), + ), (DecodedObject::List { .. }, Segment::Key(_)) | (DecodedObject::Instance { .. }, Segment::Index(_)) | ( @@ -821,7 +832,7 @@ pub fn render_code( /// reason's code. Media is described, never rendered: /// `{"$media": kind, "mime", "url" | "file", "base64_len"}`, with /// `base64_len` present when the value holds content, and its content is not -/// read. A part past a render limit is `$truncated` with `render_depth` or +/// read; a path that ends in [`MEDIA_CONTENT`] on the media value reads it. A part past a render limit is `$truncated` with `render_depth` or /// `render_size`: alone in place of a value or a piece of text, or beside /// what an object rendered before it ended. This is a display format, not a /// lossless serialization. diff --git a/baml_language/crates/btel_reader/src/value/span.rs b/baml_language/crates/btel_reader/src/value/span.rs index 62da96606d3..3ac4dabbe5c 100644 --- a/baml_language/crates/btel_reader/src/value/span.rs +++ b/baml_language/crates/btel_reader/src/value/span.rs @@ -150,6 +150,32 @@ pub fn media_content( } } +/// What a path that ends in [`MEDIA_CONTENT`] on a media value finds: its content as base64 +/// text, read from its own blob when it is stored there, which spends one of `blobs_left`. +/// A media value that holds no content, one that only names a URL or a file, has none. +pub(super) fn media_text( + source: &(impl BlobSource + ?Sized), + media: &Located, + payload: Option<&MediaPayload>, + blobs_left: &mut usize, +) -> Nav { + let content = match payload { + None => return Nav::Missing, + Some(MediaPayload::Inline(text)) => Ok(Arc::clone(text)), + Some(MediaPayload::External { child, text_len }) => media + .blob + .children + .get(child.0 as usize) + .ok_or(BROKEN_REFERENCE) + .and_then(|id| load(source, *id, blobs_left)) + .and_then(|child| child_content(&child, *text_len).cloned()), + }; + match content { + Ok(text) => found(Arc::clone(&media.blob), Found::String(text)), + Err(reason) => Nav::Unavailable(reason), + } +} + fn child_node(child: &DecodedSnapshot, node: NodeId) -> Result { if (node.0 as usize) < child.objects.len() { Ok(node) diff --git a/baml_language/crates/btel_reader/src/value/span/tests.rs b/baml_language/crates/btel_reader/src/value/span/tests.rs index 3c72513beaf..352c4113fa0 100644 --- a/baml_language/crates/btel_reader/src/value/span/tests.rs +++ b/baml_language/crates/btel_reader/src/value/span/tests.rs @@ -275,6 +275,36 @@ fn media_is_described_without_its_content_and_compared_by_it() { assert_eq!(equal(&blobs, &link(&split), &link(&other)), Ok(Some(true))); } +#[test] +fn a_path_that_ends_in_base64_on_a_media_value_finds_its_content() { + let mut blobs = Blobs::default(); + let content = "iVBORw0K".repeat(20); + // Stored in the value's own blob, and in a blob of its own. + let whole = blobs.store(&capture(&mut Shaper::new(ShapePolicy::Whole), |b| { + album(b, &content) + })); + let cut = capture(&mut cut(), |b| album(b, &content)); + let content_blob = cut.blobs().next().unwrap().id(); + let split = blobs.store(&cut); + for root in [&whole, &split] { + let Nav::Value(found) = at(&blobs, root, &["photo", "base64"]) else { + panic!("the content is found"); + }; + assert_eq!(found.value(), &Found::String(content.as_str().into())); + // No other key reads a media value, and its content has nothing under it. + assert_eq!(at(&blobs, root, &["photo", "mime"]), Nav::Missing); + assert_eq!(at(&blobs, root, &["photo", "base64", "x"]), Nav::Missing); + } + // A media value that names a URL holds no content. + assert_eq!(at(&blobs, &whole, &["link", "base64"]), Nav::Missing); + // Content whose blob is gone is unavailable, with the blob's code. + blobs.0.remove(&content_blob); + assert_eq!( + at(&blobs, &split, &["photo", "base64"]), + Nav::Unavailable(Unavailable::Blob("cas_missing")) + ); +} + #[test] fn media_content_is_read_from_its_blob_and_checked_against_its_length() { let mut blobs = Blobs::default();