Read a media value's content with ['base64']

baml_language/crates/baml_builtins2/keyword_docs/baml_keywords.yaml

@@ -1262,7 +1262,10 @@ query:
     ended process: invocations, outcomes, total/self/I/O time). Value columns
     support ordinary subscripts and comparisons —
     `input_args['customer']['age'] >= 30`,
-    `input_args = baml_value_json('{"x":1}')` — decoded on demand. `WITH
+    `input_args = baml_value_json('{"x":1}')` — decoded on demand. An image,
+    audio, pdf or video value is described without its content
+    (`{"$media", "mime", "base64_len"}`); `['base64']` on it reads the content
+    as base64 text. `WITH
     RECURSIVE` walks span trees through `parent_span_id`. Discovery:
     `--schema`, `--explain`. Output: `--format table|json|jsonl` (jsonl ends
     with an outcome frame). Exit codes: 0 complete, 1 evidence-incomplete,

baml_language/crates/baml_query_btel/tests/values.rs

@@ -410,3 +410,80 @@ async fn captured_images_store_each_content_once_and_render_as_descriptors() {
     }
     assert_eq!(result.rows.len(), 2);
 }
+
+/// A media value renders as a descriptor without its content; a path that
+/// ends in `base64` on it reads the content, from the value's own blob or
+/// from the child blob a large content lives in.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn base64_on_a_media_value_reads_its_content() {
+    let project = tempfile::tempdir().unwrap();
+    let source = r#"
+        function Show(small: image, large: image, linked: image) -> int { 0 }
+        function main(n: int) -> int {
+            let chunk = "iVBORw0K";
+            let i = 0;
+            while (i < 12) {
+                chunk = chunk + chunk;
+                i = i + 1;
+            }
+            Show(
+                baml.media.Image.from_base64("iVBORw0KGgo=", "image/png"),
+                baml.media.Image.from_base64(chunk + "a", "image/png"),
+                baml.media.Image.from_url("https://example.com/cat.png", "image/png"),
+            );
+            n
+        }
+    "#;
+    let results = record_program(project.path(), source, &["Show"], &[("main", 0)]).await;
+    assert_eq!(results, vec![Ok(bex_engine::BexExternalValue::Int(0))]);
+    // The large content is past the leaf size: it is a blob of its own.
+    let sizes = blob_sizes(project.path());
+    assert_eq!(
+        sizes.iter().filter(|size| **size >= 32 << 10).count(),
+        1,
+        "{sizes:?}"
+    );
+
+    let mut index = index(project.path());
+    let result = sql(
+        &mut index,
+        "SELECT input_args['small']['_data']['base64'] AS small,
+           input_args['large']['_data']['base64'] AS large,
+           input_args['linked']['_data']['base64'] AS linked,
+           baml_value_state(input_args['linked']['_data']['base64']) AS linked_state,
+           baml_value_state(input_args['small']['_data']['mime']) AS other_key,
+           input_args['small']['_data'] AS described,
+           input_args['linked']['_data'] AS described_link
+         FROM spans WHERE span_name = 'user.Show'",
+    );
+    assert_eq!(
+        result.outcome.status,
+        Status::Complete,
+        "{:?}",
+        result.outcome.diagnostics
+    );
+    assert_eq!(result.rows.len(), 1);
+    let row = &result.rows[0];
+    assert_eq!(row[0], json!("iVBORw0KGgo="));
+    assert_eq!(row[1], json!("iVBORw0K".repeat(1 << 12) + "a"));
+    // Media that names a URL holds no content, and no other key reads media.
+    assert_eq!(
+        row[2..5],
+        [serde_json::Value::Null, json!("missing"), json!("missing")]
+    );
+    // The value itself is still described without its content.
+    assert_eq!(
+        row[5],
+        json!({"$media": "image", "mime": "image/png", "base64_len": 12})
+    );
+    assert_eq!(row[6]["url"], json!("https://example.com/cat.png"));
+    assert_eq!(row[6].get("base64_len"), None);
+
+    // The content is a value like any other text: a predicate reads it.
+    let found = sql(
+        &mut index,
+        "SELECT COUNT(*) FROM spans
+         WHERE span_name = 'user.Show' AND input_args['small']['_data']['base64'] = 'iVBORw0KGgo='",
+    );
+    assert_eq!(found.rows, vec![vec![json!(1)]]);
+}

baml_language/crates/btel_reader/src/value.rs

@@ -40,6 +40,11 @@ pub enum Segment {
     Index(i64),
 }
 
+/// The key that reads a media value's content: `value['photo']['base64']` is the content of
+/// the image at `photo` as base64 text, or absent when the value names a URL or a file and
+/// holds no content.
+pub const MEDIA_CONTENT: &str = "base64";
+
 /// Which root of a snapshot a column refers to.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum Root {
@@ -250,6 +255,12 @@ fn step<'a>(
         (DecodedObject::Truncated(limit), _) => {
             Err(Nav::Unavailable(Unavailable::Truncated(*limit)))
         }
+        // A value's rendering describes its media and leaves the content out, so that a row
+        // of a thousand runs is not a thousand pictures. A path asks for one picture's
+        // content by name.
+        (DecodedObject::Media(media), Segment::Key(key)) if key == MEDIA_CONTENT => Err(
+            span::media_text(source, current, media.source.data(), blobs_left),
+        ),
         (DecodedObject::List { .. }, Segment::Key(_))
         | (DecodedObject::Instance { .. }, Segment::Index(_))
         | (
@@ -821,7 +832,7 @@ pub fn render_code(
 /// reason's code. Media is described, never rendered:
 /// `{"$media": kind, "mime", "url" | "file", "base64_len"}`, with
 /// `base64_len` present when the value holds content, and its content is not
-/// read. A part past a render limit is `$truncated` with `render_depth` or
+/// read; a path that ends in [`MEDIA_CONTENT`] on the media value reads it. A part past a render limit is `$truncated` with `render_depth` or
 /// `render_size`: alone in place of a value or a piece of text, or beside
 /// what an object rendered before it ended. This is a display format, not a
 /// lossless serialization.

baml_language/crates/btel_reader/src/value/span.rs

@@ -150,6 +150,32 @@ pub fn media_content(
     }
 }
 
+/// What a path that ends in [`MEDIA_CONTENT`] on a media value finds: its content as base64
+/// text, read from its own blob when it is stored there, which spends one of `blobs_left`.
+/// A media value that holds no content, one that only names a URL or a file, has none.
+pub(super) fn media_text(
+    source: &(impl BlobSource + ?Sized),
+    media: &Located,
+    payload: Option<&MediaPayload>,
+    blobs_left: &mut usize,
+) -> Nav {
+    let content = match payload {
+        None => return Nav::Missing,
+        Some(MediaPayload::Inline(text)) => Ok(Arc::clone(text)),
+        Some(MediaPayload::External { child, text_len }) => media
+            .blob
+            .children
+            .get(child.0 as usize)
+            .ok_or(BROKEN_REFERENCE)
+            .and_then(|id| load(source, *id, blobs_left))
+            .and_then(|child| child_content(&child, *text_len).cloned()),
+    };
+    match content {
+        Ok(text) => found(Arc::clone(&media.blob), Found::String(text)),
+        Err(reason) => Nav::Unavailable(reason),
+    }
+}
+
 fn child_node(child: &DecodedSnapshot, node: NodeId) -> Result<NodeId, Unavailable> {
     if (node.0 as usize) < child.objects.len() {
         Ok(node)

baml_language/crates/btel_reader/src/value/span/tests.rs

@@ -275,6 +275,36 @@ fn media_is_described_without_its_content_and_compared_by_it() {
     assert_eq!(equal(&blobs, &link(&split), &link(&other)), Ok(Some(true)));
 }
 
+#[test]
+fn a_path_that_ends_in_base64_on_a_media_value_finds_its_content() {
+    let mut blobs = Blobs::default();
+    let content = "iVBORw0K".repeat(20);
+    // Stored in the value's own blob, and in a blob of its own.
+    let whole = blobs.store(&capture(&mut Shaper::new(ShapePolicy::Whole), |b| {
+        album(b, &content)
+    }));
+    let cut = capture(&mut cut(), |b| album(b, &content));
+    let content_blob = cut.blobs().next().unwrap().id();
+    let split = blobs.store(&cut);
+    for root in [&whole, &split] {
+        let Nav::Value(found) = at(&blobs, root, &["photo", "base64"]) else {
+            panic!("the content is found");
+        };
+        assert_eq!(found.value(), &Found::String(content.as_str().into()));
+        // No other key reads a media value, and its content has nothing under it.
+        assert_eq!(at(&blobs, root, &["photo", "mime"]), Nav::Missing);
+        assert_eq!(at(&blobs, root, &["photo", "base64", "x"]), Nav::Missing);
+    }
+    // A media value that names a URL holds no content.
+    assert_eq!(at(&blobs, &whole, &["link", "base64"]), Nav::Missing);
+    // Content whose blob is gone is unavailable, with the blob's code.
+    blobs.0.remove(&content_blob);
+    assert_eq!(
+        at(&blobs, &split, &["photo", "base64"]),
+        Nav::Unavailable(Unavailable::Blob("cas_missing"))
+    );
+}
+
 #[test]
 fn media_content_is_read_from_its_blob_and_checked_against_its_length() {
     let mut blobs = Blobs::default();